diff --git a/Cargo.lock b/Cargo.lock index 3846476f9..75cde0307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - [[package]] name = "aead" version = "0.5.2" @@ -80,7 +74,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -92,7 +86,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.15", "once_cell", "serde", "version_check", @@ -284,7 +278,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "synstructure 0.13.1", ] @@ -307,7 +301,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -323,6 +317,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-recursion" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -331,7 +336,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -353,7 +358,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -364,7 +369,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -420,6 +425,34 @@ dependencies = [ "thiserror", ] +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.30", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.7.5" @@ -427,7 +460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.3", "bytes", "futures-util", "http 1.1.0", @@ -447,6 +480,23 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.3" @@ -477,7 +527,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", ] @@ -565,7 +615,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.75", + "syn 2.0.72", "which", ] @@ -584,7 +634,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -735,7 +785,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "syn_derive", ] @@ -962,9 +1012,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -972,9 +1022,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstream", "anstyle", @@ -991,7 +1041,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1135,7 +1185,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -1282,7 +1332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array 0.14.7", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -1294,7 +1344,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array 0.14.7", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -1362,7 +1412,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1410,7 +1460,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1432,7 +1482,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1495,7 +1545,7 @@ checksum = "1ab8a4ea925ce79678034870834602a2980f4b88c09e97feb266496dbb4493d2" dependencies = [ "async-trait", "deadpool 0.12.1", - "getrandom", + "getrandom 0.2.15", "tokio", "tokio-postgres", "tracing", @@ -1576,6 +1626,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_arbitrary" version = "1.3.2" @@ -1584,7 +1645,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1605,7 +1666,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1615,7 +1676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1713,7 +1774,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1839,7 +1900,7 @@ checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", - "rand_core", + "rand_core 0.6.4", "serde", "sha2 0.10.8", "subtle", @@ -1887,7 +1948,7 @@ dependencies = [ "hkdf", "pem-rfc7468", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -1938,7 +1999,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -1966,6 +2027,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "fail" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3c61c59fdc91f5dbc3ea31ee8623122ce80057058be560654c5d410d181a6" +dependencies = [ + "lazy_static", + "log", + "rand 0.7.3", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -2007,13 +2079,19 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + [[package]] name = "ff" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2031,13 +2109,13 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.32" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -2092,14 +2170,14 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "020bf4ae7238dbdb1ff01e9f981db028515cf66883c461e29faedfea130b2728" dependencies = [ - "async-recursion", + "async-recursion 1.1.1", "async-trait", "foundationdb-gen", "foundationdb-macros", "foundationdb-sys", "futures", "memchr", - "rand", + "rand 0.8.5", "serde", "serde_bytes", "serde_json", @@ -2124,7 +2202,7 @@ checksum = "f8db6653cbc621a3810d95d55bd342be3e71181d6df21a4eb29ef986202d3f9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "try_map", ] @@ -2163,7 +2241,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -2175,7 +2253,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -2187,7 +2265,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -2252,7 +2330,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -2324,6 +2402,17 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -2333,7 +2422,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2366,7 +2455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2479,7 +2568,7 @@ dependencies = [ "hickory-proto", "once_cell", "radix_trie", - "rand", + "rand 0.8.5", "rustls 0.21.12", "thiserror", "tokio", @@ -2505,7 +2594,7 @@ dependencies = [ "idna 0.4.0", "ipnet", "once_cell", - "rand", + "rand 0.8.5", "ring 0.16.20", "rustls 0.21.12", "rustls-pemfile 1.0.4", @@ -2530,7 +2619,7 @@ dependencies = [ "lru-cache", "once_cell", "parking_lot", - "rand", + "rand 0.8.5", "resolv-conf", "rustls 0.21.12", "smallvec", @@ -2750,6 +2839,18 @@ dependencies = [ "webpki-roots 0.26.3", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.30", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-timeout" version = "0.5.1" @@ -2763,6 +2864,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.4.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.7" @@ -2921,7 +3038,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -2993,7 +3110,7 @@ dependencies = [ "md5", "nlp", "parking_lot", - "rand", + "rand 0.8.5", "rustls 0.23.12", "rustls-pemfile 2.1.3", "store", @@ -3224,7 +3341,7 @@ dependencies = [ "p256", "pkcs8", "quick-xml 0.36.1", - "rand", + "rand 0.8.5", "rasn", "rasn-cms", "rasn-pkix", @@ -3582,7 +3699,7 @@ dependencies = [ "mail-parser", "parking_lot", "quick-xml 0.32.0", - "rand", + "rand 0.8.5", "ring 0.17.8", "rsa", "rustls-pemfile 2.1.3", @@ -3698,7 +3815,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -3790,15 +3907,6 @@ dependencies = [ "adler", ] -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" -dependencies = [ - "adler2", -] - [[package]] name = "mio" version = "0.8.11" @@ -3807,7 +3915,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3819,7 +3927,7 @@ checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi 0.3.9", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -3836,7 +3944,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "termcolor", "thiserror", ] @@ -3862,7 +3970,7 @@ dependencies = [ "pem", "percent-encoding", "pin-project", - "rand", + "rand 0.8.5", "rustls 0.22.4", "rustls-pemfile 2.1.3", "serde", @@ -3901,7 +4009,7 @@ dependencies = [ "mysql-common-derive", "num-bigint", "num-traits", - "rand", + "rand 0.8.5", "regex", "rust_decimal", "saturating", @@ -3917,6 +4025,23 @@ dependencies = [ "zstd", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -4005,7 +4130,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -4124,7 +4249,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -4185,11 +4310,11 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost", + "prost 0.13.1", "reqwest 0.12.7", "thiserror", "tokio", - "tonic", + "tonic 0.12.1", ] [[package]] @@ -4200,8 +4325,8 @@ checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost", - "tonic", + "prost 0.13.1", + "tonic 0.12.1", ] [[package]] @@ -4224,7 +4349,7 @@ dependencies = [ "once_cell", "opentelemetry", "percent-encoding", - "rand", + "rand 0.8.5", "serde_json", "thiserror", ] @@ -4273,7 +4398,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "rand_core", + "rand_core 0.6.4", "sha2 0.10.8", ] @@ -4307,7 +4432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -4391,7 +4516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared 0.11.2", - "rand", + "rand 0.8.5", ] [[package]] @@ -4404,7 +4529,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -4442,7 +4567,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -4533,7 +4658,7 @@ dependencies = [ "hmac 0.12.1", "md-5 0.10.6", "memchr", - "rand", + "rand 0.8.5", "sha2 0.10.8", "stringprep", ] @@ -4577,7 +4702,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -4655,6 +4780,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags 2.6.0", + "hex", + "lazy_static", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags 2.6.0", + "hex", +] + [[package]] name = "prometheus" version = "0.13.4" @@ -4664,11 +4812,25 @@ dependencies = [ "cfg-if", "fnv", "lazy_static", + "libc", "memchr", "parking_lot", + "procfs", + "protobuf", + "reqwest 0.12.7", "thiserror", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive 0.12.6", +] + [[package]] name = "prost" version = "0.13.1" @@ -4676,7 +4838,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.1", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.72", ] [[package]] @@ -4689,9 +4864,15 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "proxy-header" version = "0.1.2" @@ -4732,7 +4913,7 @@ dependencies = [ "byteorder", "hmac 0.10.1", "md-5 0.9.1", - "rand", + "rand 0.8.5", "sha-1", "sha2 0.9.9", ] @@ -4787,7 +4968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.0.0", "rustls 0.23.12", @@ -4846,6 +5027,19 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -4853,8 +5047,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -4864,7 +5068,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -4873,7 +5086,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -4995,7 +5217,7 @@ dependencies = [ "num-bigint", "percent-encoding", "pin-project-lite", - "rand", + "rand 0.8.5", "rustls 0.23.12", "rustls-native-certs 0.7.2", "rustls-pemfile 2.1.3", @@ -5034,7 +5256,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror", ] @@ -5107,7 +5329,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-rustls 0.24.1", "tokio-util", @@ -5129,6 +5351,7 @@ checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-channel", "futures-core", "futures-util", @@ -5138,12 +5361,14 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "hyper-rustls 0.27.2", + "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", "mime_guess", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -5155,7 +5380,9 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", + "system-configuration 0.6.0", "tokio", + "tokio-native-tls", "tokio-rustls 0.26.0", "tower-service", "url", @@ -5218,7 +5445,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -5307,7 +5534,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", @@ -5412,7 +5639,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand", + "rand 0.8.5", "rkyv", "serde", "serde_json", @@ -5625,9 +5852,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "scc" -version = "2.1.16" +version = "2.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeb7ac86243095b70a7920639507b71d51a63390d1ba26c4f60a552fbb914a37" +checksum = "05ccfb12511cdb770157ace92d7dda771e498445b78f9886e8cdbc5140a4eced" dependencies = [ "sdd", ] @@ -5680,9 +5907,9 @@ dependencies = [ [[package]] name = "sdd" -version = "3.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0495e4577c672de8254beb68d01a9b62d0e8a13c099edecdbedccce3223cd29f" +checksum = "177258b64c0faaa9ffd3c65cd3262c2bc7e2588dbbd9c1641d0346145c1bbda8" [[package]] name = "seahash" @@ -5775,7 +6002,7 @@ dependencies = [ "ecdsa", "ed25519", "ed25519-dalek", - "getrandom", + "getrandom 0.2.15", "idea", "idna 1.0.2", "lalrpop", @@ -5789,8 +6016,8 @@ dependencies = [ "p256", "p384", "p521", - "rand", - "rand_core", + "rand 0.8.5", + "rand_core 0.6.4", "regex", "regex-syntax", "ripemd", @@ -5806,9 +6033,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.208" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] @@ -5824,20 +6051,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.208" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" dependencies = [ "itoa", "memchr", @@ -5901,7 +6128,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -6006,7 +6233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest 0.10.7", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -6072,7 +6299,7 @@ dependencies = [ "nlp", "num_cpus", "parking_lot", - "rand", + "rand 0.8.5", "rayon", "regex", "reqwest 0.12.7", @@ -6181,7 +6408,7 @@ dependencies = [ "num_cpus", "prettytable-rs", "pwhash", - "rand", + "rand 0.8.5", "reqwest 0.12.7", "rpassword", "serde", @@ -6220,7 +6447,7 @@ dependencies = [ "num_cpus", "parking_lot", "r2d2", - "rand", + "rand 0.8.5", "rayon", "redis", "regex", @@ -6233,6 +6460,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", + "tikv-client", "tokio", "tokio-postgres", "tokio-rustls 0.26.0", @@ -6306,9 +6534,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.75" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -6324,7 +6552,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -6362,7 +6590,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -6373,7 +6601,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -6386,12 +6625,35 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "term" version = "0.7.0" @@ -6480,7 +6742,34 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", +] + +[[package]] +name = "tikv-client" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048968e4e3d04db472346770cc19914c6b5ae206fa44677f6a0874d54cd05940" +dependencies = [ + "async-recursion 0.3.2", + "async-trait", + "derive-new", + "either", + "fail", + "futures", + "lazy_static", + "log", + "pin-project", + "prometheus", + "prost 0.12.6", + "rand 0.8.5", + "regex", + "semver 1.0.23", + "serde", + "serde_derive", + "thiserror", + "tokio", + "tonic 0.10.2", ] [[package]] @@ -6576,6 +6865,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.4.0" @@ -6584,7 +6883,17 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", ] [[package]] @@ -6606,7 +6915,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand", + "rand 0.8.5", "socket2", "tokio", "tokio-util", @@ -6714,6 +7023,36 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.30", + "hyper-timeout 0.4.1", + "percent-encoding", + "pin-project", + "prost 0.12.6", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic" version = "0.12.1" @@ -6722,7 +7061,7 @@ checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.7.5", "base64 0.22.1", "bytes", "h2 0.4.6", @@ -6730,11 +7069,11 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.4.1", - "hyper-timeout", + "hyper-timeout 0.5.1", "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.1", "socket2", "tokio", "tokio-stream", @@ -6770,7 +7109,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -6810,7 +7149,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -6870,7 +7209,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand", + "rand 0.8.5", "rustls 0.22.4", "rustls-pki-types", "sha1", @@ -6891,7 +7230,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "utf-8", @@ -6913,7 +7252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand", + "rand 0.8.5", "static_assertions", ] @@ -7083,7 +7422,7 @@ dependencies = [ "parking_lot", "pem", "privdrop", - "rand", + "rand 0.8.5", "rcgen 0.13.1", "regex", "reqwest 0.12.7", @@ -7107,7 +7446,7 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -7147,6 +7486,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -7181,7 +7526,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -7215,7 +7560,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7588,7 +7933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "zeroize", ] @@ -7667,7 +8012,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "synstructure 0.13.1", ] @@ -7689,7 +8034,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -7709,7 +8054,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", "synstructure 0.13.1", ] @@ -7730,7 +8075,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -7752,7 +8097,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.72", ] [[package]] @@ -7775,7 +8120,7 @@ dependencies = [ "lzma-rs", "memchr", "pbkdf2", - "rand", + "rand 0.8.5", "sha1", "thiserror", "time", diff --git a/README.md b/README.md index b6e19643e..257f11e7f 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Key features: - **Greylisting** to temporarily defer unknown senders. - **Spam traps** to set up decoy email addresses that catch and analyze spam. - **Flexible and scalable**: - - Pluggable storage backends with **RocksDB**, **FoundationDB**, **PostgreSQL**, **mySQL**, **SQLite**, **S3-Compatible**, **Redis** and **ElasticSearch** support. + - Pluggable storage backends with **RocksDB**, **FoundationDB**, **FoundationDB**, **TiKV**, **PostgreSQL**, **mySQL**, **SQLite**, **S3-Compatible**, **Redis** and **ElasticSearch** support. - **Clustering** support with node autodiscovery and partition-tolerant failure detection. - Built-in, **LDAP** or **SQL** authentication backend support. - Full-text search available in 17 languages. diff --git a/crates/common/src/telemetry/metrics/prometheus.rs b/crates/common/src/telemetry/metrics/prometheus.rs index 35ff3c016..836a56420 100644 --- a/crates/common/src/telemetry/metrics/prometheus.rs +++ b/crates/common/src/telemetry/metrics/prometheus.rs @@ -28,7 +28,7 @@ impl Core { metric.set_name(metric_name(counter.id().name())); metric.set_help(counter.id().description().into()); metric.set_field_type(MetricType::COUNTER); - metric.set_metric(vec![new_counter(counter.value())]); + metric.set_metric(vec![new_counter(counter.value())].into()); metrics.push(metric); } @@ -38,7 +38,7 @@ impl Core { metric.set_name(metric_name(gauge.id().name())); metric.set_help(gauge.id().description().into()); metric.set_field_type(MetricType::GAUGE); - metric.set_metric(vec![new_gauge(gauge.get())]); + metric.set_metric(vec![new_gauge(gauge.get())].into()); metrics.push(metric); } @@ -48,7 +48,7 @@ impl Core { metric.set_name(metric_name(histogram.id().name())); metric.set_help(histogram.id().description().into()); metric.set_field_type(MetricType::HISTOGRAM); - metric.set_metric(vec![new_histogram(histogram)]); + metric.set_metric(vec![new_histogram(histogram)].into()); metrics.push(metric); } diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index e09d9acc0..bacf9710e 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -34,13 +34,14 @@ tokio = { version = "1.23", features = ["full"] } jemallocator = "0.5.0" [features] -default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "enterprise"] -#default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "foundationdb", "enterprise"] +default = ["sqlite", "postgres", "mysql", "rocks", "tikv", "elastic", "s3", "redis", "enterprise"] +#default = ["sqlite", "postgres", "mysql", "rocks", "tikv", "elastic", "s3", "redis", "foundationdb", "enterprise"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] mysql = ["store/mysql"] rocks = ["store/rocks"] +tikv = ["store/tikv"] elastic = ["store/elastic"] s3 = ["store/s3"] redis = ["store/redis"] diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index cfe22bb67..a0c692e5f 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -11,6 +11,7 @@ trc = { path = "../trc" } rocksdb = { version = "0.22", optional = true, features = ["multi-threaded-cf"] } foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_1"], optional = true } rusqlite = { version = "0.32", features = ["bundled"], optional = true } +tikv-client = { version = "0.3.0", optional = true } rust-s3 = { version = "=0.35.0-alpha.2", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true } tokio = { version = "1.23", features = ["sync", "fs", "io-util"] } r2d2 = { version = "0.8.10", optional = true } @@ -57,6 +58,7 @@ elastic = ["elasticsearch", "serde_json"] mysql = ["mysql_async", "futures"] s3 = ["rust-s3"] foundation = ["foundationdb", "futures"] +tikv = ["tikv-client"] fdb-chunked-bm = [] redis = ["dep:redis", "deadpool"] enterprise = [] diff --git a/crates/store/src/backend/composite/distributed_blob.rs b/crates/store/src/backend/composite/distributed_blob.rs index a51c93ef1..c60283707 100644 --- a/crates/store/src/backend/composite/distributed_blob.rs +++ b/crates/store/src/backend/composite/distributed_blob.rs @@ -71,6 +71,8 @@ impl DistributedBlob { Store::MySQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.get_blob(key, read_range).await, + #[cfg(feature = "tikv")] + Store::TiKV(store) => store.get_blob(key, read_range).await, #[cfg(all( feature = "enterprise", any(feature = "postgres", feature = "mysql") @@ -101,6 +103,8 @@ impl DistributedBlob { Store::MySQL(store) => store.put_blob(key, data).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.put_blob(key, data).await, + #[cfg(feature = "tikv")] + Store::TiKV(store) => store.put_blob(key, data).await, #[cfg(all( feature = "enterprise", any(feature = "postgres", feature = "mysql") @@ -131,6 +135,8 @@ impl DistributedBlob { Store::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.delete_blob(key).await, + #[cfg(feature = "tikv")] + Store::TiKV(store) => store.delete_blob(key).await, #[cfg(all( feature = "enterprise", any(feature = "postgres", feature = "mysql") diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index a4f780a55..964e088c7 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -24,6 +24,8 @@ pub mod rocksdb; pub mod s3; #[cfg(feature = "sqlite")] pub mod sqlite; +#[cfg(feature = "tikv")] +pub mod tikv; pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize; pub const MAX_TOKEN_MASK: usize = MAX_TOKEN_LENGTH - 1; diff --git a/crates/store/src/backend/tikv/blob.rs b/crates/store/src/backend/tikv/blob.rs new file mode 100644 index 000000000..9a1e3fc8c --- /dev/null +++ b/crates/store/src/backend/tikv/blob.rs @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: 2024 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::ops::{Bound, Range}; +use roaring::RoaringBitmap; +use tikv_client::{BoundRange, Key as TikvKey}; +use trc::EventType::Store; +use trc::StoreEvent; +use utils::BLOB_HASH_LEN; +use crate::{write::key::KeySerializer, SUBSPACE_BLOBS}; +use super::write::chunking::{delete_chunked_value, put_chunked_value}; +use super::read::chunking::get_chunked_value; +use super::{into_error, MAX_KEY_SIZE, MAX_SCAN_KEYS_SIZE, MAX_SCAN_VALUES_SIZE, MAX_VALUE_SIZE, TikvStore, MAX_CHUNKED_SIZED}; + +// TODO: Allow handling of more than MAX_SCAN_KEYS_SIZE + +impl TikvStore { + pub(crate) async fn get_blob( + &self, + key: &[u8], + range: Range, + ) -> trc::Result>> { + let mut trx = self.snapshot_read().await?; + + let block_start = range.start / MAX_VALUE_SIZE; + let bytes_start = range.start % MAX_VALUE_SIZE; + let block_end = (range.end / MAX_VALUE_SIZE) + 1; + + let mut begin = KeySerializer::new(1 + key.len() + 2) + .write(SUBSPACE_BLOBS) + .write(key) + .write(block_start as u16) + .finalize(); + let end = KeySerializer::new(1 + key.len() + 2) + .write(SUBSPACE_BLOBS) + .write(key) + .write(block_end as u16) + .write(u8::MIN) // Null byte to make the end inclusive + .finalize(); + + let mut blob_data_opt: Option> = None; + let mut blob_range = range.end - range.start; + + 'outer: loop { + let mut keys = trx.scan((begin, end.clone()), MAX_SCAN_VALUES_SIZE) + .await + .map_err(into_error)?; + + let mut counter = 0; + let mut last_key = None; + while let Some(kv_pair) = keys.next() { + let key: Vec = kv_pair.0.into(); + let mut value: Vec = kv_pair.1.into(); + + if let Some(blob_data) = &mut blob_data_opt { + blob_data.extend_from_slice( + value + .get( + ..std::cmp::min( + blob_range.saturating_sub(blob_data.len()), + value.len(), + ), + ) + .unwrap_or(&[]), + ); + if blob_data.len() == blob_range { + break 'outer; + } + } else { + let blob_size = if blob_range <= (5 * (1 << 20)) { + blob_range + } else if value.len() == MAX_VALUE_SIZE { + MAX_VALUE_SIZE * 2 + } else { + value.len() + }; + let mut blob_data = Vec::with_capacity(blob_size); + blob_data.extend_from_slice( + value + .get(bytes_start..std::cmp::min(bytes_start + blob_range, value.len())) + .unwrap_or(&[]), + ); + if blob_data.len() == blob_range { + return Ok(Some(blob_data)); + } + blob_data_opt = Some(blob_data) + } + + last_key = Some(key); + } + + if counter == MAX_SCAN_VALUES_SIZE { + // Guaranteed to have the last key + begin = last_key.unwrap(); + continue; + } else { + break; + } + + } + + Ok(blob_data_opt) + } + + pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { + let mut trx = self.write_trx_with_backoff().await?; + + for (chunk_pos, chunk_value) in data.chunks(MAX_VALUE_SIZE).enumerate() { + let chunk_key = KeySerializer::new(1 + key.len() + 2) + .write(SUBSPACE_BLOBS) + .write(key) + .write(chunk_pos as u16) + .finalize(); + + trx.put(chunk_key, chunk_value).await.map_err(into_error)?; + } + + trx.commit().await.map_err(into_error)?; + Ok(()) + } + + pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { + if key.len() < BLOB_HASH_LEN { + return Ok(false); + } + + let begin = KeySerializer::new(1 + key.len() + 1) + .write(SUBSPACE_BLOBS) + .write(key) + .write(u16::MIN) + .finalize(); + let end = KeySerializer::new(1 + key.len() + 3) + .write(SUBSPACE_BLOBS) + .write(key) + .write(u16::MAX) + .write(u8::MIN) // Null byte to make the end inclusive + .finalize(); + + let range = BoundRange::from((begin, end)); + + let mut trx = self.write_trx_with_backoff().await?; + + loop { + let keys = trx + .scan_keys(range.clone(), MAX_SCAN_KEYS_SIZE) + .await + .map_err(into_error)?; + + let mut count = 0; + for key in keys { + count += 1; + trx.delete(key).await.map_err(into_error)?; + } + + if count < MAX_SCAN_KEYS_SIZE { + break; + } + } + + trx.commit().await.map_err(into_error)?; + + Ok(true) + } +} \ No newline at end of file diff --git a/crates/store/src/backend/tikv/main.rs b/crates/store/src/backend/tikv/main.rs new file mode 100644 index 000000000..f6bd99fb2 --- /dev/null +++ b/crates/store/src/backend/tikv/main.rs @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: 2024 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ +use std::time::Duration; +use tikv_client::{Backoff, CheckLevel, RetryOptions, TransactionClient, TransactionOptions}; +use utils::config::{utils::AsKey, Config}; +use super::{into_error, TikvStore}; + +impl TikvStore { + pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { + let prefix = prefix.as_key(); + + // Parse as SocketAddr but don't use it. TransactionClient takes only a String vector + let pd_endpoints= config.properties::((&prefix, "pd-endpoints")) + .into_iter() + .map(|(_key, addr_str)| addr_str) + .collect::>(); + + let trx_client = TransactionClient::new(pd_endpoints.clone()) + .await + .map_err(|err| { + config.new_build_error( + prefix.as_str(), + format!("Failed to create TiKV database: {err:?}"), + ) + }) + .ok()?; + + let backoff_min_delay = config + .property::((&prefix, "transaction.backoff-min-delay")) + .unwrap_or_else(|| Duration::from_millis(500)); + + let backoff_max_delay = config + .property::((&prefix, "transaction.backoff-max-delay")) + .unwrap_or_else(|| Duration::from_millis(2000)); + + let max_attempts = config + .property::((&prefix, "transaction.backoff-retry-limit")) + .unwrap_or_else(|| 30); + + let backoff = if let Some(backoff_type) = config + .property::((&prefix, "transaction.backoff-type")) { + match backoff_type.as_str() { + "expo-jitter" => Backoff::no_jitter_backoff( + backoff_min_delay.as_millis() as u64, + backoff_max_delay.as_millis() as u64, + max_attempts + ), + "equal-jitter" => Backoff::equal_jitter_backoff( + backoff_min_delay.as_millis() as u64, + backoff_max_delay.as_millis() as u64, + max_attempts + ), + "decor-jitter" => Backoff::decorrelated_jitter_backoff( + backoff_min_delay.as_millis() as u64, + backoff_max_delay.as_millis() as u64, + max_attempts + ), + "none" => Backoff::no_backoff(), + // Default + "full-jitter" | &_ => Backoff::full_jitter_backoff( + backoff_min_delay.as_millis() as u64, + backoff_max_delay.as_millis() as u64, + max_attempts + ), + } + } else { + // Default + Backoff::decorrelated_jitter_backoff( + backoff_min_delay.as_millis() as u64, + backoff_max_delay.as_millis() as u64, + max_attempts + ) + // Backoff::full_jitter_backoff( + // backoff_min_delay.as_millis() as u64, + // backoff_max_delay.as_millis() as u64, + // max_attempts + // ) + }; + //println!("using backoff {:?}", backoff); + + let write_trx_options = TransactionOptions::new_pessimistic() + .drop_check(CheckLevel::Warn) + .retry_options(RetryOptions::new(backoff.clone(), backoff.clone())); + + let read_trx_options = TransactionOptions::new_optimistic() + .drop_check(CheckLevel::None) + .retry_options(RetryOptions::new(backoff.clone(), backoff.clone())) + .read_only(); + + let current_timestamp = trx_client.current_timestamp().await.map_err(|err| { + config.new_build_error( + prefix.as_str(), + format!("Failed to create TiKV database: {err:?}"), + )}).ok()?; + + let store = Self { + trx_client, + write_trx_options, + read_trx_options, + version: parking_lot::Mutex::new(current_timestamp), + backoff, + }; + + Some(store) + } +} \ No newline at end of file diff --git a/crates/store/src/backend/tikv/mod.rs b/crates/store/src/backend/tikv/mod.rs new file mode 100644 index 000000000..24a51fbcd --- /dev/null +++ b/crates/store/src/backend/tikv/mod.rs @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: 2024 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::time::{Duration, Instant}; +use tikv_client::{TransactionClient, Transaction, Error as TikvError, Snapshot, Value, Key, Timestamp, RawClient, TransactionOptions, Backoff, KvPair, BoundRange}; +use crate::write::key::KeySerializer; + +pub mod blob; +pub mod main; +pub mod read; +pub mod write; + +// https://github.com/tikv/tikv/issues/7272#issuecomment-604841372 + +// Default limit is 4194304 bytes +const MAX_KEY_SIZE: u32 = 4 * 1024; +// Default limit is 4194304 bytes. Let's use half of that as a base to be safe (2097152 bytes). +// Then, 2097152 +const MAX_GRPC_MESSAGE_SIZE: u32 = 2097152; +const MAX_ASSUMED_KEY_SIZE: u32 = 256; +const MAX_VALUE_SIZE: usize = 131072; +const MAX_CHUNKED_SIZED: usize = MAX_VALUE_SIZE * (1 + 256); +const MAX_SCAN_KEYS_SIZE: u32 = MAX_GRPC_MESSAGE_SIZE / MAX_ASSUMED_KEY_SIZE; // 8192 +const MAX_SCAN_VALUES_SIZE: u32 = MAX_GRPC_MESSAGE_SIZE / MAX_VALUE_SIZE as u32; // 16 + +// Preparation for API v2 +// RFC: https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md +const MODE_PREFIX_TXN_KV: u8 = b'x'; +const MODE_PREFIX_RAW_KV: u8 = b'x'; + +pub const TRANSACTION_EXPIRY: Duration = Duration::from_secs(1); +pub const TRANSACTION_TIMEOUT: Duration = Duration::from_secs(4); + +#[allow(dead_code)] +pub struct TikvStore { + trx_client: TransactionClient, + write_trx_options: TransactionOptions, + read_trx_options: TransactionOptions, + version: parking_lot::Mutex, + backoff: Backoff, +} + +pub(crate) struct TimedTransaction { + trx: Transaction, + expires: Instant, +} + +#[inline(always)] +fn into_error(error: TikvError) -> trc::Error { + trc::StoreEvent::TikvError + .reason(error.to_string()) +} diff --git a/crates/store/src/backend/tikv/read.rs b/crates/store/src/backend/tikv/read.rs new file mode 100644 index 000000000..7f2f73b2e --- /dev/null +++ b/crates/store/src/backend/tikv/read.rs @@ -0,0 +1,336 @@ +/* + * SPDX-FileCopyrightText: 2024 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ +use std::ops::Bound; +use tikv_client::{BoundRange, CheckLevel, Key as TikvKey, KvPair, Snapshot, Transaction, Value}; +use roaring::RoaringBitmap; +use crate::{ + backend::deserialize_i64_le, + write::{ + key::{DeserializeBigEndian, KeySerializer}, + BitmapClass, ValueClass, + }, + BitmapKey, Deserialize, IterateParams, Key, ValueKey, U32_LEN, WITH_SUBSPACE, +}; +use crate::backend::tikv::read::chunking::get_chunked_value; +use super::{into_error, MAX_KEY_SIZE, MAX_SCAN_KEYS_SIZE, MAX_SCAN_VALUES_SIZE, MAX_VALUE_SIZE, TikvStore}; + +impl TikvStore { + pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> + where + U: Deserialize, + { + let key = key.serialize(WITH_SUBSPACE); + let mut snapshot = self.snapshot_read().await?; + + match get_chunked_value(&key, &mut snapshot).await? { + Some(bytes) => U::deserialize(&bytes).map(Some), + None => Ok(None) + } + } + + pub(crate) async fn get_bitmap( + &self, + mut key: BitmapKey>, + ) -> trc::Result> { + let mut trx = self.snapshot_read().await?; + let mut bm = RoaringBitmap::new(); + + let mut begin = key.serialize(WITH_SUBSPACE); + key.document_id = u32::MAX; + let mut end = key.serialize(WITH_SUBSPACE); + end.push(u8::MIN); // Inclusive + let key_len = begin.len(); + + 'outer: loop { + let keys = trx + .scan_keys((begin, end.clone()), MAX_SCAN_KEYS_SIZE) + .await + .map_err(into_error)?; + + let mut count = 0; + let mut last_key = None; + + for key in keys { + count += 1; + let key_slice: &[u8] = key.as_ref().into(); + if key.len() == key_len { + bm.insert(key_slice.deserialize_be_u32(key.len() - U32_LEN)?); + } + last_key = Some(key) + } + + if count == MAX_SCAN_KEYS_SIZE { + // Guaranteed to have a key unless MAX_SCAN_KEYS_SIZE is 0 + begin = last_key.unwrap().into(); + begin.push(u8::MIN); // To make the start range exclusive + continue; + } else { + break; + } + } + + Ok(if !bm.is_empty() { Some(bm) } else { None }) + } + + pub(crate) async fn iterate( + &self, + params: IterateParams, + mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, + ) -> trc::Result<()> { + let mut begin = params.begin.serialize(WITH_SUBSPACE); + let mut end = params.end.serialize(WITH_SUBSPACE); + end.push(u8::MIN); // Inclusive + + let mut trx = self.snapshot_read().await?; + + if !params.first { + if params.ascending { + loop { + let keys = trx + .scan((begin, end.clone()), MAX_SCAN_VALUES_SIZE) + .await + .map_err(into_error)?; + + let mut count = 0; + let mut last_key = None; + for kv_pair in keys { + count += 1; + let key_slice: &[u8] = kv_pair.key().into(); + let value = kv_pair.value().as_slice(); + + if !cb(key_slice.get(1..).unwrap_or_default(), value)? { + return Ok(()); + } + + last_key = Some(kv_pair.into_key()); + } + + if count == MAX_SCAN_VALUES_SIZE { + begin = last_key.unwrap().into(); + begin.push(u8::MIN); + continue; + } else { + break; + } + } + } else { + loop { + let keys = trx + .scan_reverse((begin.clone(), end), MAX_SCAN_VALUES_SIZE) + .await + .map_err(into_error)?; + + let mut count = 0; + let mut last_key = None; + for kv_pair in keys { + count += 1; + let key_slice: &[u8] = kv_pair.key().into(); + let value = kv_pair.value().as_slice(); + + if !cb(key_slice.get(1..).unwrap_or_default(), value)? { + return Ok(()); + } + + last_key = Some(kv_pair.into_key()); + } + + if count == MAX_SCAN_VALUES_SIZE { + end = last_key.unwrap().into(); + continue; + } else { + break; + } + } + } + } else { + let result = if params.ascending { + trx.scan((begin, end), 1) + .await + .map_err(into_error)? + .next() + } else { + trx.scan_reverse((begin, end), 1) + .await + .map_err(into_error)? + .next() + }; + + if let Some(kv_pair) = result { + let key: &[u8] = kv_pair.key().into(); + let value = kv_pair.value().as_slice(); + cb(key.get(1..).unwrap_or_default(), value)?; + } + } + + Ok(()) + } + + pub(crate) async fn get_counter( + &self, + key: impl Into>> + Sync + Send, + ) -> trc::Result { + let key = key.into().serialize(WITH_SUBSPACE); + + if let Some(bytes) = self + .snapshot_read() + .await? + .get(key.clone()) + .await + .map_err(into_error)? + { + deserialize_i64_le(&key, &bytes) + } else { + Ok(0) + } + } + + pub(crate) async fn read_trx(&self) -> trc::Result { + self.trx_client + .begin_with_options(self.read_trx_options.clone()) + .await + .map_err(into_error) + } + + pub(crate) async fn snapshot_read(&self) -> trc::Result { + let current_timestamp = self + .trx_client + .current_timestamp() + .await + .map_err(into_error)?; + + Ok(self.trx_client.snapshot(current_timestamp, self.read_trx_options.clone())) + } + +} + +pub(super) mod chunking { + use super::*; + + pub(in super::super) async fn get_chunked_value( + key: &[u8], + trx: &mut ReadTrx + ) -> trc::Result>> { + let Some(mut bytes) = trx.get(key.to_vec()).await? else { + return Ok(None); + }; + + if bytes.len() != MAX_VALUE_SIZE { + return Ok(Some(bytes)) + } + + let start_key = KeySerializer::new(key.len() + 1) + .write(key) + .write(u8::MIN) + .finalize(); + let end_key = KeySerializer::new(key.len() + 2) + .write(key) + .write(u8::MAX) + .write(u8::MIN) // Null byte to make the end inclusive + .finalize(); + + let mut keys: Vec = trx + .scan_keys((start_key, end_key), 256 + 1) + .await? + .collect(); + + for chunk_key in keys { + // Any scanned keys are guaranteed to have a value + let mut value = trx.get(chunk_key).await?.unwrap(); + bytes.append(&mut value); + } + + Ok(Some(bytes)) + } + + trait ReadTransaction { + async fn get(&mut self, key: impl Into) -> trc::Result>; + async fn key_exists(&mut self, key: impl Into) -> trc::Result; + async fn batch_get( + &mut self, + keys: impl IntoIterator> + ) -> trc::Result>; + async fn scan( + &mut self, + range: impl Into, + limit: u32 + ) -> trc::Result>; + async fn scan_keys( + &mut self, + range: impl Into, + limit: u32 + ) -> trc::Result>; + async fn scan_reverse( + &mut self, + range: impl Into, + limit: u32 + ) -> trc::Result>; + async fn scan_keys_reverse( + &mut self, + range: impl Into, + limit: u32 + ) -> trc::Result>; + } + + impl ReadTransaction for Transaction { + async fn get(&mut self, key: impl Into) -> trc::Result> { + self.get(key).await.map_err(into_error) + } + + async fn key_exists(&mut self, key: impl Into) -> trc::Result { + self.key_exists(key).await.map_err(into_error) + } + + async fn batch_get(&mut self, keys: impl IntoIterator>) -> trc::Result> { + self.batch_get(keys).await.map_err(into_error) + } + + async fn scan(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan(range, limit).await.map_err(into_error) + } + + async fn scan_keys(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan_keys(range, limit).await.map_err(into_error) + } + + async fn scan_reverse(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan_reverse(range, limit).await.map_err(into_error) + } + + async fn scan_keys_reverse(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan_keys_reverse(range, limit).await.map_err(into_error) + } + } + + impl ReadTransaction for Snapshot { + async fn get(&mut self, key: impl Into) -> trc::Result> { + self.get(key).await.map_err(into_error) + } + + async fn key_exists(&mut self, key: impl Into) -> trc::Result { + self.key_exists(key).await.map_err(into_error) + } + + async fn batch_get(&mut self, keys: impl IntoIterator>) -> trc::Result> { + self.batch_get(keys).await.map_err(into_error) + } + + async fn scan(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan(range, limit).await.map_err(into_error) + } + + async fn scan_keys(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan_keys(range, limit).await.map_err(into_error) + } + + async fn scan_reverse(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan_reverse(range, limit).await.map_err(into_error) + } + + async fn scan_keys_reverse(&mut self, range: impl Into, limit: u32) -> trc::Result> { + self.scan_keys_reverse(range, limit).await.map_err(into_error) + } + } +} \ No newline at end of file diff --git a/crates/store/src/backend/tikv/write.rs b/crates/store/src/backend/tikv/write.rs new file mode 100644 index 000000000..6249a533d --- /dev/null +++ b/crates/store/src/backend/tikv/write.rs @@ -0,0 +1,472 @@ +/* + * SPDX-FileCopyrightText: 2024 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{cmp::Ordering, iter, time::{Duration, Instant}}; +use std::collections::Bound; +use std::ops::DerefMut; +use tikv_client::{Backoff, BoundRange, CheckLevel, Key as TikvKey, RetryOptions, TimestampExt, Transaction, Value}; +use rand::Rng; +use roaring::RoaringBitmap; +use tikv_client::TransactionOptions; +use tikv_client::proto::kvrpcpb::{Assertion, Mutation, Op}; +use tikv_client::transaction::ResolveLocksOptions; +use trc::Error; +use crate::{ + backend::deserialize_i64_le, + write::{ + key::{DeserializeBigEndian, KeySerializer}, + AssignedIds, Batch, BitmapClass, Operation, RandomAvailableId, ValueOp, + MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, + }, + BitmapKey, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_QUOTA, U32_LEN, WITH_SUBSPACE, +}; +use crate::write::key; +use super::write::chunking::{put_chunked_value, delete_chunked_value}; +use super::read::chunking::get_chunked_value; +use super::{into_error, TikvStore, MAX_VALUE_SIZE, MAX_SCAN_KEYS_SIZE}; + +impl TikvStore { + pub(crate) async fn write(&self, batch: Batch) -> trc::Result { + let mut backoff = self.backoff.clone(); + + loop { + let mut trx = self.write_trx_no_backoff().await?; + + match self.write_trx(&mut trx, &batch).await { + Ok(result) => return Ok(result), + Err(err) => { + let _ = trx.rollback().await; + let version = self.version.lock().clone(); + // match self.trx_client.gc(version).await.map_err(into_error) { + // Ok(a) => {} + // Err(_) => {} + // } + //self.trx_client.cleanup_locks(BoundRange::range_from(TikvKey::from(vec![0])), &version, ResolveLocksOptions::default()).await.map_err(into_error)?; + drop(version); + let Some(backoff_duration) = backoff.next_delay_duration() else { + //println!("giving up, error: {}", err); + return Err(err); + + }; + //println!("backing off because of error: {}", err); + //println!("backoff for {} secs with {} attempts", backoff_duration.as_secs_f32(), backoff.current_attempts()); + tokio::time::sleep(backoff_duration).await; + continue; + } + } + } + + + } + + async fn write_trx(&self, trx: &mut Transaction, batch: &Batch) -> trc::Result { + let mut account_id = u32::MAX; + let mut collection = u8::MAX; + let mut document_id = u32::MAX; + let mut change_id = u64::MAX; + let mut result = AssignedIds::default(); + + for op in &batch.ops { + match op { + Operation::AccountId { + account_id: account_id_, + } => { + account_id = *account_id_; + } + Operation::Collection { + collection: collection_, + } => { + collection = *collection_; + } + Operation::DocumentId { + document_id: document_id_, + } => { + document_id = *document_id_; + } + Operation::ChangeId { + change_id: change_id_, + } => { + change_id = *change_id_; + } + Operation::Value { class, op } => { + let key = class.serialize( + account_id, + collection, + document_id, + WITH_SUBSPACE, + (&result).into(), + ); + //println!("writing key: {:?}", key); + let do_chunk = !class.is_counter(collection); + + match op { + ValueOp::Set(value) => { + let value = value.resolve(&result)?; + if do_chunk { + put_chunked_value(&key, &value, trx, false).await?; + } else { + trx.put(key, value.as_ref()).await.map_err(into_error)?; + } + } + ValueOp::AtomicAdd(by) => { + get_and_add(trx, key, *by).await?; + } + ValueOp::AddAndGet(by) => { + let num = get_and_add(trx, key, *by).await?; + result.push_counter_id(num); + } + ValueOp::Clear => { + if do_chunk { + delete_chunked_value(&key, trx, false).await?; + } else { + trx.delete(key).await.map_err(into_error)?; + } + } + } + } + Operation::Index { field, key, set } => { + let key = IndexKey { + account_id, + collection, + document_id, + field: *field, + key, + }.serialize(0); + //println!("writing index key: {:?}", key); + + if *set { + trx.put(key, &[]).await.map_err(into_error)?; + } else { + trx.delete(key).await.map_err(into_error)?; + } + } + Operation::Bitmap { class, set } => { + let assign_id = *set + && matches!(class, BitmapClass::DocumentIds) + && document_id == u32::MAX; + + if assign_id { + let mut begin = BitmapKey { + account_id, + collection, + class: BitmapClass::DocumentIds, + document_id: 0, + }.serialize(WITH_SUBSPACE); + let mut end = BitmapKey { + account_id, + collection, + class: BitmapClass::DocumentIds, + document_id: u32::MAX, + }.serialize(WITH_SUBSPACE); + end.push(u8::MIN); // Null byte to make the end inclusive + + + let key_len = begin.len(); + + let mut found_ids = RoaringBitmap::new(); + + 'outer: loop { + //println!("scanning keys {:?} and {:?}", begin, end); + let mut keys = trx.scan_keys((begin, end.clone()), MAX_SCAN_KEYS_SIZE) + .await + .map_err(into_error)? + .peekable(); + + let mut count = 0; + while let Some(key) = keys.next() { + count += 1; + let key_slice: &[u8] = key.as_ref().into(); + //println!("found key {:?}", key_slice); + if key_slice.len() == key_len { + found_ids.insert(key_slice.deserialize_be_u32(key_len - U32_LEN)?); + } else { + break 'outer; + } + + if keys.peek().is_none() { + if count < MAX_SCAN_KEYS_SIZE { + break 'outer; + } else { + begin = key.into(); + begin.push(u8::MIN); // Null byte to make the beginning exclusive + continue 'outer; + } + } + } + // Empty + break; + } + + document_id = found_ids.random_available_id(); + //println!("using document id: {} from found IDs: {:?}", document_id, found_ids); + result.push_document_id(document_id); + } + + let key = class.serialize( + account_id, + collection, + document_id, + WITH_SUBSPACE, + (&result).into(), + ); + + if *set { + let first = key.clone(); + let second = class.serialize( + account_id, + collection, + document_id + 1, + WITH_SUBSPACE, + (&result).into(), + ); + trx.lock_keys([first, second]).await.map_err(into_error)?; + trx.put(key, &[]).await.map_err(into_error)?; + } else { + trx.delete(key).await.map_err(into_error)?; + } + } + Operation::Log { set } => { + let key = LogKey { + account_id, + collection, + change_id, + }.serialize(WITH_SUBSPACE); + + trx.put(key, set.resolve(&result)?.as_ref()).await.map_err(into_error)?; + } + Operation::AssertValue { + class, + assert_value, + } => { + let key = class.serialize( + account_id, + collection, + document_id, + WITH_SUBSPACE, + (&result).into(), + ); + + let matches = match get_chunked_value(&key, trx).await { + Ok(Some(bytes)) => assert_value.matches(bytes.as_slice()), + Ok(None) => { + assert_value.is_none() + } + Err(_) => false, + }; + + if !matches { + trx.rollback().await.map_err(into_error)?; + return Err(trc::StoreEvent::AssertValueFailed.into()); + } + } + } + } + + if let Some(current_ts) = trx.commit().await.map_err(into_error)? { + let mut previous_ts = self.version.lock(); + if previous_ts.version() < current_ts.version() { + *previous_ts = current_ts; + } + } + // if ! result.counter_ids.is_empty() || ! result.document_ids.is_empty() { + // println!("success with counters: [{:?}] and doc ids: [{:?}]", result.counter_ids, result.document_ids); + // } + Ok(result) + } + + pub(crate) async fn purge_store(&self) -> trc::Result<()> { + //let mut delete_keys = Vec::new(); + + for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA] { + let from_key = [subspace, 0u8]; + let to_key = [subspace, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX]; + + // Since we are deleting all of them anyways. No point moving the start bound + let mut begin = Bound::Included(TikvKey::from(from_key.to_vec())); + + let mut backoff = self.backoff.clone(); + + 'outer: loop { + let end = Bound::Included(TikvKey::from(to_key.to_vec())); + let range = BoundRange::new(begin, end); + + let mut trx = self.write_trx_no_backoff().await?; + let mut keys_iter = trx.scan_keys(range.clone(), MAX_SCAN_KEYS_SIZE) + .await + .map_err(into_error)?; + + let mut count = 0; + let mut last_key = TikvKey::default(); + while let Some(key) = keys_iter.next() { + count += 1; + if let Some(value) = trx.get_for_update(key.clone()).await.map_err(into_error)? { + if deserialize_i64_le((&key).into(), value.as_slice())? == 0 { + trx.delete(key.clone()).await.map_err(into_error)?; + } + } + last_key = key; + } + + if self.commit(trx, Some(&mut backoff)).await? {} else { + begin = Bound::Excluded(last_key); + continue; + } + + if count < MAX_SCAN_KEYS_SIZE { + break 'outer; + } else { + begin = Bound::Excluded(last_key); + continue; + } + } + } + + Ok(()) + } + + pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { + let begin_range = Bound::Included(TikvKey::from(from.serialize(WITH_SUBSPACE))); + let end_range = Bound::Included(TikvKey::from(to.serialize(WITH_SUBSPACE))); + let range = BoundRange::new(begin_range, end_range); + + let mut trx = self.write_trx_with_backoff().await?; + + loop { + let keys = trx + .scan_keys(range.clone(), MAX_SCAN_KEYS_SIZE) + .await + .map_err(into_error)?; + + let mut count = 0; + for key in keys { + count += 1; + trx.delete(key).await.map_err(into_error)?; + } + + if count != MAX_SCAN_KEYS_SIZE { + break; + } + } + + trx.commit().await.map_err(into_error)?; + Ok(()) + } + + pub(crate) async fn commit(&self, mut trx: Transaction, ext_backoff: Option<&mut Backoff>) -> trc::Result { + if let Err(e) = trx.commit().await { + if let Some(backoff) = ext_backoff { + let Some(backoff_duration) = backoff.next_delay_duration() else { + return Err(into_error(e)); + }; + tokio::time::sleep(backoff_duration).await; + Ok(false) + } else { + Err(into_error(e)) + } + } else { + Ok(true) + } + } + + pub(super) async fn write_trx_no_backoff(&self) -> trc::Result { + let write_trx_options = TransactionOptions::new_optimistic() + .drop_check(CheckLevel::Warn) + .use_async_commit() + .retry_options(RetryOptions::none()); + + self.trx_client + .begin_with_options(write_trx_options) + .await + .map_err(into_error) + } + + pub(super) async fn write_trx_with_backoff(&self) -> trc::Result { + self.trx_client + .begin_with_options(self.write_trx_options.clone()) + .await + .map_err(into_error) + } +} + +async fn get_and_add(trx: &mut Transaction, key: impl Into, by: i64) -> trc::Result { + let key = key.into(); + if let Some(previous) = trx.get_for_update(key.clone()).await.map_err(into_error)? { + let addend = deserialize_i64_le((&key).into(), &previous)?; + let sum = addend + by; + trx.put(key, sum.to_le_bytes().as_slice()).await.map_err(into_error)?; + Ok(sum) + } else { + trx.put(key, by.to_le_bytes().as_slice()).await.map_err(into_error)?; + Ok(by) + } +} + +pub(super) mod chunking { + use super::*; + + pub(in super::super) async fn delete_chunked_value( + key: &[u8], + trx: &mut Transaction, + commit: bool, + ) -> trc::Result<()> { + let begin_key = key.to_vec(); + + let end_key = KeySerializer::new(key.len() + 1) + .write(key) + .write(u8::MAX) + .finalize(); + + let keys = trx.scan_keys((begin_key, end_key), 256) + .await + .map_err(into_error)?; + + for chunk_key in keys { + trx.delete(chunk_key).await.map_err(into_error)?; + } + + if commit { + trx.commit().await.map_err(into_error)?; + } + + Ok(()) + } + + pub(in super::super) async fn put_chunked_value( + key: &[u8], + value: &[u8], + trx: &mut Transaction, + commit: bool + ) -> trc::Result<()> { + let mut chunk_iter = value.chunks(MAX_VALUE_SIZE); + + if chunk_iter.len() > 1 + 256 { + // Expected to be thrown back so might as well roll it back. + trx.rollback().await.map_err(into_error)?; + return Err(trc::StoreEvent::TikvError + .ctx( + trc::Key::Reason, + "Value is too large", + )); + } + + let first_chunk = chunk_iter.next().unwrap_or_else(|| &[]); + trx.put(key.to_vec(), first_chunk).await.map_err(into_error)?; + + for (chunk_pos, value_chunk) in chunk_iter.enumerate() { + let chunk_key = KeySerializer::new(key.len() + 1) + .write(key) + .write(chunk_pos as u8) + .finalize(); + trx.put(chunk_key, value_chunk).await.map_err(into_error)?; + } + + if commit { + trx.commit().await.map_err(into_error)?; + } + + Ok(()) + } +} \ No newline at end of file diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 29e79c993..2568512eb 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -32,6 +32,9 @@ use crate::backend::foundationdb::FdbStore; #[cfg(feature = "rocks")] use crate::backend::rocksdb::RocksDbStore; +#[cfg(feature = "tikv")] +use crate::backend::tikv::TikvStore; + #[cfg(feature = "elastic")] use crate::backend::elastic::ElasticSearchStore; @@ -181,6 +184,28 @@ impl Stores { self.lookup_stores.insert(store_id.clone(), db.into()); } } + #[cfg(feature = "tikv")] + "tikv" => { + // Avoid opening the same store twice + if is_reload + && self + .stores + .values() + .any(|store| matches!(store, Store::TiKV(_))) + { + continue; + } + + if let Some(db) = TikvStore::open(config, prefix).await.map(Store::from) { + self.stores.insert(store_id.clone(), db.clone()); + self.fts_stores.insert(store_id.clone(), db.clone().into()); + self.blob_stores.insert( + store_id.clone(), + BlobStore::from(db.clone()).with_compression(compression_algo), + ); + self.lookup_stores.insert(store_id, db.into()); + } + } "fs" => { if let Some(db) = FsStore::open(config, prefix).await.map(BlobStore::from) { self.blob_stores diff --git a/crates/store/src/dispatch/blob.rs b/crates/store/src/dispatch/blob.rs index a06b6f4df..d794fad3c 100644 --- a/crates/store/src/dispatch/blob.rs +++ b/crates/store/src/dispatch/blob.rs @@ -30,6 +30,8 @@ impl BlobStore { Store::MySQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.get_blob(key, read_range).await, + #[cfg(feature = "tikv")] + Store::TiKV(store) => store.get_blob(key, read_range).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.get_blob(key, read_range).await, Store::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -110,6 +112,8 @@ impl BlobStore { Store::MySQL(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.put_blob(key, data.as_ref()).await, + #[cfg(feature = "tikv")] + Store::TiKV(store) => store.put_blob(key, data.as_ref()).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.put_blob(key, data.as_ref()).await, Store::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -146,6 +150,8 @@ impl BlobStore { Store::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.delete_blob(key).await, + #[cfg(feature = "tikv")] + Store::TiKV(store) => store.delete_blob(key).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.delete_blob(key).await, Store::None => Err(trc::StoreEvent::NotConfigured.into()), diff --git a/crates/store/src/dispatch/mod.rs b/crates/store/src/dispatch/mod.rs index b6d1c32ac..1a1ae73ff 100644 --- a/crates/store/src/dispatch/mod.rs +++ b/crates/store/src/dispatch/mod.rs @@ -26,6 +26,8 @@ impl Store { Self::MySQL(_) => "mysql", #[cfg(feature = "rocks")] Self::RocksDb(_) => "rocksdb", + #[cfg(feature = "tikv")] + Self::TiKV(_) => "tikv", #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(_) => "read_replica", Self::None => "none", diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 9a938120b..7dee17c48 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -50,6 +50,8 @@ impl Store { Self::MySQL(store) => store.get_value(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_value(key).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.get_value(key).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_value(key).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -72,6 +74,8 @@ impl Store { Self::MySQL(store) => store.get_bitmap(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_bitmap(key).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.get_bitmap(key).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_bitmap(key).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -118,6 +122,8 @@ impl Store { Self::MySQL(store) => store.iterate(params, cb).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.iterate(params, cb).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.iterate(params, cb).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.iterate(params, cb).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -147,6 +153,8 @@ impl Store { Self::MySQL(store) => store.get_counter(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_counter(key).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.get_counter(key).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_counter(key).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -212,6 +220,8 @@ impl Store { Self::MySQL(store) => store.write(batch).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.write(batch).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.write(batch).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.write(batch).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -259,6 +269,8 @@ impl Store { Self::MySQL(store) => store.write(batch).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.write(batch).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.write(batch).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.write(batch).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -315,6 +327,8 @@ impl Store { Self::MySQL(store) => store.purge_store().await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.purge_store().await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.purge_store().await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.purge_store().await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -334,6 +348,8 @@ impl Store { Self::MySQL(store) => store.delete_range(from, to).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.delete_range(from, to).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.delete_range(from, to).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.delete_range(from, to).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -489,6 +505,8 @@ impl Store { Self::MySQL(store) => store.get_blob(key, range).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_blob(key, range).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.get_blob(key, range).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_blob(key, range).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -508,6 +526,8 @@ impl Store { Self::MySQL(store) => store.put_blob(key, data).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.put_blob(key, data).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.put_blob(key, data).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.put_blob(key, data).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), @@ -527,6 +547,8 @@ impl Store { Self::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.delete_blob(key).await, + #[cfg(feature = "tikv")] + Self::TiKV(store) => store.delete_blob(key).await, #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.delete_blob(key).await, Self::None => Err(trc::StoreEvent::NotConfigured.into()), diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index d102e7193..ec11bae51 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -46,6 +46,9 @@ use backend::elastic::ElasticSearchStore; #[cfg(feature = "redis")] use backend::redis::RedisStore; +#[cfg(feature = "tikv")] +use backend::tikv::TikvStore; + pub trait Deserialize: Sized + Sync + Send { fn deserialize(bytes: &[u8]) -> trc::Result; } @@ -184,6 +187,8 @@ pub enum Store { MySQL(Arc), #[cfg(feature = "rocks")] RocksDb(Arc), + #[cfg(feature = "tikv")] + TiKV(Arc), #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] SQLReadReplica(Arc), #[default] @@ -268,6 +273,13 @@ impl From for Store { } } +#[cfg(feature = "tikv")] +impl From for Store { + fn from(store: TikvStore) -> Self { + Self::TiKV(Arc::new(store)) + } +} + impl From for BlobStore { fn from(store: FsStore) -> Self { BlobStore { @@ -713,6 +725,8 @@ impl std::fmt::Debug for Store { Self::MySQL(_) => f.debug_tuple("MySQL").finish(), #[cfg(feature = "rocks")] Self::RocksDb(_) => f.debug_tuple("RocksDb").finish(), + #[cfg(feature = "tikv")] + Self::TiKV(_) => f.debug_tuple("TiKV").finish(), #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(_) => f.debug_tuple("SQLReadReplica").finish(), Self::None => f.debug_tuple("None").finish(), diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index c0ecb311c..0f7d4cf65 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -1536,6 +1536,7 @@ impl StoreEvent { StoreEvent::PostgresqlError => "PostgreSQL error", StoreEvent::RocksdbError => "RocksDB error", StoreEvent::SqliteError => "SQLite error", + StoreEvent::TikvError => "TiKV error", StoreEvent::LdapError => "LDAP error", StoreEvent::ElasticsearchError => "ElasticSearch error", StoreEvent::RedisError => "Redis error", @@ -1570,6 +1571,7 @@ impl StoreEvent { StoreEvent::PostgresqlError => "A PostgreSQL error occurred", StoreEvent::RocksdbError => "A RocksDB error occurred", StoreEvent::SqliteError => "An SQLite error occurred", + StoreEvent::TikvError => "A TiKV error occured", StoreEvent::LdapError => "An LDAP error occurred", StoreEvent::ElasticsearchError => "An ElasticSearch error occurred", StoreEvent::RedisError => "A Redis error occurred", diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index e4de67e17..176abbcd3 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -27,6 +27,7 @@ impl EventType { | StoreEvent::PostgresqlError | StoreEvent::RocksdbError | StoreEvent::SqliteError + | StoreEvent::TikvError | StoreEvent::LdapError | StoreEvent::ElasticsearchError | StoreEvent::RedisError diff --git a/crates/trc/src/imple.rs b/crates/trc/src/imple.rs new file mode 100644 index 000000000..e69de29bb diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index c3e914aeb..4fad6ef47 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -822,6 +822,7 @@ pub enum StoreEvent { PostgresqlError, RocksdbError, SqliteError, + TikvError, LdapError, ElasticsearchError, RedisError, diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index 68f346b1f..7042850e8 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -858,6 +858,7 @@ impl EventType { EventType::Security(SecurityEvent::BruteForceBan) => 549, EventType::Security(SecurityEvent::LoiterBan) => 550, EventType::Smtp(SmtpEvent::MailFromNotAllowed) => 551, + EventType::Store(StoreEvent::TikvError) => 552, } } @@ -1455,6 +1456,7 @@ impl EventType { 549 => Some(EventType::Security(SecurityEvent::BruteForceBan)), 550 => Some(EventType::Security(SecurityEvent::LoiterBan)), 551 => Some(EventType::Smtp(SmtpEvent::MailFromNotAllowed)), + 552 => Some(EventType::Store(StoreEvent::TikvError)), _ => None, } } diff --git a/resources/config/config.toml b/resources/config/config.toml index f043265e6..e68bf764a 100644 --- a/resources/config/config.toml +++ b/resources/config/config.toml @@ -70,4 +70,4 @@ enable = true [authentication.fallback-admin] user = "admin" -secret = "%{env:ADMIN_SECRET}%" +secret = "%{env:ADMIN_SECRET}%" \ No newline at end of file diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 51d2d81b1..7e78b6eae 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -5,8 +5,8 @@ edition = "2021" resolver = "2" [features] -default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "foundationdb"] -#default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "foundationdb"] +default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "foundationdb", "tikv"] +#default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "foundationdb", "tikv"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] @@ -15,6 +15,7 @@ rocks = ["store/rocks"] elastic = ["store/elastic"] s3 = ["store/s3"] redis = ["store/redis"] +tikv = ["store/tikv"] [dev-dependencies] store = { path = "../crates/store", features = ["test_mode", "enterprise"] } diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 503eb1a20..b9877ab68 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -67,6 +67,10 @@ type = "redis" urls = "redis://127.0.0.1" redis-type = "single" +[store."tikv"] +type = "tikv" +pd-endpoints = ["localhost:2379"] + "#; #[tokio::test(flavor = "multi_thread")]