diff --git a/Cargo.lock b/Cargo.lock index 0d484ff..fee0b37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -165,7 +165,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -187,7 +187,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -198,7 +198,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -368,9 +368,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", @@ -397,7 +397,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.65+curl-8.2.1" +version = "0.4.66+curl-8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961ba061c9ef2fe34bbd12b807152d96f0badd2bebe7b90ce6c8c8b7572a0986" +checksum = "70c44a72e830f0e40ad90dda8a6ab6ed6314d39776599a58a2e5e37fbc6db5b9" dependencies = [ "cc", "libc", @@ -556,7 +556,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "winapi", + "windows-sys", ] [[package]] @@ -580,7 +580,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -591,7 +591,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -602,9 +602,9 @@ checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" [[package]] name = "der" -version = "0.6.1" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", "pem-rfc7468", @@ -646,7 +646,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -666,7 +666,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -687,13 +687,22 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys 0.3.7", +] + [[package]] name = "dirs" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", ] [[package]] @@ -706,6 +715,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dirs-sys" version = "0.4.1" @@ -744,7 +764,7 @@ checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] name = "dropshot" version = "0.9.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot#c7a5e43d694cbf37b8e7ce9f04710d46b157233e" +source = "git+https://github.com/oxidecomputer/dropshot#fa728d07970824fd5f3bd57a3d4dc0fdbea09bfd" dependencies = [ "async-stream", "async-trait", @@ -820,20 +840,20 @@ dependencies = [ [[package]] name = "dropshot_endpoint" version = "0.9.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot#c7a5e43d694cbf37b8e7ce9f04710d46b157233e" +source = "git+https://github.com/oxidecomputer/dropshot#fa728d07970824fd5f3bd57a3d4dc0fdbea09bfd" dependencies = [ "proc-macro2", "quote", "serde", "serde_tokenstream", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] name = "dyn-clone" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" [[package]] name = "either" @@ -1017,7 +1037,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -1244,9 +1264,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -1655,16 +1675,17 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] @@ -1950,7 +1971,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.3.3", "libc", ] @@ -2002,6 +2023,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "dirs 3.0.2", "http", "jsonwebtoken", "log", @@ -2065,7 +2087,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -2226,9 +2248,9 @@ dependencies = [ [[package]] name = "pem-rfc7468" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d159833a9105500e0398934e205e0773f0b27529557134ecfc51c27646adac" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ "base64ct", ] @@ -2276,7 +2298,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -2307,7 +2329,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -2324,21 +2346,20 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkcs1" -version = "0.4.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff33bdbdfc54cc98a2eca766ebdec3e1b8fb7387523d5c9c9a2891da856f719" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ "der", "pkcs8", "spki", - "zeroize", ] [[package]] name = "pkcs8" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", "spki", @@ -2423,9 +2444,8 @@ dependencies = [ [[package]] name = "progenitor" version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor#5de727cc08082aae3371632dde9c44a840292145" dependencies = [ - "progenitor-client", + "progenitor-client 0.3.0", "progenitor-impl", "progenitor-macro", "serde_json", @@ -2434,7 +2454,20 @@ dependencies = [ [[package]] name = "progenitor-client" version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor#5de727cc08082aae3371632dde9c44a840292145" +dependencies = [ + "bytes", + "futures-core", + "percent-encoding 2.3.0", + "reqwest", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "progenitor-client" +version = "0.3.0" +source = "git+https://github.com/oxidecomputer/progenitor#5c941c0b41b0235031f3ade33a9c119945f1fd51" dependencies = [ "bytes", "futures-core", @@ -2448,7 +2481,6 @@ dependencies = [ [[package]] name = "progenitor-impl" version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor#5de727cc08082aae3371632dde9c44a840292145" dependencies = [ "getopts", "heck", @@ -2461,7 +2493,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.36", + "syn 2.0.37", "thiserror", "typify", "unicode-ident", @@ -2470,7 +2502,6 @@ dependencies = [ [[package]] name = "progenitor-macro" version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor#5de727cc08082aae3371632dde9c44a840292145" dependencies = [ "openapiv3", "proc-macro2", @@ -2481,7 +2512,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -2759,15 +2790,18 @@ dependencies = [ "hyper", "hyper-tls", "jsonwebtoken", + "meilisearch-sdk", "mockall", "oauth2", "octorust", + "openssl", "partial-struct", "rand", "rand_core", "regex", "reqwest", "rfd-model", + "ring", "rsa", "schemars", "serde", @@ -2797,12 +2831,15 @@ dependencies = [ "chrono", "clap", "config", - "dirs", + "dirs 5.0.1", "oauth2", + "progenitor-client 0.3.0 (git+https://github.com/oxidecomputer/progenitor)", "reqwest", "rfd-sdk", "serde", "serde_json", + "tabwriter", + "textwrap", "tokio", "toml 0.5.11", "uuid", @@ -2853,6 +2890,10 @@ dependencies = [ "octorust", "parse-rfd", "regex", + "reqwest", + "reqwest-middleware", + "reqwest-retry", + "reqwest-tracing", "rfd-data", "rfd-model", "serde", @@ -2889,7 +2930,7 @@ name = "rfd-sdk" version = "0.1.0" dependencies = [ "chrono", - "progenitor-client", + "progenitor-client 0.3.0 (git+https://github.com/oxidecomputer/progenitor)", "reqwest", "schemars", "serde", @@ -2924,11 +2965,12 @@ dependencies = [ [[package]] name = "rsa" -version = "0.8.2" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55a77d189da1fee555ad95b7e50e7457d91c0e089ec68ca69ad2989413bbdab4" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" dependencies = [ "byteorder", + "const-oid", "digest", "num-bigint-dig", "num-integer", @@ -2937,7 +2979,9 @@ dependencies = [ "pkcs1", "pkcs8", "rand_core", + "sha2", "signature", + "spki", "subtle", "zeroize", ] @@ -2982,9 +3026,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ "bitflags 2.4.0", "errno", @@ -3028,9 +3072,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ "ring", "untrusted", @@ -3077,9 +3121,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763f8cd0d4c71ed8389c90cb8100cba87e763bd01a8e614d4f0af97bcd50a161" +checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" dependencies = [ "bytes", "chrono", @@ -3093,9 +3137,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0f696e21e10fa546b7ffb1c9672c6de8fbc7a81acf59524386d8639bf12737" +checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" dependencies = [ "proc-macro2", "quote", @@ -3198,7 +3242,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -3251,7 +3295,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -3291,7 +3335,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -3309,9 +3353,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -3452,9 +3496,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" @@ -3490,9 +3540,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "spki" -version = "0.6.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", "der", @@ -3529,15 +3579,24 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.36" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e02e55d62894af2a08aca894c6577281f76769ba47c94d5756bec8ac6e7373" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "tabwriter" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08e1173ee641651a3095fe95d86ae314cd1f959888097debce3e0f9ca532eef1" +dependencies = [ + "unicode-width", +] + [[package]] name = "take_mut" version = "0.2.2" @@ -3589,6 +3648,17 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.48" @@ -3606,7 +3676,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -3691,7 +3761,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -3716,9 +3786,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -3823,7 +3893,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", ] [[package]] @@ -3913,7 +3983,7 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#5c3ee32a6d909432ef2f06e4b34440f5046f3656" +source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" dependencies = [ "typify-impl", "typify-macro", @@ -3922,7 +3992,7 @@ dependencies = [ [[package]] name = "typify-impl" version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#5c3ee32a6d909432ef2f06e4b34440f5046f3656" +source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" dependencies = [ "heck", "log", @@ -3931,7 +4001,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.36", + "syn 2.0.37", "thiserror", "unicode-ident", ] @@ -3939,7 +4009,7 @@ dependencies = [ [[package]] name = "typify-macro" version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#5c3ee32a6d909432ef2f06e4b34440f5046f3656" +source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" dependencies = [ "proc-macro2", "quote", @@ -3947,7 +4017,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.36", + "syn 2.0.37", "typify-impl", ] @@ -3978,6 +4048,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.22" @@ -3995,9 +4071,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unsafe-libyaml" @@ -4129,7 +4205,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -4163,7 +4239,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4236,9 +4312,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index ba8ef89..a5e1dc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ mockall = "0.11.3" newline-converter = "0.2.2" oauth2 = "4.1.0" octorust = "0.7.0-rc.1" +openssl = "0.10.57" partial-struct = { git = "https://github.com/oxidecomputer/partial-struct" } progenitor = { git = "https://github.com/oxidecomputer/progenitor" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" } @@ -53,10 +54,14 @@ rand = "0.8.5" rand_core = "0.6" regex = "1.7.1" reqwest = { version = "0.11", features = ["json", "stream"] } -rsa = "0.8.2" +reqwest-middleware = "0.2" +reqwest-retry = "0.2.2" +reqwest-tracing = "0.4.6" +ring = "0.16.20" +rsa = "0.9.2" rustfmt-wrapper = "0.2.0" schemars = "0.8.11" -sha2 = "0.10.6" +sha2 = "0.10.7" serde = "1" serde_bytes = "0.11.9" serde_json = "1" @@ -66,6 +71,7 @@ slog = "2.7.0" slog-async = "2.7.0" tabwriter = "1.3.0" tap = "1.0.1" +textwrap = "0.16.0" thiserror = "1.0.38" tokio = "1.25.0" toml = "0.5.10" @@ -77,8 +83,8 @@ uuid = "1.2.2" valuable = "0.1.0" yup-oauth2 = "8.1.0" -# [patch."https://github.com/oxidecomputer/progenitor"] -# progenitor = { path = "../progenitor/progenitor" } +[patch."https://github.com/oxidecomputer/progenitor"] +progenitor = { path = "../progenitor/progenitor" } # [patch."https://github.com/oxidecomputer/typify"] # typify = { path = "../typify/typify" } diff --git a/README.md b/README.md index 5d12915..9912a76 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ Work in progress replacement for RFD processing and programmatic access. +## TODO + +Add job state and locking + ## Authentication Rough sketch of how users can authenticate to the RFD API diff --git a/rfd-api-spec.json b/rfd-api-spec.json index 4af7224..8e3aae2 100644 --- a/rfd-api-spec.json +++ b/rfd-api-spec.json @@ -874,6 +874,44 @@ } } }, + "/rfd-search": { + "get": { + "summary": "Search the RFD index and get a list of results", + "operationId": "search_rfds", + "parameters": [ + { + "in": "query", + "name": "q", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ListRfd", + "type": "array", + "items": { + "$ref": "#/components/schemas/ListRfd" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/self": { "get": { "summary": "Retrieve the user information of the calling user", @@ -1000,6 +1038,7 @@ "GetAssignedRfds", "GetAllDiscussions", "GetAssignedDiscussions", + "SearchRfds", "CreateOAuthClient", "GetAssignedOAuthClients", "UpdateAssignedOAuthClients", diff --git a/rfd-api/Cargo.toml b/rfd-api/Cargo.toml index ca3e545..b2ef392 100644 --- a/rfd-api/Cargo.toml +++ b/rfd-api/Cargo.toml @@ -22,15 +22,18 @@ http = { workspace = true } hyper = { workspace = true } hyper-tls = { workspace = true } jsonwebtoken = { workspace = true } +meilisearch-sdk = { workspace = true } oauth2 = { workspace = true } octorust = { workspace = true } +openssl = { workspace = true } partial-struct = { workspace = true } rand = { workspace = true, features = ["std"] } rand_core = { workspace = true, features = ["std"] } regex = { workspace = true } reqwest = { workspace = true } +ring = { workspace = true } rfd-model = { path = "../rfd-model" } -rsa = { workspace = true } +rsa = { workspace = true, features = ["sha2"] } schemars = { workspace = true, features = ["chrono"] } sha2 = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/rfd-api/src/authn/jwt.rs b/rfd-api/src/authn/jwt.rs index 88bd267..ec5a839 100644 --- a/rfd-api/src/authn/jwt.rs +++ b/rfd-api/src/authn/jwt.rs @@ -6,7 +6,6 @@ use jsonwebtoken::{ jwk::{AlgorithmParameters, CommonParameters, Jwk, PublicKeyUse, RSAKeyParameters, RSAKeyType}, Algorithm, DecodingKey, Header, Validation, }; -use rsa::PublicKeyParts; use serde::{Deserialize, Serialize}; use thiserror::Error; use tracing::instrument; @@ -68,7 +67,7 @@ impl Jwt { .map(|key| (key, Jwt::algo(&jwk))) .map_err(JwtError::InvalidJwk)?; - tracing::debug!("Kid matched known decoding key"); + tracing::debug!(?jwk, ?algorithm, "Kid matched known decoding key"); let data = decode(token, &key, &Validation::new(algorithm?)).map_err(JwtError::Decode)?; @@ -93,6 +92,8 @@ impl Jwt { #[derive(Debug, Error)] pub enum JwtSignerError { + #[error("Failed to encode header")] + Header(serde_json::Error), #[error("Failed to generate signer: {0}")] InvalidKey(SigningKeyError), #[error("Failed to serialize claims: {0}")] @@ -113,11 +114,13 @@ impl JwtSigner { let jwk = key.as_jwk().await?; let mut header = Header::new(Algorithm::RS256); header.kid = Some(key.kid().to_string()); + let encoded_header = to_base64_json(&header)?; + let signer = key.as_signer().await.map_err(JwtSignerError::InvalidKey)?; Ok(Self { header, - encoded_header: String::new(), + encoded_header, jwk, signer, }) @@ -145,10 +148,7 @@ impl JwtSigner { .map_err(JwtSignerError::Signature)?; let enc_signature = URL_SAFE_NO_PAD.encode(signature); - Ok(format!( - "{}.{}.{}", - self.encoded_header, encoded_claims, enc_signature - )) + Ok(format!("{}.{}", message, enc_signature)) } pub fn header(&self) -> &Header { @@ -160,6 +160,8 @@ impl JwtSigner { } } +use rsa::traits::PublicKeyParts; + impl AsymmetricKey { pub async fn as_jwk(&self) -> Result { let key_id = self.kid(); diff --git a/rfd-api/src/authn/key.rs b/rfd-api/src/authn/key.rs index da6bde1..f4a5812 100644 --- a/rfd-api/src/authn/key.rs +++ b/rfd-api/src/authn/key.rs @@ -40,12 +40,7 @@ impl RawApiKey { pub async fn sign(self, signer: &dyn Signer) -> Result { let key = format!("{}.{}", self.id, self.clear); - let signature = hex::encode( - signer - .sign(&key) - .await - .map_err(ApiKeyError::Signing)?, - ); + let signature = hex::encode(signer.sign(&key).await.map_err(ApiKeyError::Signing)?); Ok(SignedApiKey::new(key, signature)) } } @@ -55,18 +50,14 @@ impl TryFrom<&str> for RawApiKey { fn try_from(value: &str) -> Result { match value.split_once(".") { - Some((id, key)) => { - Ok(RawApiKey { - id: id.parse().map_err(|err| { - tracing::info!(?err, "Api key prefix is not a valid uuid"); - ApiKeyError::FailedToParse - })?, - clear: key.to_string(), - }) - } - None => { - Err(ApiKeyError::FailedToParse) - } + Some((id, key)) => Ok(RawApiKey { + id: id.parse().map_err(|err| { + tracing::info!(?err, "Api key prefix is not a valid uuid"); + ApiKeyError::FailedToParse + })?, + clear: key.to_string(), + }), + None => Err(ApiKeyError::FailedToParse), } } } @@ -78,10 +69,7 @@ pub struct SignedApiKey { impl SignedApiKey { fn new(key: String, signature: String) -> Self { - Self { - key, - signature, - } + Self { key, signature } } pub fn key(self) -> String { @@ -101,7 +89,7 @@ mod tests { use crate::util::tests::mock_key; #[tokio::test] - async fn test_rejects_invalid_source() { + async fn test_generates_signatures() { let id = Uuid::new_v4(); let signer = mock_key().as_signer().await.unwrap(); diff --git a/rfd-api/src/authn/mod.rs b/rfd-api/src/authn/mod.rs index 59a0ade..17d9e2a 100644 --- a/rfd-api/src/authn/mod.rs +++ b/rfd-api/src/authn/mod.rs @@ -6,8 +6,9 @@ use dropshot_authorization_header::bearer::BearerAuth; use google_cloudkms1::{api::AsymmetricSignRequest, hyper_rustls::HttpsConnector, CloudKMS}; use hyper::client::HttpConnector; use rsa::{ + pkcs1v15::Signature, + pkcs1v15::{SigningKey, VerifyingKey}, pkcs8::{DecodePrivateKey, DecodePublicKey}, - pss::{BlindedSigningKey, Signature, VerifyingKey}, signature::{Keypair, RandomizedSigner, SignatureEncoding, Verifier}, RsaPrivateKey, RsaPublicKey, }; @@ -18,9 +19,10 @@ use thiserror::Error; use tracing::instrument; use crate::{ + authn::key::RawApiKey, config::AsymmetricKey, context::ApiContext, - util::{cloud_kms_client, response::unauthorized}, authn::key::RawApiKey, + util::{cloud_kms_client, response::unauthorized}, }; use self::jwt::Jwt; @@ -93,6 +95,8 @@ impl From for HttpError { pub enum SigningKeyError { #[error("Cloud signing failed: {0}")] CloudKmsError(#[from] CloudKmsError), + #[error("Failed to immediately verify generated signature")] + GeneratedInvalidSignature, #[error("Failed to parse public key: {0}")] InvalidPublicKey(#[from] rsa::pkcs8::spki::Error), #[error("Invalid signature: {0}")] @@ -107,7 +111,7 @@ pub trait Signer: Send + Sync { // A signer that stores a local in memory key for signing new JWTs pub struct LocalKey { - signing_key: BlindedSigningKey, + signing_key: SigningKey, verifying_key: VerifyingKey, } @@ -115,14 +119,21 @@ pub struct LocalKey { impl Signer for LocalKey { #[instrument(skip(self, message), err(Debug))] async fn sign(&self, message: &str) -> Result, SigningKeyError> { + tracing::trace!("Signing message"); let mut rng = rand::thread_rng(); - Ok(self + let signature = self .signing_key .sign_with_rng(&mut rng, message.as_bytes()) - .to_vec()) + .to_vec(); + + self.verify(message, &Signature::try_from(signature.as_ref()).unwrap()) + .map_err(|_| SigningKeyError::GeneratedInvalidSignature)?; + + Ok(signature) } fn verify(&self, message: &str, signature: &Signature) -> Result<(), SigningKeyError> { + tracing::trace!("Verifying message"); Ok(self.verifying_key.verify(message.as_bytes(), &signature)?) } } @@ -257,6 +268,15 @@ impl AsymmetricKey { } } + pub async fn private_key(&self) -> Result { + Ok(match self { + AsymmetricKey::Local { private, .. } => { + RsaPrivateKey::from_pkcs8_pem(&private).unwrap() + } + _ => unimplemented!(), + }) + } + pub async fn public_key(&self) -> Result { Ok(match self { AsymmetricKey::Local { public, .. } => RsaPublicKey::from_public_key_pem(&public)?, @@ -283,7 +303,7 @@ impl AsymmetricKey { Ok(match self { AsymmetricKey::Local { private, .. } => { let private_key = RsaPrivateKey::from_pkcs8_pem(&private).unwrap(); - let signing_key = BlindedSigningKey::new(private_key); + let signing_key = SigningKey::new(private_key); let verifying_key = signing_key.verifying_key(); Arc::new(LocalKey { diff --git a/rfd-api/src/config.rs b/rfd-api/src/config.rs index e912a4f..902d082 100644 --- a/rfd-api/src/config.rs +++ b/rfd-api/src/config.rs @@ -24,6 +24,7 @@ pub struct AppConfig { pub jwt: JwtConfig, pub spec: Option, pub authn: AuthnProviders, + pub search: SearchConfig, } #[derive(Debug)] @@ -102,20 +103,6 @@ impl AsymmetricKey { Self::Ckms { kid, .. } => kid, } } - - pub fn mock_private_key(&self) -> &str { - match &self { - Self::Local { private, .. } => private, - _ => unimplemented!(), - } - } - - pub fn mock_public_key(&self) -> &str { - match &self { - Self::Local { public, .. } => public, - _ => unimplemented!(), - } - } } #[derive(Debug, Deserialize)] @@ -155,6 +142,13 @@ pub struct GoogleOAuthWebConfig { pub redirect_uri: String, } +#[derive(Debug, Default, Deserialize)] +pub struct SearchConfig { + pub host: String, + pub key: String, + pub index: String, +} + impl AppConfig { pub fn new() -> Result { let config = Config::builder() diff --git a/rfd-api/src/context.rs b/rfd-api/src/context.rs index 756b2f2..1c575a9 100644 --- a/rfd-api/src/context.rs +++ b/rfd-api/src/context.rs @@ -4,6 +4,7 @@ use http::StatusCode; use hyper::{client::HttpConnector, Body, Client}; use hyper_tls::HttpsConnector; use jsonwebtoken::jwk::JwkSet; +use meilisearch_sdk::Client as SearchClient; use oauth2::CsrfToken; use partial_struct::partial; use rfd_model::{ @@ -34,7 +35,7 @@ use crate::{ jwt::{Claims, JwtSigner, JwtSignerError}, AuthError, AuthToken, Signer, }, - config::{AsymmetricKey, JwtConfig, PermissionsConfig}, + config::{AsymmetricKey, JwtConfig, PermissionsConfig, SearchConfig}, email_validator::EmailValidator, endpoints::login::{ oauth::{OAuthProvider, OAuthProviderError, OAuthProviderFn, OAuthProviderName}, @@ -46,6 +47,8 @@ use crate::{ ApiCaller, ApiPermissions, User, UserToken, }; +static UNLIMITED: i64 = 9999999; + pub trait Storage: RfdStore + RfdRevisionStore @@ -92,6 +95,7 @@ pub struct ApiContext { pub jwt: JwtContext, pub secrets: SecretContext, pub oauth_providers: HashMap>, + pub search: SearchContext, } pub struct PermissionsContext { @@ -110,6 +114,11 @@ pub struct SecretContext { pub signer: Arc, } +pub struct SearchContext { + pub client: SearchClient, + pub index: String, +} + pub struct RegisteredAccessToken { pub access_token: AccessToken, pub signed_token: String, @@ -178,6 +187,7 @@ impl ApiContext { permissions: PermissionsConfig, jwt: JwtConfig, keys: Vec, + search: SearchConfig, ) -> Result { let mut jwt_signers = vec![]; @@ -206,6 +216,10 @@ impl ApiContext { signer: keys[0].as_signer().await?, }, oauth_providers: HashMap::new(), + search: SearchContext { + client: SearchClient::new(search.host, search.key), + index: search.index, + }, }) } @@ -252,6 +266,8 @@ impl ApiContext { ) .await?; + // TODO: Verify found signature + if let Some(key) = key.pop() { tracing::debug!("Verified caller key"); @@ -326,37 +342,46 @@ impl ApiContext { pub async fn list_rfds( &self, caller: &Caller, + filter: Option, ) -> Result, StoreError> { - let mut filter = RfdFilter::default(); + let mut filter = filter.unwrap_or_default(); if !caller.can(&ApiPermission::GetAllRfds) { - let numbers = caller.permissions.iter().filter_map(|p| { - match p { + let numbers = caller + .permissions + .iter() + .filter_map(|p| match p { ApiPermission::GetRfd(number) => Some(*number), - _ => None - } - }).collect::>(); + _ => None, + }) + .collect::>(); filter = filter.rfd_number(Some(numbers)); } - let rfds = RfdStore::list( + let mut rfds = RfdStore::list( &*self.storage, filter, - &ListPagination::default().limit(1), + &ListPagination::default().limit(UNLIMITED), ) - .await.tap_err(|err| tracing::error!(?err, "Failed to lookup RFDs"))?; + .await + .tap_err(|err| tracing::error!(?err, "Failed to lookup RFDs"))?; - let rfd_revisions = RfdRevisionStore::list_unique_rfd( + let mut rfd_revisions = RfdRevisionStore::list_unique_rfd( &*self.storage, - RfdRevisionFilter::default() - .rfd(Some(rfds.iter().map(|rfd| rfd.id).collect())), - &ListPagination::default(), + RfdRevisionFilter::default().rfd(Some(rfds.iter().map(|rfd| rfd.id).collect())), + &ListPagination::default().limit(UNLIMITED), ) - .await.tap_err(|err| tracing::error!(?err, "Failed to lookup RFD revisions"))?; + .await + .tap_err(|err| tracing::error!(?err, "Failed to lookup RFD revisions"))?; - let rfd_list = rfds.into_iter().zip(rfd_revisions).map(|(rfd, revision)| { - ListRfd { + rfds.sort_by(|a, b| a.id.cmp(&b.id)); + rfd_revisions.sort_by(|a, b| a.rfd_id.cmp(&b.rfd_id)); + + let mut rfd_list = rfds + .into_iter() + .zip(rfd_revisions) + .map(|(rfd, revision)| ListRfd { id: rfd.id, rfd_number: rfd.rfd_number, link: rfd.link, @@ -367,8 +392,10 @@ impl ApiContext { sha: revision.sha, commit: revision.commit_sha, committed_at: revision.committed_at, - } - }).collect::>(); + }) + .collect::>(); + + rfd_list.sort_by(|a, b| b.rfd_number.cmp(&a.rfd_number)); Ok(rfd_list) } @@ -556,10 +583,12 @@ impl ApiContext { // API User Operations pub async fn get_api_user(&self, id: &Uuid) -> Result, StoreError> { - let user = ApiUserStore::get(&*self.storage, id, false).await?.map(|mut user| { - user.permissions = user.permissions.expand(&user); - user - }); + let user = ApiUserStore::get(&*self.storage, id, false) + .await? + .map(|mut user| { + user.permissions = user.permissions.expand(&user); + user + }); Ok(user) } @@ -583,7 +612,7 @@ impl ApiContext { pub async fn add_permissions_to_user( &self, api_user: &ApiUser, - new_permissions: Permissions + new_permissions: Permissions, ) -> Result { let mut user_update: NewApiUser = api_user.clone().into(); for permission in new_permissions.into_iter() { @@ -597,7 +626,7 @@ impl ApiContext { pub async fn remove_permissions_from_user( &self, api_user: &ApiUser, - new_permissions: Permissions + new_permissions: Permissions, ) -> Result { let mut user_update: NewApiUser = api_user.clone().into(); for permission in new_permissions.into_iter() { @@ -744,7 +773,10 @@ impl ApiContext { pub async fn list_oauth_clients(&self) -> Result, StoreError> { OAuthClientStore::list( &*self.storage, - OAuthClientFilter { id: None, deleted: false }, + OAuthClientFilter { + id: None, + deleted: false, + }, &ListPagination::default(), ) .await @@ -818,7 +850,7 @@ pub(crate) mod tests { use std::sync::Arc; use crate::{ - config::{JwtConfig, PermissionsConfig}, + config::{JwtConfig, PermissionsConfig, SearchConfig}, permissions::ApiPermission, util::tests::{mock_key, AnyEmailValidator}, }; @@ -837,6 +869,7 @@ pub(crate) mod tests { // We are in the context of a test and do not care about the key leaking mock_key(), ], + SearchConfig::default(), ) .await .unwrap() diff --git a/rfd-api/src/endpoints/api_user.rs b/rfd-api/src/endpoints/api_user.rs index 0018403..7100417 100644 --- a/rfd-api/src/endpoints/api_user.rs +++ b/rfd-api/src/endpoints/api_user.rs @@ -661,7 +661,7 @@ mod tests { let no_permissions = ApiCaller { id: user1.id, permissions: Vec::new().into(), - user: user1 + user: user1, }; let resp = list_api_user_tokens_op( diff --git a/rfd-api/src/endpoints/login/oauth/client.rs b/rfd-api/src/endpoints/login/oauth/client.rs index 1aca652..4dd181b 100644 --- a/rfd-api/src/endpoints/login/oauth/client.rs +++ b/rfd-api/src/endpoints/login/oauth/client.rs @@ -38,7 +38,12 @@ async fn list_oauth_clients_op( caller: &ApiCaller, ) -> Result>, HttpError> { Ok(HttpResponseOk( - ctx.list_oauth_clients().await.map_err(ApiError::Storage)?.into_iter().filter(|client| caller.can(&ApiPermission::GetOAuthClient(client.id))).collect(), + ctx.list_oauth_clients() + .await + .map_err(ApiError::Storage)? + .into_iter() + .filter(|client| caller.can(&ApiPermission::GetOAuthClient(client.id))) + .collect(), )) } @@ -67,12 +72,18 @@ async fn create_oauth_client_op( // Create the new client let client = ctx.create_oauth_client().await.map_err(ApiError::Storage)?; - // Give the caller permission to perform actions on the client - ctx.add_permissions_to_user(&caller.user, vec![ - ApiPermission::GetOAuthClient(client.id), - ApiPermission::UpdateOAuthClient(client.id), - ApiPermission::DeleteOAuthClient(client.id), - ].into()).await.map_err(ApiError::Storage)?; + // Give the caller permission to perform actions on the client + ctx.add_permissions_to_user( + &caller.user, + vec![ + ApiPermission::GetOAuthClient(client.id), + ApiPermission::UpdateOAuthClient(client.id), + ApiPermission::DeleteOAuthClient(client.id), + ] + .into(), + ) + .await + .map_err(ApiError::Storage)?; Ok(HttpResponseOk(client)) } else { diff --git a/rfd-api/src/endpoints/login/oauth/code.rs b/rfd-api/src/endpoints/login/oauth/code.rs index 21a5d0f..41e6db0 100644 --- a/rfd-api/src/endpoints/login/oauth/code.rs +++ b/rfd-api/src/endpoints/login/oauth/code.rs @@ -25,7 +25,7 @@ use super::{OAuthProviderNameParam, UserInfoProvider}; use crate::{ authn::key::RawApiKey, context::ApiContext, - endpoints::login::{LoginError, oauth::ClientType}, + endpoints::login::{oauth::ClientType, LoginError}, error::ApiError, util::{ request::RequestCookies, @@ -97,7 +97,9 @@ pub async fn authz_code_redirect( // TODO: This behavior should be changed so that clients are precomputed. We do not need to be // constructing a new client on every request. That said, we need to ensure the client does not // maintain state between requests - let client = provider.as_client(&ClientType::Web).map_err(to_internal_error)?; + let client = provider + .as_client(&ClientType::Web) + .map_err(to_internal_error)?; let (pkce_challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); // Construct a new login attempt with the minimum required values @@ -310,7 +312,7 @@ pub async fn authz_code_exchange( let client_secret = RawApiKey::try_from(body.client_secret.as_str()).map_err(|err| { tracing::warn!(?err, "Failed to parse OAuth client secret"); - + // TODO: Change this to a bad request with invalid_client ? unauthorized() })?; @@ -325,12 +327,11 @@ pub async fn authz_code_exchange( if client.is_secret_valid(&client_secret.id().to_string()) { Ok(client) } else { - // TODO: Change this to a bad request with invalid_client ? Err(client_error(StatusCode::UNAUTHORIZED, "Invalid secret")) } } else { - // TODO: Change this to a bad request with invalid_client ? + // TODO: Change this to a bad request with invalid_client ? Err(client_error( StatusCode::UNAUTHORIZED, "Invalid redirect uri", @@ -366,7 +367,9 @@ pub async fn authz_code_exchange( })?; // Exchange the stored authorization code with the remote provider for a remote access token - let client = provider.as_client(&ClientType::Web).map_err(to_internal_error)?; + let client = provider + .as_client(&ClientType::Web) + .map_err(to_internal_error)?; let mut request = client.exchange_code(AuthorizationCode::new( attempt.provider_authz_code.ok_or_else(|| { internal_error("Expected authorization code to exist due to attempt state") diff --git a/rfd-api/src/endpoints/login/oauth/device_token.rs b/rfd-api/src/endpoints/login/oauth/device_token.rs index ab76afb..64edc72 100644 --- a/rfd-api/src/endpoints/login/oauth/device_token.rs +++ b/rfd-api/src/endpoints/login/oauth/device_token.rs @@ -10,7 +10,9 @@ use tap::TapFallible; use trace_request::trace_request; use tracing::instrument; -use super::{OAuthProvider, OAuthProviderInfo, OAuthProviderNameParam, UserInfoProvider, ClientType}; +use super::{ + ClientType, OAuthProvider, OAuthProviderInfo, OAuthProviderNameParam, UserInfoProvider, +}; use crate::{ context::ApiContext, endpoints::login::LoginError, error::ApiError, util::response::bad_request, }; @@ -34,9 +36,10 @@ pub async fn get_device_provider( .await .map_err(ApiError::OAuth)?; - Ok(HttpResponseOk( - provider.provider_info(&rqctx.context().public_url, &ClientType::Device), - )) + Ok(HttpResponseOk(provider.provider_info( + &rqctx.context().public_url, + &ClientType::Device, + ))) } #[derive(Debug, Deserialize, JsonSchema, Serialize)] @@ -65,15 +68,17 @@ impl AccessTokenExchange { req: AccessTokenExchangeRequest, provider: &Box, ) -> Option { - provider.client_secret(&ClientType::Device).map(|client_secret| Self { - provider: ProviderTokenExchange { - client_id: provider.client_id(&ClientType::Device).to_string(), - device_code: req.device_code, - grant_type: req.grant_type, - client_secret: client_secret.to_string(), - }, - expires_at: req.expires_at, - }) + provider + .client_secret(&ClientType::Device) + .map(|client_secret| Self { + provider: ProviderTokenExchange { + client_id: provider.client_id(&ClientType::Device).to_string(), + device_code: req.device_code, + grant_type: req.grant_type, + client_secret: client_secret.to_string(), + }, + expires_at: req.expires_at, + }) } } diff --git a/rfd-api/src/endpoints/login/oauth/github.rs b/rfd-api/src/endpoints/login/oauth/github.rs index 7828516..d5d593d 100644 --- a/rfd-api/src/endpoints/login/oauth/github.rs +++ b/rfd-api/src/endpoints/login/oauth/github.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use crate::endpoints::login::{ExternalUserId, UserInfo, UserInfoError}; -use super::{ExtractUserInfo, OAuthProvider, OAuthProviderName, ClientType}; +use super::{ClientType, ExtractUserInfo, OAuthProvider, OAuthProviderName}; pub struct GitHubOAuthProvider { public: GitHubPublicProvider, diff --git a/rfd-api/src/endpoints/login/oauth/google.rs b/rfd-api/src/endpoints/login/oauth/google.rs index b5be1e4..ca16fc5 100644 --- a/rfd-api/src/endpoints/login/oauth/google.rs +++ b/rfd-api/src/endpoints/login/oauth/google.rs @@ -5,7 +5,10 @@ use serde::Deserialize; use crate::endpoints::login::{ExternalUserId, UserInfo, UserInfoError}; -use super::{ExtractUserInfo, OAuthProvider, OAuthProviderName, ClientType, OAuthPublicCredentials, OAuthPrivateCredentials}; +use super::{ + ClientType, ExtractUserInfo, OAuthPrivateCredentials, OAuthProvider, OAuthProviderName, + OAuthPublicCredentials, +}; pub struct GoogleOAuthProvider { device_public: OAuthPublicCredentials, @@ -28,10 +31,18 @@ impl GoogleOAuthProvider { web_client_secret: String, ) -> Self { Self { - device_public: OAuthPublicCredentials { client_id: device_client_id }, - device_private: Some(OAuthPrivateCredentials { client_secret: device_client_secret }), - web_public: OAuthPublicCredentials { client_id: web_client_id }, - web_private: Some(OAuthPrivateCredentials { client_secret: web_client_secret }), + device_public: OAuthPublicCredentials { + client_id: device_client_id, + }, + device_private: Some(OAuthPrivateCredentials { + client_secret: device_client_secret, + }), + web_public: OAuthPublicCredentials { + client_id: web_client_id, + }, + web_private: Some(OAuthPrivateCredentials { + client_secret: web_client_secret, + }), } } } @@ -79,8 +90,14 @@ impl OAuthProvider for GoogleOAuthProvider { fn client_secret(&self, client_type: &ClientType) -> Option<&str> { match client_type { - ClientType::Device => self.device_private.as_ref().map(|private| private.client_secret.as_str()), - ClientType::Web => self.web_private.as_ref().map(|private| private.client_secret.as_str()), + ClientType::Device => self + .device_private + .as_ref() + .map(|private| private.client_secret.as_str()), + ClientType::Web => self + .web_private + .as_ref() + .map(|private| private.client_secret.as_str()), } } diff --git a/rfd-api/src/endpoints/rfd.rs b/rfd-api/src/endpoints/rfd.rs index a660ef5..6c58e6a 100644 --- a/rfd-api/src/endpoints/rfd.rs +++ b/rfd-api/src/endpoints/rfd.rs @@ -1,15 +1,17 @@ -use dropshot::{endpoint, HttpError, HttpResponseOk, Path, RequestContext}; +use dropshot::{endpoint, HttpError, HttpResponseOk, Path, Query, RequestContext}; use http::StatusCode; +use rfd_model::storage::RfdFilter; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use trace_request::trace_request; use tracing::instrument; use crate::{ context::{ApiContext, FullRfd, ListRfd}, + error::ApiError, permissions::ApiPermission, - util::response::{client_error, internal_error, not_found}, - ApiCaller, error::ApiError, + util::response::{client_error, internal_error, not_found, unauthorized}, + ApiCaller, }; #[derive(Debug, Deserialize, JsonSchema)] @@ -37,7 +39,10 @@ async fn get_rfds_op( ctx: &ApiContext, caller: &ApiCaller, ) -> Result>, HttpError> { - let rfds = ctx.list_rfds(caller).await.map_err(ApiError::Storage)?; + let rfds = ctx + .list_rfds(caller, None) + .await + .map_err(ApiError::Storage)?; Ok(HttpResponseOk(rfds)) } @@ -89,3 +94,72 @@ async fn get_rfd_op( )) } } + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct RfdSearchQuery { + q: String, +} + +/// Search the RFD index and get a list of results +#[trace_request] +#[endpoint { + method = GET, + path = "/rfd-search", +}] +#[instrument(skip(rqctx), fields(request_id = rqctx.request_id), err(Debug))] +pub async fn search_rfds( + rqctx: RequestContext, + query: Query, +) -> Result>, HttpError> { + let ctx = rqctx.context(); + let auth = ctx.authn_token(&rqctx).await?; + search_rfds_op(ctx, &ctx.get_caller(&auth).await?, query.into_inner()).await +} + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +struct MinimalSearchResult { + rfd_number: i32, +} + +#[instrument(skip(ctx, caller), fields(caller = ?caller.id), err(Debug))] +async fn search_rfds_op( + ctx: &ApiContext, + caller: &ApiCaller, + query: RfdSearchQuery, +) -> Result>, HttpError> { + if caller.can(&ApiPermission::SearchRfds) { + let results = ctx + .search + .client + .index(&ctx.search.index) + .search() + .with_query(&query.q) + .with_limit(999999) + .execute::() + .await; + tracing::trace!(?results, "Fetched search results from remote"); + + match results { + Ok(results) => { + let rfds = results + .hits + .into_iter() + .map(|result| result.result.rfd_number) + .collect::>(); + + let found_rfds = ctx + .list_rfds(caller, Some(RfdFilter::default().rfd_number(Some(rfds)))) + .await + .map_err(ApiError::Storage)?; + + Ok(HttpResponseOk(found_rfds)) + } + Err(err) => { + tracing::error!(?err, "Search request failed"); + Err(internal_error("Search failed".to_string())) + } + } + } else { + Err(unauthorized()) + } +} diff --git a/rfd-api/src/main.rs b/rfd-api/src/main.rs index b04a57e..f31ce71 100644 --- a/rfd-api/src/main.rs +++ b/rfd-api/src/main.rs @@ -28,7 +28,6 @@ mod email_validator; mod endpoints; mod error; mod permissions; -mod seed; mod server; mod util; @@ -64,12 +63,10 @@ async fn main() -> Result<(), Box> { config.permissions, config.jwt, config.keys, + config.search, ) .await?; - // let initial_user = seed::seed(&context).await.unwrap(); - // panic!("{:#?}", initial_user); - if let Some(github) = config.authn.oauth.github { context.insert_oauth_provider( OAuthProviderName::GitHub, diff --git a/rfd-api/src/permissions.rs b/rfd-api/src/permissions.rs index 5b9516b..6802638 100644 --- a/rfd-api/src/permissions.rs +++ b/rfd-api/src/permissions.rs @@ -46,47 +46,47 @@ impl PermissionStorage for Permissions { for p in self.iter() { match p { - ApiPermission::GetRfd(number) => { rfds.insert(*number); }, - ApiPermission::GetDiscussion(number) => { discussions.insert(*number); }, - ApiPermission::GetOAuthClient(id) => { read_oauth_clients.insert(*id); }, - ApiPermission::UpdateOAuthClient(id) => { update_oauth_clients.insert(*id); }, - ApiPermission::DeleteOAuthClient(id) => { delete_oauth_clients.insert(*id); }, - - ApiPermission::GetApiUser(id) => { - contracted.push(if id == user.id() { - ApiPermission::GetApiUserSelf - } else { - ApiPermission::GetApiUser(*id) - }) - } - ApiPermission::CreateApiUserToken(id) => { - contracted.push(if id == user.id() { - ApiPermission::CreateApiUserTokenSelf - } else { - ApiPermission::CreateApiUserToken(*id) - }) - } - ApiPermission::GetApiUserToken(id) => { - contracted.push(if id == user.id() { - ApiPermission::GetApiUserTokenSelf - } else { - ApiPermission::GetApiUserToken(*id) - }) - } - ApiPermission::DeleteApiUserToken(id) => { - contracted.push(if id == user.id() { - ApiPermission::DeleteApiUserTokenSelf - } else { - ApiPermission::DeleteApiUserToken(*id) - }) - } - ApiPermission::UpdateApiUser(id) => { - contracted.push(if id == user.id() { - ApiPermission::UpdateApiUserSelf - } else { - ApiPermission::UpdateApiUser(*id) - }) + ApiPermission::GetRfd(number) => { + rfds.insert(*number); } + ApiPermission::GetDiscussion(number) => { + discussions.insert(*number); + } + ApiPermission::GetOAuthClient(id) => { + read_oauth_clients.insert(*id); + } + ApiPermission::UpdateOAuthClient(id) => { + update_oauth_clients.insert(*id); + } + ApiPermission::DeleteOAuthClient(id) => { + delete_oauth_clients.insert(*id); + } + + ApiPermission::GetApiUser(id) => contracted.push(if id == user.id() { + ApiPermission::GetApiUserSelf + } else { + ApiPermission::GetApiUser(*id) + }), + ApiPermission::CreateApiUserToken(id) => contracted.push(if id == user.id() { + ApiPermission::CreateApiUserTokenSelf + } else { + ApiPermission::CreateApiUserToken(*id) + }), + ApiPermission::GetApiUserToken(id) => contracted.push(if id == user.id() { + ApiPermission::GetApiUserTokenSelf + } else { + ApiPermission::GetApiUserToken(*id) + }), + ApiPermission::DeleteApiUserToken(id) => contracted.push(if id == user.id() { + ApiPermission::DeleteApiUserTokenSelf + } else { + ApiPermission::DeleteApiUserToken(*id) + }), + ApiPermission::UpdateApiUser(id) => contracted.push(if id == user.id() { + ApiPermission::UpdateApiUserSelf + } else { + ApiPermission::UpdateApiUser(*id) + }), other => contracted.push(other.clone()), } @@ -110,85 +110,93 @@ impl PermissionStorage for Permissions { for number in numbers { expanded.push(ApiPermission::GetRfd(*number)) } - }, + } ApiPermission::GetDiscussions(numbers) => { for number in numbers { expanded.push(ApiPermission::GetDiscussion(*number)) } - }, + } ApiPermission::GetOAuthClients(ids) => { for id in ids { expanded.push(ApiPermission::GetOAuthClient(*id)) } - }, + } ApiPermission::UpdateOAuthClients(ids) => { for id in ids { expanded.push(ApiPermission::UpdateOAuthClient(*id)) } - }, + } ApiPermission::DeleteOAuthClients(ids) => { for id in ids { expanded.push(ApiPermission::DeleteOAuthClient(*id)) } - }, + } ApiPermission::GetAssignedRfds => { expanded.push(p.clone()); for p in user.permissions().iter() { match p { - ApiPermission::GetRfd(number) => expanded.push(ApiPermission::GetRfd(*number)), + ApiPermission::GetRfd(number) => { + expanded.push(ApiPermission::GetRfd(*number)) + } _ => (), } } - }, + } ApiPermission::GetAssignedOAuthClients => { expanded.push(p.clone()); for p in user.permissions().iter() { match p { - ApiPermission::GetOAuthClient(id) => expanded.push(ApiPermission::GetOAuthClient(*id)), + ApiPermission::GetOAuthClient(id) => { + expanded.push(ApiPermission::GetOAuthClient(*id)) + } _ => (), } } - }, + } ApiPermission::UpdateAssignedOAuthClients => { expanded.push(p.clone()); for p in user.permissions().iter() { match p { - ApiPermission::UpdateOAuthClient(id) => expanded.push(ApiPermission::UpdateOAuthClient(*id)), + ApiPermission::UpdateOAuthClient(id) => { + expanded.push(ApiPermission::UpdateOAuthClient(*id)) + } _ => (), } } - }, + } ApiPermission::DeleteAssignedOAuthClients => { expanded.push(p.clone()); for p in user.permissions().iter() { match p { - ApiPermission::DeleteOAuthClient(id) => expanded.push(ApiPermission::DeleteOAuthClient(*id)), + ApiPermission::DeleteOAuthClient(id) => { + expanded.push(ApiPermission::DeleteOAuthClient(*id)) + } _ => (), } } - }, + } ApiPermission::GetApiUserSelf => { expanded.push(p.clone()); expanded.push(ApiPermission::GetApiUser(*user.id())) - }, + } ApiPermission::CreateApiUserTokenSelf => { expanded.push(p.clone()); expanded.push(ApiPermission::CreateApiUserToken(*user.id())) - }, + } ApiPermission::GetApiUserTokenSelf => { expanded.push(p.clone()); expanded.push(ApiPermission::GetApiUserToken(*user.id())) - }, + } ApiPermission::DeleteApiUserTokenSelf => { expanded.push(p.clone()); expanded.push(ApiPermission::DeleteApiUserToken(*user.id())) - }, + } ApiPermission::UpdateApiUserSelf => { expanded.push(p.clone()); expanded.push(ApiPermission::UpdateApiUser(*user.id())) - }, + } other => expanded.push(other.clone()), } @@ -198,7 +206,9 @@ impl PermissionStorage for Permissions { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, PartialOrd, Ord)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, PartialOrd, Ord, +)] pub enum ApiPermission { // User information permissions CreateApiUserToken(Uuid), @@ -226,6 +236,7 @@ pub enum ApiPermission { GetDiscussions(BTreeSet), GetAllDiscussions, GetAssignedDiscussions, + SearchRfds, // OAuth client manage permissions CreateOAuthClient, @@ -238,4 +249,4 @@ pub enum ApiPermission { DeleteOAuthClient(Uuid), DeleteOAuthClients(BTreeSet), DeleteAssignedOAuthClients, -} \ No newline at end of file +} diff --git a/rfd-api/src/seed.rs b/rfd-api/src/seed.rs deleted file mode 100644 index d3ed6d8..0000000 --- a/rfd-api/src/seed.rs +++ /dev/null @@ -1,83 +0,0 @@ -use chrono::{Duration, Utc}; -use rfd_model::{storage::StoreError, ApiUser, NewApiUser, NewApiKey}; -use serde::Serialize; -use thiserror::Error; -use uuid::Uuid; - -use crate::{ - authn::key::RawApiKey, - context::ApiContext, - endpoints::api_user::InitialApiKeyResponse, - permissions::ApiPermission, -}; - -#[derive(Debug, Serialize)] -pub struct SeedApiUser { - pub user: ApiUser, - pub token: InitialApiKeyResponse, -} - -#[derive(Debug, Error)] -pub enum SeedError { - #[error("Unable create a seed user in a storage system that already contains users")] - ExistingData, - #[error("Storage failed when creating seed user: {0}")] - Storage(#[from] StoreError), -} - -pub async fn seed(ctx: &ApiContext) -> Result { - if !ctx.is_empty().await? { - tracing::error!("Failed to start service with seeded user. Data in the configured database already exists"); - return Err(SeedError::ExistingData); - } - - let user_id = Uuid::new_v4(); - - let user = ctx - .update_api_user(NewApiUser { - id: user_id, - permissions: vec![ - ApiPermission::CreateApiUserTokenAll, - ApiPermission::GetApiUserAll, - ApiPermission::GetApiUserTokenSelf, - ApiPermission::DeleteApiUserTokenAll, - ApiPermission::CreateApiUser, - ApiPermission::UpdateApiUserAll, - ApiPermission::GetAllRfds, - ApiPermission::GetAllDiscussions, - ApiPermission::CreateOAuthClient, - ApiPermission::GetAssignedOAuthClients, - ApiPermission::UpdateAssignedOAuthClients, - ApiPermission::DeleteAssignedOAuthClients, - ] - .into(), - }) - .await?; - - let token_id = Uuid::new_v4(); - let raw_key = RawApiKey::generate::<24>(&token_id); - let encrypted_key = raw_key.sign(&*ctx.secrets.signer).await.unwrap(); - - let stored_token = ctx - .create_api_user_token( - NewApiKey { - id: token_id, - api_user_id: user.id, - key_signature: encrypted_key.signature().to_string(), - permissions: user.permissions.clone(), - expires_at: Utc::now() + Duration::seconds(7 * 24 * 60 * 60), - }, - &user, - ) - .await?; - - Ok(SeedApiUser { - user, - token: InitialApiKeyResponse { - id: stored_token.id, - key: encrypted_key.key(), - permissions: stored_token.permissions, - created_at: stored_token.created_at, - }, - }) -} diff --git a/rfd-api/src/server.rs b/rfd-api/src/server.rs index d5a37d7..1d518b5 100644 --- a/rfd-api/src/server.rs +++ b/rfd-api/src/server.rs @@ -22,7 +22,7 @@ use crate::{ code::{authz_code_callback, authz_code_exchange, authz_code_redirect}, device_token::{exchange_device_token, get_device_provider}, }, - rfd::{get_rfd, get_rfds}, + rfd::{get_rfd, get_rfds, search_rfds}, webhook::github_webhook, }, }; @@ -73,6 +73,7 @@ pub fn server( // RFDs api.register(get_rfds).unwrap(); api.register(get_rfd).unwrap(); + api.register(search_rfds).unwrap(); // Webhooks api.register(github_webhook).unwrap(); diff --git a/rfd-cli/Cargo.toml b/rfd-cli/Cargo.toml index 7391935..2330bae 100644 --- a/rfd-cli/Cargo.toml +++ b/rfd-cli/Cargo.toml @@ -12,10 +12,13 @@ clap = { workspace = true } config = { workspace = true } dirs = { workspace = true } oauth2 = { workspace = true } +progenitor-client = { workspace = true } reqwest = { workspace = true } rfd-sdk = { path = "../rfd-sdk" } serde = { workspace = true } serde_json = { workspace = true } +tabwriter = { workspace = true } +textwrap = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } toml = { workspace = true } uuid = { workspace = true, features = ["serde"] } \ No newline at end of file diff --git a/rfd-cli/src/generated/cli.rs b/rfd-cli/src/generated/cli.rs index f7b65bb..5e808d7 100644 --- a/rfd-cli/src/generated/cli.rs +++ b/rfd-cli/src/generated/cli.rs @@ -2,14 +2,19 @@ use rfd_sdk::*; -pub struct Cli { +pub struct Cli { client: rfd_sdk::Client, over: T, + output: U, } impl Cli { pub fn new(client: rfd_sdk::Client) -> Self { - Self { client, over: () } + Self { + client, + over: (), + output: (), + } } pub fn get_command(cmd: CliCommand) -> clap::Command { @@ -40,6 +45,7 @@ impl Cli { CliCommand::DeleteOauthClientSecret => Self::cli_delete_oauth_client_secret(), CliCommand::GetRfds => Self::cli_get_rfds(), CliCommand::GetRfd => Self::cli_get_rfd(), + CliCommand::SearchRfds => Self::cli_search_rfds(), CliCommand::GetSelf => Self::cli_get_self(), } } @@ -512,14 +518,29 @@ impl Cli { .about("Get the latest representation of an RFD") } + pub fn cli_search_rfds() -> clap::Command { + clap::Command::new("") + .arg( + clap::Arg::new("q") + .long("q") + .value_parser(clap::value_parser!(String)) + .required(true), + ) + .about("Search the RFD index and get a list of results") + } + pub fn cli_get_self() -> clap::Command { clap::Command::new("").about("Retrieve the user information of the calling user") } } -impl Cli { - pub fn new_with_override(client: rfd_sdk::Client, over: T) -> Self { - Self { client, over } +impl Cli { + pub fn new_with_override(client: rfd_sdk::Client, over: T, output: U) -> Self { + Self { + client, + over, + output, + } } pub async fn execute(&self, cmd: CliCommand, matches: &clap::ArgMatches) { @@ -590,6 +611,9 @@ impl Cli { CliCommand::GetRfd => { self.execute_get_rfd(matches).await; } + CliCommand::SearchRfds => { + self.execute_search_rfds(matches).await; + } CliCommand::GetSelf => { self.execute_get_self(matches).await; } @@ -609,12 +633,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_create_api_user(Ok(r.into_inner())), + Err(r) => self.output.output_create_api_user(Err(r)), } } @@ -629,12 +649,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_api_user(Ok(r.into_inner())), + Err(r) => self.output.output_get_api_user(Err(r)), } } @@ -655,12 +671,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_update_api_user(Ok(r.into_inner())), + Err(r) => self.output.output_update_api_user(Err(r)), } } @@ -675,12 +687,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_list_api_user_tokens(Ok(r.into_inner())), + Err(r) => self.output.output_list_api_user_tokens(Err(r)), } } @@ -706,12 +714,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_create_api_user_token(Ok(r.into_inner())), + Err(r) => self.output.output_create_api_user_token(Err(r)), } } @@ -730,12 +734,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_api_user_token(Ok(r.into_inner())), + Err(r) => self.output.output_get_api_user_token(Err(r)), } } @@ -754,12 +754,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_delete_api_user_token(Ok(r.into_inner())), + Err(r) => self.output.output_delete_api_user_token(Err(r)), } } @@ -780,12 +776,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_github_webhook(Ok(r.into_inner())), + Err(r) => self.output.output_github_webhook(Err(r)), } } @@ -851,9 +843,7 @@ impl Cli { Ok(r) => { todo!() } - Err(r) => { - println!("error\n{:#?}", r) - } + Err(r) => self.output.output_authz_code_callback(Err(r)), } } @@ -899,12 +889,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_authz_code_exchange(Ok(r.into_inner())), + Err(r) => self.output.output_authz_code_exchange(Err(r)), } } @@ -919,12 +905,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_device_provider(Ok(r.into_inner())), + Err(r) => self.output.output_get_device_provider(Err(r)), } } @@ -975,12 +957,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_list_oauth_clients(Ok(r.into_inner())), + Err(r) => self.output.output_list_oauth_clients(Err(r)), } } @@ -991,12 +969,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_create_oauth_client(Ok(r.into_inner())), + Err(r) => self.output.output_create_oauth_client(Err(r)), } } @@ -1011,12 +985,8 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_oauth_client(Ok(r.into_inner())), + Err(r) => self.output.output_get_oauth_client(Err(r)), } } @@ -1042,12 +1012,10 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self + .output + .output_create_oauth_client_redirect_uri(Ok(r.into_inner())), + Err(r) => self.output.output_create_oauth_client_redirect_uri(Err(r)), } } @@ -1066,12 +1034,10 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self + .output + .output_delete_oauth_client_redirect_uri(Ok(r.into_inner())), + Err(r) => self.output.output_delete_oauth_client_redirect_uri(Err(r)), } } @@ -1086,12 +1052,10 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self + .output + .output_create_oauth_client_secret(Ok(r.into_inner())), + Err(r) => self.output.output_create_oauth_client_secret(Err(r)), } } @@ -1110,12 +1074,10 @@ impl Cli { .unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self + .output + .output_delete_oauth_client_secret(Ok(r.into_inner())), + Err(r) => self.output.output_delete_oauth_client_secret(Err(r)), } } @@ -1124,12 +1086,8 @@ impl Cli { self.over.execute_get_rfds(matches, &mut request).unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_rfds(Ok(r.into_inner())), + Err(r) => self.output.output_get_rfds(Err(r)), } } @@ -1142,12 +1100,24 @@ impl Cli { self.over.execute_get_rfd(matches, &mut request).unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_rfd(Ok(r.into_inner())), + Err(r) => self.output.output_get_rfd(Err(r)), + } + } + + pub async fn execute_search_rfds(&self, matches: &clap::ArgMatches) { + let mut request = self.client.search_rfds(); + if let Some(value) = matches.get_one::("q") { + request = request.q(value.clone()); + } + + self.over + .execute_search_rfds(matches, &mut request) + .unwrap(); + let result = request.send().await; + match result { + Ok(r) => self.output.output_search_rfds(Ok(r.into_inner())), + Err(r) => self.output.output_search_rfds(Err(r)), } } @@ -1156,12 +1126,8 @@ impl Cli { self.over.execute_get_self(matches, &mut request).unwrap(); let result = request.send().await; match result { - Ok(r) => { - println!("success\n{:#?}", r) - } - Err(r) => { - println!("error\n{:#?}", r) - } + Ok(r) => self.output.output_get_self(Ok(r.into_inner())), + Err(r) => self.output.output_get_self(Err(r)), } } } @@ -1343,6 +1309,14 @@ pub trait CliOverride { Ok(()) } + fn execute_search_rfds( + &self, + matches: &clap::ArgMatches, + request: &mut builder::SearchRfds, + ) -> Result<(), String> { + Ok(()) + } + fn execute_get_self( &self, matches: &clap::ArgMatches, @@ -1354,6 +1328,145 @@ pub trait CliOverride { impl CliOverride for () {} +pub trait CliOutput { + fn output_create_api_user( + &self, + response: Result>, + ) { + } + + fn output_get_api_user( + &self, + response: Result>, + ) { + } + + fn output_update_api_user( + &self, + response: Result>, + ) { + } + + fn output_list_api_user_tokens( + &self, + response: Result, progenitor_client::Error>, + ) { + } + + fn output_create_api_user_token( + &self, + response: Result>, + ) { + } + + fn output_get_api_user_token( + &self, + response: Result>, + ) { + } + + fn output_delete_api_user_token( + &self, + response: Result>, + ) { + } + + fn output_github_webhook(&self, response: Result<(), progenitor_client::Error>) {} + + fn output_authz_code_redirect(&self, response: Result<(), progenitor_client::Error<()>>) {} + + fn output_authz_code_callback( + &self, + response: Result<(), progenitor_client::Error>, + ) { + } + + fn output_authz_code_exchange( + &self, + response: Result< + types::OAuthAuthzCodeExchangeResponse, + progenitor_client::Error, + >, + ) { + } + + fn output_get_device_provider( + &self, + response: Result>, + ) { + } + + fn output_exchange_device_token(&self, response: Result<(), progenitor_client::Error<()>>) {} + + fn output_list_oauth_clients( + &self, + response: Result, progenitor_client::Error>, + ) { + } + + fn output_create_oauth_client( + &self, + response: Result>, + ) { + } + + fn output_get_oauth_client( + &self, + response: Result>, + ) { + } + + fn output_create_oauth_client_redirect_uri( + &self, + response: Result>, + ) { + } + + fn output_delete_oauth_client_redirect_uri( + &self, + response: Result>, + ) { + } + + fn output_create_oauth_client_secret( + &self, + response: Result>, + ) { + } + + fn output_delete_oauth_client_secret( + &self, + response: Result>, + ) { + } + + fn output_get_rfds( + &self, + response: Result, progenitor_client::Error>, + ) { + } + + fn output_get_rfd( + &self, + response: Result>, + ) { + } + + fn output_search_rfds( + &self, + response: Result, progenitor_client::Error>, + ) { + } + + fn output_get_self( + &self, + response: Result>, + ) { + } +} + +impl CliOutput for () {} + #[derive(Copy, Clone, Debug)] pub enum CliCommand { CreateApiUser, @@ -1378,6 +1491,7 @@ pub enum CliCommand { DeleteOauthClientSecret, GetRfds, GetRfd, + SearchRfds, GetSelf, } @@ -1406,6 +1520,7 @@ impl CliCommand { CliCommand::DeleteOauthClientSecret, CliCommand::GetRfds, CliCommand::GetRfd, + CliCommand::SearchRfds, CliCommand::GetSelf, ] .into_iter() diff --git a/rfd-cli/src/main.rs b/rfd-cli/src/main.rs index 1bd8a25..3d1ad76 100644 --- a/rfd-cli/src/main.rs +++ b/rfd-cli/src/main.rs @@ -3,6 +3,7 @@ use anyhow::{anyhow, Result}; use clap::{Arg, ArgAction, Command, CommandFactory, FromArgMatches}; use generated::cli::*; +use printer::RfdCliPrinter; use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use rfd_sdk::Client; use std::time::Duration; @@ -13,6 +14,7 @@ mod auth; mod cmd; mod err; mod generated; +mod printer; mod store; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -41,7 +43,6 @@ impl Context { pub fn client(&mut self) -> Result<&Client> { if self.client.is_none() { - let mut default_headers = HeaderMap::new(); if let Ok(token) = self.config.token() { @@ -99,6 +100,7 @@ fn cmd_path<'a>(cmd: &CliCommand) -> Option<&'a str> { CliCommand::GetApiUserToken => Some("user token get"), CliCommand::GetRfd => Some("rfd get"), CliCommand::GetRfds => Some("rfd list"), + CliCommand::SearchRfds => Some("rfd search"), CliCommand::GetSelf => Some("self"), CliCommand::ListApiUserTokens => Some("user token list"), CliCommand::UpdateApiUser => Some("user update"), @@ -196,7 +198,7 @@ async fn main() -> Result<(), Box> { sm = sub_matches; } - let cli = Cli::new_with_override(ctx.client()?.clone(), ()); + let cli = Cli::new_with_override(ctx.client()?.clone(), (), RfdCliPrinter {}); cli.execute(node.cmd.unwrap(), sm).await; } }; diff --git a/rfd-cli/src/printer.rs b/rfd-cli/src/printer.rs new file mode 100644 index 0000000..fb6f661 --- /dev/null +++ b/rfd-cli/src/printer.rs @@ -0,0 +1,265 @@ +use rfd_sdk::types::{Error, ListRfd}; +use std::{fs::File, io::Write, process::Command}; +use tabwriter::TabWriter; + +use crate::generated::cli::CliOutput; + +static HEADER_COLOR: &str = "\x1b[38;5;245m"; +static TEXT_COLOR: &str = "253"; +static ERROR_COLOR: &str = "\x1b[38;2;251;110;136m"; +static RESET_COLOR: &str = "\x1b[0m"; +pub struct RfdCliPrinter {} + +impl CliOutput for RfdCliPrinter { + fn output_get_rfd( + &self, + response: Result>, + ) { + match response { + Ok(rfd) => { + let mut tw = TabWriter::new(vec![]).ansi(true); + + writeln!( + &mut tw, + "{}Number:\t\x1b[38;5;{}m{}", + HEADER_COLOR, TEXT_COLOR, rfd.rfd_number, + ); + writeln!( + &mut tw, + "{}Title:\t\x1b[38;5;{}m{}", + HEADER_COLOR, TEXT_COLOR, rfd.title, + ); + writeln!( + &mut tw, + "{}State:\t{}{}", + HEADER_COLOR, + state_color(&rfd.state), + rfd.state.unwrap_or_default(), + ); + writeln!( + &mut tw, + "{}Authors:\t\x1b[38;5;{}m{}", + HEADER_COLOR, + TEXT_COLOR, + rfd.authors.unwrap_or_default(), + ); + writeln!( + &mut tw, + "{}Latest Commit:\t\x1b[38;5;{}m{}", + HEADER_COLOR, TEXT_COLOR, rfd.commit, + ); + writeln!( + &mut tw, + "{}Updated At:\t\x1b[38;5;{}m{}", + HEADER_COLOR, TEXT_COLOR, rfd.committed_at, + ); + writeln!( + &mut tw, + "{}Url:\t\x1b[38;5;{}mhttps://rfd.shared.oxide.computer/rfd/{}", + HEADER_COLOR, TEXT_COLOR, rfd.rfd_number, + ); + writeln!( + &mut tw, + "{}Discussion Url:\t\x1b[38;5;{}m{}", + HEADER_COLOR, + TEXT_COLOR, + rfd.discussion.unwrap_or_default(), + ); + writeln!(&mut tw, "{}---------------", HEADER_COLOR,); + writeln!(&mut tw, "\x1b[38;5;{}m", TEXT_COLOR); + + let body = print_rfd_html(&rfd.content); + + if let Some((header, body)) = body.split_once("State") { + for line in textwrap::wrap(body.trim_start(), 200).iter().skip(1) { + writeln!(&mut tw, "{}", line); + } + + writeln!(&mut tw, "{}---------------", HEADER_COLOR,); + writeln!(&mut tw, ""); + writeln!( + &mut tw, + "{}...someday the content will be nicely formatted once I can render AsciiDoc to a terminal...", + HEADER_COLOR, + ); + } + + let written = String::from_utf8(tw.into_inner().unwrap()).unwrap(); + println!("{}", written); + } + Err(err) => print_error(err), + } + } + + fn output_get_rfds( + &self, + mut response: Result, progenitor_client::Error>, + ) { + match response { + Ok(mut response) => print_rfd_list(&mut response), + Err(err) => print_error(err), + } + } + + fn output_search_rfds( + &self, + response: Result, progenitor_client::Error>, + ) { + match response { + Ok(mut response) => print_rfd_list(&mut response), + Err(err) => print_error(err), + } + } + + fn output_get_self( + &self, + response: Result>, + ) { + } + + fn output_get_api_user( + &self, + response: Result>, + ) { + } +} + +fn print_error(error: progenitor_client::Error) { + let mut tw = TabWriter::new(vec![]).ansi(true); + + match error { + progenitor_client::Error::CommunicationError(err) => { + writeln!( + &mut tw, + "{}Failed to reach the API server{}", + ERROR_COLOR, RESET_COLOR + ); + writeln!(&mut tw, "{:#?}", err); + } + progenitor_client::Error::ErrorResponse(err) => { + writeln!( + &mut tw, + "{}Received error from the remote API{}", + ERROR_COLOR, RESET_COLOR + ); + writeln!( + &mut tw, + "{}Message\t{}{}", + HEADER_COLOR, RESET_COLOR, err.message + ); + if let Some(code) = &err.error_code { + writeln!(&mut tw, "{}Code\t{}{}", HEADER_COLOR, RESET_COLOR, code); + } + writeln!( + &mut tw, + "{}Request\t{}{}", + HEADER_COLOR, RESET_COLOR, err.request_id + ); + } + progenitor_client::Error::InvalidRequest(err) => { + writeln!(&mut tw, "{}Internal CLI error{}", ERROR_COLOR, RESET_COLOR); + writeln!(&mut tw, "{:?}", err); + writeln!( + &mut tw, + "{}Please report this as a bug against the rfd-api{}", + HEADER_COLOR, RESET_COLOR + ); + } + progenitor_client::Error::InvalidResponsePayload(err) => { + writeln!(&mut tw, "{}Internal CLI error{}", ERROR_COLOR, RESET_COLOR); + writeln!(&mut tw, "{:?}", err); + writeln!( + &mut tw, + "{}Please report this as a bug against the rfd-api{}", + HEADER_COLOR, RESET_COLOR + ); + } + progenitor_client::Error::UnexpectedResponse(err) => { + writeln!(&mut tw, "{}Internal CLI error{}", ERROR_COLOR, RESET_COLOR); + writeln!(&mut tw, "{:?}", err); + writeln!( + &mut tw, + "{}Please report this as a bug against the rfd-api{}", + HEADER_COLOR, RESET_COLOR + ); + } + } + tw.flush().unwrap(); + + let written = String::from_utf8(tw.into_inner().unwrap()).unwrap(); + println!("{}", written); +} + +fn print_rfd_html(content: &str) -> String { + let mut tmp_content = File::create("adoc-source.adoc").unwrap(); + tmp_content.write_all(content.as_bytes()); + + let html = Command::new("asciidoctor") + .arg("adoc-source.adoc") + .output() + .unwrap() + .stdout; + + let text = String::from_utf8( + Command::new("w3m") + .arg("-dump") + .arg("adoc-source.html") + .output() + .unwrap() + .stdout, + ) + .unwrap(); + + std::fs::remove_file("adoc-source.adoc").unwrap(); + std::fs::remove_file("adoc-source.html").unwrap(); + + text +} + +fn print_rfd_list(rfds: &mut [ListRfd]) { + let mut tw = TabWriter::new(vec![]).ansi(true); + + writeln!( + &mut tw, + "\x1b[38;5;{}mNumber\tTitle\tState\tLatest Commit\tUpdated At", + HEADER_COLOR + ); + writeln!( + &mut tw, + "\x1b[38;5;{}m------\t-----\t-----\t-------------\t----------", + HEADER_COLOR + ); + + for mut rfd in rfds.iter_mut() { + rfd.title.truncate(90); + + writeln!( + &mut tw, + "\x1b[38;5;{}m{}\t{}\t{}{}\t\x1b[38;5;{}m{}", + TEXT_COLOR, + rfd.rfd_number, + rfd.title, + state_color(&rfd.state), + rfd.state.as_deref().unwrap_or_default(), + TEXT_COLOR, + // rfd.sha, + rfd.committed_at + ); + } + tw.flush().unwrap(); + + let written = String::from_utf8(tw.into_inner().unwrap()).unwrap(); + println!("{}", written); +} + +fn state_color(state: &Option) -> &'static str { + match state.as_deref() { + Some("published") => "\x1b[38;2;72;213;151m", + Some("committed") => "\x1b[38;2;72;213;151m", + Some("discussion") => "\x1b[38;2;139;161;255m", + Some("prediscussion") => "\x1b[38;2;163;128;203m", + Some("ideation") => "\x1b[38;2;245;185;68m", + Some("abandoned") => "\x1b[38;2;231;231;232m", + _ => "\x1b[38;2;231;231;232m", + } +} diff --git a/rfd-model/src/permissions.rs b/rfd-model/src/permissions.rs index da9ee7b..35106cd 100644 --- a/rfd-model/src/permissions.rs +++ b/rfd-model/src/permissions.rs @@ -82,7 +82,8 @@ where } pub fn intersect(&self, other: &Permissions) -> Permissions { - self.0.intersection(&other.0) + self.0 + .intersection(&other.0) .into_iter() .map(|p| p.clone()) .collect::>() diff --git a/rfd-model/src/storage/postgres.rs b/rfd-model/src/storage/postgres.rs index 02744c1..e0cd6af 100644 --- a/rfd-model/src/storage/postgres.rs +++ b/rfd-model/src/storage/postgres.rs @@ -124,6 +124,8 @@ impl RfdStore for PostgresStore { .get_results_async::(&self.conn) .await?; + tracing::trace!(count = ?results.len(), "Found RFDs"); + Ok(results.into_iter().map(|rfd| rfd.into()).collect()) } @@ -227,7 +229,9 @@ impl RfdRevisionStore for PostgresStore { filter: RfdRevisionFilter, pagination: &ListPagination, ) -> Result, StoreError> { - let mut query = rfd_revision::dsl::rfd_revision.distinct_on(rfd_revision::rfd_id).into_boxed(); + let mut query = rfd_revision::dsl::rfd_revision + .distinct_on(rfd_revision::rfd_id) + .into_boxed(); tracing::trace!(?filter, "Lookup unique RFD revisions"); @@ -261,6 +265,8 @@ impl RfdRevisionStore for PostgresStore { .get_results_async::(&self.conn) .await?; + tracing::trace!(count = ?results.len(), "Found unique RFD revisions"); + Ok(results .into_iter() .map(|revision| revision.into()) @@ -452,6 +458,7 @@ impl JobStore for PostgresStore { job::sha.eq(new_job.sha.clone()), job::rfd.eq(new_job.rfd.clone()), job::webhook_delivery_id.eq(new_job.webhook_delivery_id.clone()), + job::processed.eq(false), job::committed_at.eq(new_job.committed_at.clone()), )) .get_result_async(&self.conn) diff --git a/rfd-model/tests/postgres.rs b/rfd-model/tests/postgres.rs index b36a3b2..f0fc7ce 100644 --- a/rfd-model/tests/postgres.rs +++ b/rfd-model/tests/postgres.rs @@ -24,7 +24,9 @@ fn leakable_dbs() -> Vec { leaks.split(',').map(|s| s.to_string()).collect() } -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, JsonSchema, PartialOrd, Ord)] +#[derive( + Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, JsonSchema, PartialOrd, Ord, +)] enum TestPermission { CreateApiUser, CreateApiKey(Uuid), diff --git a/rfd-processor/Cargo.toml b/rfd-processor/Cargo.toml index 0e6716f..e0979da 100644 --- a/rfd-processor/Cargo.toml +++ b/rfd-processor/Cargo.toml @@ -18,9 +18,13 @@ http = { workspace = true } meilisearch-sdk = { workspace = true } md-5 = { workspace = true } mime_guess = { workspace = true } -octorust = { workspace = true } +octorust = { workspace = true, features = ["httpcache"] } parse-rfd = { path = "../parse-rfd" } regex = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +reqwest-retry = { workspace = true } +reqwest-tracing = { workspace = true } rfd-data = { path = "../rfd-data" } rfd-model = { path = "../rfd-model" } serde = { workspace = true } diff --git a/rfd-processor/src/context.rs b/rfd-processor/src/context.rs index cef3226..d494b71 100644 --- a/rfd-processor/src/context.rs +++ b/rfd-processor/src/context.rs @@ -1,9 +1,14 @@ +use std::time::Duration; + use async_trait::async_trait; use google_drive::{traits::FileOps, Client as GDriveClient}; use google_storage1::{ hyper, hyper::client::HttpConnector, hyper_rustls, hyper_rustls::HttpsConnector, Storage, }; -use octorust::{auth::Credentials, Client as GitHubClient, ClientError}; +use octorust::{ + auth::Credentials, http_cache::FileBasedCache, Client as GitHubClient, ClientError, +}; +use reqwest::Error as ReqwestError; use rfd_model::storage::postgres::PostgresStore; use thiserror::Error; @@ -35,6 +40,8 @@ impl Database { #[derive(Debug, Error)] pub enum ContextError { + #[error(transparent)] + ClientConstruction(ReqwestError), #[error(transparent)] FailedToCreateGitHubClient(#[from] ClientError), #[error("Failed to find GCP credentials {0}")] @@ -46,6 +53,8 @@ pub enum ContextError { } pub struct Context { + pub processor: ProcessorCtx, + pub scanner: ScannerCtx, pub db: Database, pub github: GitHubCtx, pub actions: Vec, @@ -56,10 +65,27 @@ pub struct Context { impl Context { pub async fn new(db: Database, config: &AppConfig) -> Result { - let github_client = GitHubClient::new( + let http = reqwest::Client::builder() + .build() + .map_err(ContextError::ClientConstruction)?; + let retry_policy = + reqwest_retry::policies::ExponentialBackoff::builder().build_with_max_retries(3); + let client = reqwest_middleware::ClientBuilder::new(http) + // Trace HTTP requests. See the tracing crate to make use of these traces. + .with(reqwest_tracing::TracingMiddleware::default()) + // Retry failed requests. + .with(reqwest_retry::RetryTransientMiddleware::new_with_policy( + retry_policy, + )) + .build(); + let http_cache = Box::new(FileBasedCache::new("/tmp/.cache/github")); + + let github_client = GitHubClient::custom( "rfd-processor", Credentials::Token(config.auth.github.token.to_string()), - )?; + client, + http_cache, + ); let repository = GitHubRfdRepo::new( &github_client, @@ -70,6 +96,13 @@ impl Context { .await?; Ok(Self { + processor: ProcessorCtx { + batch_size: config.processor_batch_size, + interval: Duration::from_secs(config.processor_interval), + }, + scanner: ScannerCtx { + interval: Duration::from_secs(config.scanner_interval), + }, db, github: GitHubCtx { client: github_client, @@ -87,6 +120,15 @@ impl Context { } } +pub struct ProcessorCtx { + pub batch_size: i64, + pub interval: Duration, +} + +pub struct ScannerCtx { + pub interval: Duration, +} + pub struct GitHubCtx { pub client: GitHubClient, pub repository: GitHubRfdRepo, diff --git a/rfd-processor/src/github/ext.rs b/rfd-processor/src/github/ext.rs index 5169622..cd15468 100644 --- a/rfd-processor/src/github/ext.rs +++ b/rfd-processor/src/github/ext.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use base64::{prelude::BASE64_STANDARD, DecodeError, Engine}; use octorust::{repos::Repos, types::ContentFile, ClientError}; +use tracing::instrument; #[async_trait] pub trait ReposExt { @@ -15,6 +16,7 @@ pub trait ReposExt { #[async_trait] impl ReposExt for Repos { + #[instrument(skip(self))] async fn get_content_blob( &self, owner: &str, @@ -22,6 +24,7 @@ impl ReposExt for Repos { ref_: &str, file: &str, ) -> Result { + tracing::trace!("Fetching content from GitHub"); let mut file = self.get_content_file(owner, repo, file, ref_).await?.body; // If the content is empty and the encoding is none then we likely hit a "too large" file case. diff --git a/rfd-processor/src/github/mod.rs b/rfd-processor/src/github/mod.rs index a035211..b2c6bd9 100644 --- a/rfd-processor/src/github/mod.rs +++ b/rfd-processor/src/github/mod.rs @@ -114,7 +114,7 @@ impl GitHubRfdRepo { // Each RFD should exist at a path that looks like rfd/{number}/README.adoc for entry in rfd_files { - tracing::trace!(?entry.name, ?entry.path, "Processing file on default branch"); + tracing::trace!(?entry.name, ?entry.path, ?entry, ?entry.sha, "Processing file on default branch"); let path_parts = entry.path.split('/').collect::>(); @@ -199,7 +199,7 @@ impl GitHubRfdRepo { // 404s are returned as errors, but that should not stop processing. This only // means that the branch should be skipped match response { - Ok(Response { body: file, status, .. }) if status == StatusCode::OK => { + Ok(Response { body: file, status, .. }) if status == StatusCode::OK || status == StatusCode::NOT_MODIFIED => { tracing::debug!(?file.path, "Retrieved RFD file contents"); let path_parts = file.path.split('/').collect::>(); @@ -259,7 +259,11 @@ impl GitHubRfdLocation { let mut path = format!("{}/README.adoc", dir); let response = self.fetch_content(&client, &path, &self.commit).await; - if response.is_err() { + if let Err(err) = response { + tracing::trace!( + ?err, + "Failed to find asciidoc README, falling back to markdown" + ); path = format!("{}/README.md", dir); } @@ -282,9 +286,9 @@ impl GitHubRfdLocation { client: &Client, rfd_number: &RfdNumber, ) -> Result, GitHubError> { - tracing::info!("Fetch readme contents"); - let readme_path = self.readme_path(client, rfd_number).await; + tracing::info!(?readme_path, "Fetch readme contents"); + let is_markdown = readme_path.ends_with(".md"); let FetchedRfdContent { parsed, sha, url, .. @@ -326,7 +330,7 @@ impl GitHubRfdLocation { ) -> Result { let file = client .repos() - .get_content_blob(&self.owner, &self.repo, path, ref_) + .get_content_blob(&self.owner, &self.repo, ref_, path) .await?; let decoded = decode_base64(&file.content)?; @@ -541,7 +545,7 @@ pub struct GitHubRfdReadmeLocation { pub branch: GitHubRfdLocation, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct GitHubRfdUpdate { pub number: RfdNumber, pub location: GitHubRfdLocation, diff --git a/rfd-processor/src/main.rs b/rfd-processor/src/main.rs index ec6d863..b52bab6 100644 --- a/rfd-processor/src/main.rs +++ b/rfd-processor/src/main.rs @@ -24,6 +24,9 @@ mod util; #[derive(Debug, Deserialize, Serialize)] pub struct AppConfig { + pub processor_batch_size: i64, + pub processor_interval: u64, + pub scanner_interval: u64, pub database_url: String, pub actions: Vec, pub auth: AuthConfig, @@ -74,7 +77,8 @@ pub struct SearchConfig { impl AppConfig { pub fn new() -> Result { let config = Config::builder() - .add_source(File::with_name("config.toml")) + .add_source(File::with_name("config.toml").required(false)) + .add_source(File::with_name("rfd-processor/config.toml").required(false)) .add_source(Environment::default()) .build()?; @@ -110,12 +114,12 @@ async fn main() -> Result<(), Box> { // Tasks should run for the lifetime of the program. If any of them complete for any reason // then the entire application should exit select! { - value = scanner_handle => { - tracing::info!(?value, "Scanner task exited") - } value = processor_handle => { tracing::info!(?value, "Processor task exited") } + value = scanner_handle => { + tracing::info!(?value, "Scanner task exited") + } }; Ok(()) diff --git a/rfd-processor/src/processor.rs b/rfd-processor/src/processor.rs index 08059cd..b3f6b5b 100644 --- a/rfd-processor/src/processor.rs +++ b/rfd-processor/src/processor.rs @@ -1,5 +1,5 @@ use rfd_model::storage::{JobFilter, JobStore, ListPagination, StoreError}; -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use tap::TapFallible; use thiserror::Error; use tokio::time::interval; @@ -17,10 +17,10 @@ pub enum JobError { } pub async fn processor(ctx: Arc) -> Result<(), JobError> { - let mut interval = interval(Duration::from_secs(1)); + let mut interval = interval(ctx.processor.interval); interval.tick().await; - let pagination = ListPagination::default().limit(10); + let pagination = ListPagination::default().limit(ctx.processor.batch_size); loop { let jobs = JobStore::list( diff --git a/rfd-processor/src/scanner.rs b/rfd-processor/src/scanner.rs index 88956c2..ac82678 100644 --- a/rfd-processor/src/scanner.rs +++ b/rfd-processor/src/scanner.rs @@ -2,7 +2,7 @@ use rfd_model::{ storage::{JobStore, StoreError}, NewJob, }; -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use thiserror::Error; use tokio::time::interval; @@ -20,7 +20,7 @@ pub enum ScannerError { } pub async fn scanner(ctx: Arc) -> Result<(), ScannerError> { - let mut interval = interval(Duration::from_secs(1)); + let mut interval = interval(ctx.scanner.interval); interval.tick().await; loop { @@ -31,7 +31,14 @@ pub async fn scanner(ctx: Arc) -> Result<(), ScannerError> { .await?; for update in updates { - JobStore::upsert(&ctx.db.storage, update.into()).await?; + match JobStore::upsert(&ctx.db.storage, update.clone().into()).await { + Ok(job) => tracing::trace!(?job.id, "Added job to the queue"), + Err(err) => { + // TODO: Do not warn on uniqueness violations. It is expected that the scanner + // picks ups redundant jobs for RFDs that have not changed since the last scan + tracing::warn!(?err, ?update, "Failed to add job") + } + } } } } diff --git a/rfd-sdk/src/generated/sdk.rs b/rfd-sdk/src/generated/sdk.rs index cc1cf72..af5b876 100644 --- a/rfd-sdk/src/generated/sdk.rs +++ b/rfd-sdk/src/generated/sdk.rs @@ -99,6 +99,7 @@ pub mod types { GetAssignedRfds, GetAllDiscussions, GetAssignedDiscussions, + SearchRfds, CreateOAuthClient, GetAssignedOAuthClients, UpdateAssignedOAuthClients, @@ -3063,6 +3064,20 @@ impl Client { builder::GetRfd::new(self) } + /// Search the RFD index and get a list of results + /// + /// Sends a `GET` request to `/rfd-search` + /// + /// ```ignore + /// let response = client.search_rfds() + /// .q(q) + /// .send() + /// .await; + /// ``` + pub fn search_rfds(&self) -> builder::SearchRfds { + builder::SearchRfds::new(self) + } + /// Retrieve the user information of the calling user /// /// Sends a `GET` request to `/self` @@ -4707,6 +4722,64 @@ pub mod builder { } } + /// Builder for [`Client::search_rfds`] + /// + /// [`Client::search_rfds`]: super::Client::search_rfds + #[derive(Debug, Clone)] + pub struct SearchRfds<'a> { + client: &'a super::Client, + q: Result, + } + + impl<'a> SearchRfds<'a> { + pub fn new(client: &'a super::Client) -> Self { + Self { + client, + q: Err("q was not initialized".to_string()), + } + } + + pub fn q(mut self, value: V) -> Self + where + V: std::convert::TryInto, + { + self.q = value + .try_into() + .map_err(|_| "conversion to `String` for q failed".to_string()); + self + } + + /// Sends a `GET` request to `/rfd-search` + pub async fn send(self) -> Result>, Error> { + let Self { client, q } = self; + let q = q.map_err(Error::InvalidRequest)?; + let url = format!("{}/rfd-search", client.baseurl,); + let mut query = Vec::with_capacity(1usize); + query.push(("q", q.to_string())); + let request = client + .client + .get(url) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .query(&query) + .build()?; + let result = client.client.execute(request).await; + let response = result?; + match response.status().as_u16() { + 200u16 => ResponseValue::from_response(response).await, + 400u16..=499u16 => Err(Error::ErrorResponse( + ResponseValue::from_response(response).await?, + )), + 500u16..=599u16 => Err(Error::ErrorResponse( + ResponseValue::from_response(response).await?, + )), + _ => Err(Error::UnexpectedResponse(response)), + } + } + } + /// Builder for [`Client::get_self`] /// /// [`Client::get_self`]: super::Client::get_self