diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b019d0d3d..98b963ccc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,33 @@ on: types: [published] jobs: + release-tools: + name: Build tools/${{ matrix.build }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + build: [system-stats] + steps: + - uses: actions/checkout@v3 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + target: x86_64-unknown-linux-musl + + - name: Build binary + working-directory: tools/${{ matrix.build }} + run: cargo build --verbose --release --target x86_64-unknown-linux-musl + + - name: Upload release archive + uses: softprops/action-gh-release@v1 + with: + files: tools/${{ matrix.build }}/target/x86_64-unknown-linux-musl/release/${{ matrix.build }} + build-release: name: Build release for ${{ matrix.target }} runs-on: ${{ matrix.os }} @@ -26,7 +53,7 @@ jobs: target: aarch64-apple-darwin - build: linux-arm-gnu os: ubuntu-latest - rust: 1.69.0 + rust: stable target: armv7-unknown-linux-gnueabihf - build: linux-aarch-musl os: ubuntu-latest @@ -34,7 +61,7 @@ jobs: target: aarch64-unknown-linux-musl - build: linux-aarch-gnu os: ubuntu-latest - rust: 1.69.0 + rust: stable target: aarch64-unknown-linux-gnu - build: windows os: ubuntu-latest @@ -96,11 +123,40 @@ jobs: - name: Rename ${{ matrix.target }} binary if: matrix.build != 'windows' run: mv "target/${{ matrix.target }}/release/uplink" "uplink-${{ matrix.target }}" - + - name: Rename ${{ matrix.target }} binary if: matrix.build == 'windows' run: mv "target/${{ matrix.target }}/release/uplink.exe" "uplink-${{ matrix.target }}.exe" - + + - name: Upload release archive + uses: softprops/action-gh-release@v1 + with: + files: uplink* + + build-release-android: + name: Build release for android + runs-on: ubuntu-latest + container: + image: bytebeamio/rust-android + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: perms + run: chown root:root . + + - name: Build for armv7 + run: cargo ndk --target armv7-linux-androideabi --platform 23 build --release --bin uplink + + - name: Build for aarch64 + run: cargo ndk --target aarch64-linux-android --platform 23 build --release --bin uplink + + - name: Rename uplink binaries + run: | + mv "target/aarch64-linux-android/release/uplink" "uplink-aarch64-linux-android" + mv "target/armv7-linux-androideabi/release/uplink" "uplink-armv7-linux-androideabi" + - name: Upload release archive uses: softprops/action-gh-release@v1 with: diff --git a/Cargo.lock b/Cargo.lock index 493acaab8..a70c94efa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "afl" version = "0.8.0" @@ -17,13 +32,28 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -35,19 +65,19 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] @@ -77,20 +107,20 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", - "bitflags", - "bytes 1.4.0", + "bitflags 1.3.2", + "bytes 1.6.0", "futures-util", "http", "http-body", @@ -100,14 +130,14 @@ dependencies = [ "memchr", "mime", "percent-encoding", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", "rustversion", "serde", "serde_json", "serde_path_to_error", "serde_urlencoded", "sync_wrapper", - "tokio 1.28.2", + "tokio 1.37.0", "tower", "tower-layer", "tower-service", @@ -120,7 +150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.4.0", + "bytes 1.6.0", "futures-util", "http", "http-body", @@ -130,6 +160,21 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backtrace" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.11.0" @@ -144,9 +189,21 @@ checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bincode" @@ -173,6 +230,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "block-buffer" version = "0.9.0" @@ -182,26 +245,35 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" -version = "1.6.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", ] [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -211,17 +283,19 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" dependencies = [ "jobserver", + "libc", + "once_cell", ] [[package]] @@ -236,6 +310,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.52.5", +] + [[package]] name = "clap" version = "2.34.0" @@ -244,7 +331,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim 0.8.0", "textwrap", "unicode-width", @@ -253,12 +340,12 @@ dependencies = [ [[package]] name = "clickhouse" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33816ee1fea4f60d97abfeb773b9b566ae85f8bfa891758d00a1fb1e5a606591" +checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" dependencies = [ "bstr", - "bytes 1.4.0", + "bytes 1.6.0", "clickhouse-derive", "clickhouse-rs-cityhash-sys", "futures", @@ -269,7 +356,7 @@ dependencies = [ "serde", "static_assertions", "thiserror", - "tokio 1.28.2", + "tokio 1.37.0", "url", ] @@ -296,9 +383,9 @@ dependencies = [ [[package]] name = "config" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" +checksum = "23738e11972c7643e4ec947840fc463b6a571afcd3e735bdfce7d03c7a784aca" dependencies = [ "async-trait", "lazy_static", @@ -329,11 +416,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -341,61 +434,43 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if 1.0.0", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" @@ -403,10 +478,10 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2102ea4f781910f8a5b98dd061f4c2023f479ce7bb1236330099ceb5a93cf17" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crossterm_winapi", "libc", - "mio 0.8.8", + "mio 0.8.11", "parking_lot", "signal-hook", "signal-hook-mio", @@ -422,11 +497,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" -version = "0.20.3" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ "darling_core", "darling_macro", @@ -434,29 +519,56 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.18", + "strsim 0.11.1", + "syn 2.0.66", ] [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", - "syn 2.0.18", + "syn 2.0.66", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "deunicode" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" + [[package]] name = "digest" version = "0.9.0" @@ -466,51 +578,62 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common", +] + [[package]] name = "dummy" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba338a15d93c01c2f117a2b0bd1bfa3c780fe771e7db7c69fc70bda265e2115" +checksum = "7e57e12b69e57fad516e01e2b3960f122696fdb13420e1a88ed8e210316f2876" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] name = "either" -version = "1.8.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] @@ -526,6 +649,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -539,13 +668,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -560,23 +688,20 @@ dependencies = [ [[package]] name = "fake" -version = "2.8.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9af7b0c58ac9d03169e27f080616ce9f64004edca3d2ef4147a811c21b23b319" +checksum = "1c25829bde82205da46e1823b2259db6273379f626fc211f126f65654a2669be" dependencies = [ + "deunicode", "dummy", "rand 0.8.5", - "unidecode", ] [[package]] name = "fastrand" -version = "1.9.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "filedescriptor" @@ -591,16 +716,22 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.21" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.16", - "windows-sys 0.48.0", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", ] +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flume" version = "0.10.14" @@ -610,7 +741,18 @@ dependencies = [ "futures-core", "futures-sink", "nanorand", - "pin-project 1.1.0", + "pin-project 1.1.5", + "spin 0.9.8", +] + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", "spin 0.9.8", ] @@ -637,13 +779,23 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -656,7 +808,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" dependencies = [ - "bitflags", + "bitflags 1.3.2", "fuchsia-zircon-sys", ] @@ -668,9 +820,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -683,9 +835,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -693,15 +845,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -710,38 +862,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-test" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84af27744870a4a325fa342ce65a940dfba08957b260b790ec278c1d81490349" +checksum = "ce388237b32ac42eca0df1ba55ed3bbda4eaf005d7d4b5dbc0b20ab962928ac9" dependencies = [ "futures-core", "futures-executor", @@ -750,15 +902,15 @@ dependencies = [ "futures-sink", "futures-task", "futures-util", - "pin-project 1.1.0", + "pin-project 1.1.5", "pin-utils", ] [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -767,7 +919,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", "pin-utils", "slab", ] @@ -804,9 +956,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -827,13 +979,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + [[package]] name = "git2" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "libgit2-sys", "log", @@ -848,20 +1006,20 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "fnv", "futures-core", "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 2.2.6", "slab", - "tokio 1.28.2", - "tokio-util 0.7.8", + "tokio 1.37.0", + "tokio-util 0.7.11", "tracing", ] @@ -871,6 +1029,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" version = "0.3.3" @@ -891,18 +1055,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -910,35 +1065,26 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "http" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "fnv", "itoa", ] [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "http", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", ] [[package]] @@ -949,9 +1095,15 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "human_bytes" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "91f255a4535024abf7640cb288260811fc14794f62b063652ed349f9a6c2348e" [[package]] name = "humantime" @@ -964,11 +1116,11 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "futures-channel", "futures-core", "futures-util", @@ -978,9 +1130,9 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.9", - "socket2 0.4.9", - "tokio 1.28.2", + "pin-project-lite 0.2.14", + "socket2", + "tokio 1.37.0", "tower-service", "tracing", "want", @@ -988,14 +1140,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ + "futures-util", "http", "hyper", - "rustls 0.21.1", - "tokio 1.28.2", + "rustls 0.21.12", + "tokio 1.37.0", "tokio-rustls 0.24.1", ] @@ -1005,13 +1158,36 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "hyper", "native-tls", - "tokio 1.28.2", + "tokio 1.37.0", "tokio-native-tls 0.3.1", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1020,9 +1196,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1035,36 +1211,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", -] - -[[package]] -name = "input_buffer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" -dependencies = [ - "bytes 0.5.6", + "hashbrown 0.12.3", + "serde", ] [[package]] -name = "instant" -version = "0.1.12" +name = "indexmap" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ - "cfg-if 1.0.0", + "equivalent", + "hashbrown 0.14.5", + "serde", ] [[package]] -name = "io-lifetimes" -version = "1.0.11" +name = "input_buffer" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", + "bytes 0.5.6", ] [[package]] @@ -1087,30 +1255,30 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1130,12 +1298,15 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" -version = "0.2.146" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libgit2-sys" @@ -1149,11 +1320,17 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" dependencies = [ "cc", "libc", @@ -1163,15 +1340,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1179,9 +1356,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lz4" @@ -1218,29 +1395,20 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memory_units" @@ -1260,6 +1428,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.6.23" @@ -1281,9 +1458,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", @@ -1335,31 +1512,21 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "mqttbytes" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9b7801639a5398353b9acf0764907d6571ca92e6b7a3662013b03861789449c" -dependencies = [ - "bytes 1.4.0", -] - [[package]] name = "nanorand" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.15", ] [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -1373,9 +1540,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.38" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d0df99cfcd2530b2e694f6e17e7f37b8e26bb23983ac530c0c97408837c631" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" dependencies = [ "cfg-if 0.1.10", "libc", @@ -1407,35 +1574,107 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi 0.3.9", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.9", "libc", ] +[[package]] +name = "object" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.54" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b3f656a17a6cbc115b5c7a40c616947d213ba182135b014d6051b73ab6f019" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags", + "bitflags 2.5.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1452,7 +1691,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] @@ -1463,18 +1702,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.26.0+1.1.1u" +version = "300.3.0+3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc62c9f12b22b8f5208c23a7200a442b2e5999f8bdf80233852122b5a4f6f37" +checksum = "eba8804a1c5765b18c4b3f907e6897ebabeedebc9830e1a0046c4a4cf44663e1" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.88" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -1483,11 +1722,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1495,15 +1740,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.5.1", "smallvec", - "windows-targets", + "windows-targets 0.52.5", ] [[package]] @@ -1512,11 +1757,20 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" @@ -1529,11 +1783,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.0" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ - "pin-project-internal 1.1.0", + "pin-project-internal 1.1.5", ] [[package]] @@ -1549,13 +1803,13 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] @@ -1566,9 +1820,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1576,47 +1830,68 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "pnet_base" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e" +checksum = "fe4cf6fb3ab38b68d01ab2aea03ed3d1132b4868fa4e06285f29f16da01c5f4c" dependencies = [ "no-std-net", ] [[package]] name = "pnet_macros" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a780e80005c2e463ec25a6e9f928630049a10b43945fea83207207d4a7606f4" +checksum = "688b17499eee04a0408aca0aa5cba5fc86401d7216de8a63fdf7a4c227871804" dependencies = [ "proc-macro2", "quote", "regex", - "syn 1.0.109", + "syn 2.0.66", ] [[package]] name = "pnet_macros_support" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d932134f32efd7834eb8b16d42418dac87086347d1bc7d142370ef078582bc" +checksum = "eea925b72f4bd37f8eab0f221bbe4c78b63498350c983ffa9dd4bcde7e030f56" dependencies = [ "pnet_base", ] [[package]] name = "pnet_packet" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bde678bbd85cb1c2d99dc9fc596e57f03aa725f84f3168b0eaf33eeccb41706" +checksum = "a9a005825396b7fe7a38a8e288dbc342d5034dac80c15212436424fef8ea90ba" dependencies = [ "glob", "pnet_base", @@ -1631,7 +1906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dfe828cdc03091f621a7c9a257769fbb9c66d75f7e45d0fbb563fa3bf669815" dependencies = [ "anyhow", - "bitflags", + "bitflags 1.3.2", "filedescriptor", "lazy_static", "libc", @@ -1643,6 +1918,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1695,9 +1976,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" dependencies = [ "unicode-ident", ] @@ -1710,9 +1991,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -1804,7 +2085,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.15", ] [[package]] @@ -1818,9 +2099,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -1828,14 +2109,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -1849,31 +2128,32 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags", + "bitflags 2.5.0", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -1885,6 +2165,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.3", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -1893,9 +2184,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "remote-pty-common" @@ -1931,12 +2222,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.2", - "bytes 1.4.0", + "base64 0.21.7", + "bytes 1.6.0", "encoding_rs", "futures-core", "futures-util", @@ -1951,22 +2242,24 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "pin-project-lite 0.2.9", - "rustls 0.21.1", - "rustls-pemfile", + "pin-project-lite 0.2.14", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "tokio 1.28.2", + "sync_wrapper", + "system-configuration", + "tokio 1.37.0", "tokio-rustls 0.24.1", - "tokio-util 0.7.8", + "tokio-util 0.7.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.4", "winreg", ] @@ -1980,28 +2273,73 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi 0.3.9", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "getrandom 0.2.15", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rumqttc" -version = "0.22.0" -source = "git+https://github.com/bytebeamio/rumqtt#714f71c9bc3369fa186353201dc2e8197d07594c" +version = "0.24.0" +source = "git+https://github.com/bytebeamio/rumqtt#e63bcab6cdfe22e4e8cca85011f34cc3bff21948" dependencies = [ - "bytes 1.4.0", - "flume", - "futures", + "bytes 1.6.0", + "fixedbitset", + "flume 0.11.0", + "futures-util", "log", "rustls-native-certs", - "rustls-pemfile", - "rustls-webpki", + "rustls-pemfile 2.1.2", + "rustls-webpki 0.102.4", "thiserror", - "tokio 1.28.2", - "tokio-rustls 0.24.1", + "tokio 1.37.0", + "tokio-rustls 0.25.0", + "tokio-stream", + "tokio-util 0.7.11", ] +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2017,21 +2355,20 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.17", + "semver 1.0.23", ] [[package]] name = "rustix" -version = "0.37.20" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags", - "errno 0.3.1", - "io-lifetimes", + "bitflags 2.5.0", + "errno 0.3.9", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2042,80 +2379,122 @@ checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" dependencies = [ "base64 0.11.0", "log", - "ring", + "ring 0.16.20", "sct 0.6.1", - "webpki 0.21.4", + "webpki", ] [[package]] name = "rustls" -version = "0.21.1" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring", - "rustls-webpki", - "sct 0.7.0", + "ring 0.17.8", + "rustls-webpki 0.101.7", + "sct 0.7.1", +] + +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-pki-types", + "rustls-webpki 0.102.4", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.2", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "base64 0.21.2", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", ] [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.42.0", + "windows-sys 0.52.0", ] [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -2123,18 +2502,18 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -2157,11 +2536,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -2170,9 +2549,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -2189,9 +2568,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "semver-parser" @@ -2201,22 +2580,22 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] @@ -2232,9 +2611,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -2243,10 +2622,11 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.11" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ + "itoa", "serde", ] @@ -2262,6 +2642,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "serial" version = "0.4.1" @@ -2306,18 +2716,29 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -2340,9 +2761,9 @@ checksum = "39acde55a154c4cd3ae048ac78cc21c25f3a0145e44111b523279113dce0d94a" [[package]] name = "signal-hook" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" dependencies = [ "libc", "signal-hook-registry", @@ -2355,15 +2776,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio 0.8.8", + "mio 0.8.11", "signal-hook", ] [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -2377,42 +2798,42 @@ dependencies = [ "futures-core", "libc", "signal-hook", - "tokio 1.28.2", + "tokio 1.37.0", ] [[package]] -name = "slab" -version = "0.4.8" +name = "signature" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "autocfg", + "digest 0.10.7", + "rand_core 0.6.4", ] [[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "socket2" +name = "slab" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "libc", - "winapi 0.3.9", + "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2430,6 +2851,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2440,11 +2871,11 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" name = "storage" version = "0.1.0" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "log", - "mqttbytes", "pretty_env_logger", - "rand 0.7.3", + "rand 0.8.5", + "rumqttc", "seahash", "tempdir", "thiserror", @@ -2458,9 +2889,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "structopt" @@ -2486,19 +2917,25 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "surge-ping" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af341b2be485d647b5dc4cfb2da99efac35b5c95748a08fb7233480fedc5ead3" +checksum = "efbf95ce4c7c5b311d2ce3f088af2b93edef0f09727fa50fbe03c7a979afce77" dependencies = [ "hex", "parking_lot", "pnet_packet", "rand 0.8.5", - "socket2 0.5.3", + "socket2", "thiserror", - "tokio 1.28.2", + "tokio 1.37.0", "tracing", ] @@ -2515,9 +2952,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -2559,11 +2996,32 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" -version = "0.4.38" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" dependencies = [ "filetime", "libc", @@ -2582,23 +3040,21 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ - "autocfg", "cfg-if 1.0.0", "fastrand", - "redox_syscall 0.3.5", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -2623,29 +3079,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if 1.0.0", "once_cell", @@ -2653,11 +3109,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -2665,16 +3124,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -2718,20 +3178,19 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.2" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ - "autocfg", - "bytes 1.4.0", + "backtrace", + "bytes 1.6.0", "libc", - "mio 0.8.8", + "mio 0.8.11", "num_cpus", - "parking_lot", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", "signal-hook-registry", - "socket2 0.4.9", - "tokio-macros 2.1.0", + "socket2", + "tokio-macros 2.2.0", "windows-sys 0.48.0", ] @@ -2743,9 +3202,9 @@ checksum = "e7d4237822b7be8fff0a7a27927462fad435dcb6650f95cea9e946bf6bdc7e07" dependencies = [ "bytes 0.5.6", "once_cell", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", "tokio 0.2.25", - "tokio 1.28.2", + "tokio 1.37.0", "tokio-stream", ] @@ -2762,13 +3221,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] @@ -2788,7 +3247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.2", + "tokio 1.37.0", ] [[package]] @@ -2800,7 +3259,7 @@ dependencies = [ "futures-core", "rustls 0.17.0", "tokio 0.2.25", - "webpki 0.21.4", + "webpki", ] [[package]] @@ -2809,19 +3268,30 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.1", - "tokio 1.28.2", + "rustls 0.21.12", + "tokio 1.37.0", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio 1.37.0", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", - "pin-project-lite 0.2.9", - "tokio 1.28.2", + "pin-project-lite 0.2.14", + "tokio 1.37.0", ] [[package]] @@ -2841,17 +3311,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ - "bytes 1.4.0", + "bytes 1.6.0", "futures-core", "futures-sink", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", "slab", - "tokio 1.28.2", - "tracing", + "tokio 1.37.0", ] [[package]] @@ -2871,9 +3340,9 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "pin-project 1.1.0", - "pin-project-lite 0.2.9", - "tokio 1.28.2", + "pin-project 1.1.5", + "pin-project-lite 0.2.14", + "tokio 1.37.0", "tower-layer", "tower-service", "tracing", @@ -2893,33 +3362,32 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.9", + "pin-project-lite 0.2.14", "tracing-attributes", "tracing-core", ] [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -2927,23 +3395,23 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.14" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a713421342a5a666b7577783721d3117f1b69a393df803ee17bb73b1e122a59" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ - "ansi_term", "matchers", + "nu-ansi-term", "once_cell", "regex", "sharded-slab", @@ -2956,9 +3424,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" @@ -2994,7 +3462,7 @@ dependencies = [ "crossterm", "env_logger", "futures", - "getrandom 0.2.10", + "getrandom 0.2.15", "js-sys", "libc", "log", @@ -3004,7 +3472,7 @@ dependencies = [ "rand 0.7.3", "remote-pty-common", "remote-pty-master", - "ring", + "ring 0.16.20", "serde", "serde_json", "thiserror", @@ -3017,7 +3485,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki 0.21.4", + "webpki", "webpki-roots 0.19.0", "wee_alloc", ] @@ -3049,9 +3517,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uds_windows" @@ -3065,62 +3533,65 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] -name = "unidecode" -version = "0.3.0" +name = "untrusted" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402bb19d8e03f1d1a7450e2bd613980869438e0666331be3e073089124aa1adc" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "uplink" -version = "2.5.0" +version = "2.12.2" dependencies = [ "anyhow", "async-trait", "axum", - "bytes 1.4.0", + "bytes 1.6.0", "clickhouse", "config", "fake", - "flume", + "flume 0.10.14", + "fs2", "futures-util", + "hex", + "human_bytes", "lazy_static", "log", "lz4_flex", @@ -3128,9 +3599,11 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest", + "rsa", "rumqttc", "serde", "serde_json", + "serde_with", "signal-hook", "signal-hook-tokio", "storage", @@ -3141,10 +3614,10 @@ dependencies = [ "tempdir", "thiserror", "time", - "tokio 1.28.2", + "tokio 1.37.0", "tokio-compat-02", "tokio-stream", - "tokio-util 0.7.8", + "tokio-util 0.7.11", "tracing", "tracing-subscriber", "tunshell-client", @@ -3153,9 +3626,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -3168,6 +3641,20 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utils" +version = "0.1.0" +dependencies = [ + "flume 0.10.14", + "futures-util", + "serde", + "serde_json", + "tokio 1.37.0", + "tokio-stream", + "tokio-util 0.7.11", + "uplink", +] + [[package]] name = "valuable" version = "0.1.0" @@ -3218,11 +3705,10 @@ checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -3240,9 +3726,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3250,24 +3736,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3277,9 +3763,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3287,28 +3773,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -3319,9 +3805,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -3333,18 +3819,8 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -3353,17 +3829,14 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739" dependencies = [ - "webpki 0.21.4", + "webpki", ] [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki 0.22.0", -] +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "wee_alloc" @@ -3407,11 +3880,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi 0.3.9", + "windows-sys 0.52.0", ] [[package]] @@ -3421,18 +3894,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.42.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets 0.52.5", ] [[package]] @@ -3441,115 +3908,147 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi 0.3.9", + "cfg-if 1.0.0", + "windows-sys 0.48.0", ] [[package]] @@ -3564,18 +4063,23 @@ dependencies = [ [[package]] name = "xattr" -version = "0.2.3" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", + "linux-raw-sys", + "rustix", ] [[package]] name = "xdg" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688597db5a750e9cad4511cb94729a078e274308099a0382b5b8203bbc767fee" -dependencies = [ - "home", -] +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml index c0b20764b..cc644fba4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,17 +2,34 @@ serial = { git = "https://github.com/bytebeamio/serial-rs", branch = "android_fix" } [workspace] +resolver = "2" members = [ "uplink", - "storage" + "storage", + "tools/utils", ] exclude = [ "tools/deserialize-backup", "tools/simulator", + "tools/system-stats", "tools/tunshell", - "tools/utils", ] +[workspace.dependencies] +bytes = "1" +flume = "0.10" +futures-util = "0.3" +log = "0.4" +rand = "0.8" +rumqttc = { git = "https://github.com/bytebeamio/rumqtt" } +serde = { version = "1", features = ["derive"] } +serde_json = "1.0" +tempdir = "0.3" +thiserror = "1" +tokio = { version = "1", features = ["rt-multi-thread", "process"] } +tokio-stream = "0.1.15" +tokio-util = { version = "0.7", features = ["codec", "time"] } + [profile.dev] opt-level = 1 debug = true diff --git a/Dockerfile b/Dockerfile index 3867a378f..d00ad8a42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,53 +1,31 @@ -FROM ubuntu:18.04 AS base -SHELL ["/bin/bash", "-c"] - -RUN echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries - -RUN apt-get upgrade -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y curl runit vim libssl-dev pkg-config - -RUN mkdir -p /etc/bytebeam /usr/share/bytebeam - -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 - -CMD ["/usr/bin/runsvdir", "/etc/runit"] -COPY runit/ /etc/runit -RUN rm -rf /etc/runit/runsvdir -RUN chmod +x /etc/runit/uplink/run +FROM rust:alpine as builder +RUN apk add build-base openssl-dev WORKDIR "/usr/share/bytebeam/uplink" -##################################################################################### - -FROM base as staging - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup -RUN chmod +x /tmp/rustup -RUN /tmp/rustup -y -RUN source $HOME/.cargo/env - -COPY . /usr/share/bytebeam/uplink +COPY uplink/ uplink +COPY storage/ storage +COPY tools/utils/ tools/utils +COPY Cargo.* . +COPY .git/ .git RUN mkdir -p /usr/share/bytebeam/uplink/bin -WORKDIR /usr/share/bytebeam/uplink/tools/simulator -RUN $HOME/.cargo/bin/cargo build --release -RUN cp target/release/simulator /usr/share/bytebeam/uplink/bin/ -WORKDIR /usr/share/bytebeam/uplink -RUN $HOME/.cargo/bin/cargo build --release +RUN cargo build --release RUN cp target/release/uplink /usr/share/bytebeam/uplink/bin/ ################################################################################################### -FROM base AS production +FROM alpine:latest + +RUN apk add runit bash curl coreutils +WORKDIR "/usr/share/bytebeam/uplink" RUN mkdir -p /usr/share/bytebeam/uplink -RUN mkdir -p /usr/share/bytebeam/uplink/shared -#RUN mkdir -P /usr/share/bytebeam/uplink/bin -COPY --from=staging /usr/share/bytebeam/uplink/bin /usr/bin -COPY --from=staging /usr/share/bytebeam/uplink/paths /usr/share/bytebeam/uplink/paths -COPY --from=staging /usr/share/bytebeam/uplink/simulator.sh /usr/share/bytebeam/uplink -COPY --from=staging /usr/share/bytebeam/uplink/bin /usr/share/bytebeam/uplink -#CMD uplink -a /usr/share/bytebeam/uplink/shared/device.json -#CMD cp /usr/share/bytebeam/uplink/uplink /usr/share/bytebeam/uplink/shared/uplink +COPY --from=builder /usr/share/bytebeam/uplink/bin /usr/bin +COPY runit/ /etc/runit + +CMD ["/usr/bin/runsvdir", "/etc/runit"] + +COPY paths/ paths +COPY simulator.sh . + diff --git a/README.md b/README.md index aef65303f..61e5441da 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ Uplink is a rust based utility for efficiently sending data and receiving comman - Provides remote shell access through [Tunshell][tunshell] - Supports TLS with easy cross-compilation. - ### Quickstart #### Install Uplink @@ -46,7 +45,6 @@ Actions are messages that uplink expects to receive from the broker and is execu ```js { "action_id": "...", - "kind": "...", "name": "...", "payload": "..." } @@ -122,7 +120,6 @@ Once enabled, Actions with the following JSON will trigger uplink to download th ```js { "action_id": "...", - "kind": "process", "name": "update_firmware", "payload": "{ \"url\": \"https://example.com/file\", @@ -134,7 +131,6 @@ Once downloded, the payload JSON is updated with the file's on device path, as s ```js { "action_id": "...", - "kind": "process", "name": "update_firmware", "payload": "{ \"url\": \"https://example.com/file\", @@ -149,7 +145,6 @@ With the help of tunshell, uplink allows you to remotely connect to a device she ```js { "action_id": "...", - "kind": "...", "name": "tunshell", "payload": "{ \"session\": \"...\", diff --git a/configs/config.toml b/configs/config.toml index 2064c980a..b068a5eec 100644 --- a/configs/config.toml +++ b/configs/config.toml @@ -66,7 +66,7 @@ blacklist = ["cancollector_metrics", "candump_metrics", "pinger"] # be collected, batched and forwarded to serializer to then be published onto platform. # # Required Parameters -# - buf-size: Number of data points that shall be included in each Publish +# - batch-size: Number of data points that shall be included in each Publish # - topic(optional): topic-filter to which data shall be published. If left # unconfigured, stream will be created dynamically. # - flush-period(optional): Duration in seconds after a data point enters the stream @@ -75,19 +75,23 @@ blacklist = ["cancollector_metrics", "candump_metrics", "pinger"] # should perform on the data transiting through the stream. Currently supported # compression schemes are Lz4 and Disabled. Defaults to Disabled. # - persistence(optional): helps persist relevant information for data recovery purposes, -# used when there is a network/system failure. +# used when there is a network/system failure. +# - priority(optional, u8): Higher prioirity streams get to push their data +# onto the network first. # -# In the following config for the device_shadow stream we set buf_size to 1 and mark +# In the following config for the device_shadow stream we set batch_size to 1 and mark # it as non-persistent. streams are internally constructed as a map of Name -> Config [streams.device_shadow] topic = "/tenants/{tenant_id}/devices/{device_id}/events/device_shadow/jsonarray" flush_period = 5 +priority = 75 # Example using compression [streams.imu] topic = "/tenants/{tenant_id}/devices/{device_id}/events/imu/jsonarray/lz4" -buf_size = 100 +batch_size = 100 compression = "Lz4" +priority = 50 # Configuration details associated with uplink's persistent storage module which writes publish # packets to disk in case of slow or crashed network, for recovery purposes. @@ -103,7 +107,7 @@ compression = "Lz4" # configuration, we use transient(in-memory) storage to handle network downtime only. [streams.gps] topic = "/tenants/{tenant_id}/devices/{device_id}/events/gps/jsonarray" -buf_size = 10 +batch_size = 10 persistence = { max_file_size = 1048576, max_file_count = 10 } # NOTE: While it is possible to configure persistence to be disabled, including only max_file_count @@ -111,7 +115,7 @@ persistence = { max_file_size = 1048576, max_file_count = 10 } # config values will be default. Configuring only max_file_size is also allowed, without persistance. [streams.motor] topic = "/tenants/{tenant_id}/devices/{device_id}/events/motor/jsonarray/lz4" -buf_size = 50 +batch_size = 50 compression = "Lz4" persistence = { max_file_count = 3 } @@ -121,9 +125,12 @@ persistence = { max_file_count = 3 } # # NOTE: Action statuses are expected on a specifc topic as configured in example below. # This also means that we require a topic to be configured or uplink will error out. +# Given the importance of conveying action status at the earliest to platform, +# it has highest priority by default of 255. [action_status] topic = "/tenants/{tenant_id}/devices/{device_id}/action/status" flush_period = 2 +priority = 255 # Configurations for uplink's built-in file downloader, including the actions that can trigger # a download, the location in file system where uplink will download and store files from the diff --git a/docs/apps.md b/docs/apps.md index 779f84d70..33b31f6c7 100644 --- a/docs/apps.md +++ b/docs/apps.md @@ -18,7 +18,6 @@ An `Action` is the term used to refer to messages that carry commands and other ```js { "action_id": "...", // An integer value that can be used to maintain indempotence - "kind": "...", // May hold values such as process, depending on end-use "name": "...", // Name given to Action "payload": "..." // Can contain JSON formatted data as a string } diff --git a/docs/downloader.md b/docs/downloader.md new file mode 100644 index 000000000..f9fe0fa35 --- /dev/null +++ b/docs/downloader.md @@ -0,0 +1,25 @@ +# File Downloader +Uplink consists of a file downloader, a built-in application that can perform the initial stage of any OTA, downloading the update onto the device. + +## Configuring the Downloader +As described in the [example `config.toml`](https://github.com/bytebeamio/uplink/blob/513f4c2def843f3c27bf0a7dc993667010944535/configs/config.toml#L135C1-L145C27), uplink will handle a special class of actions as [Download Actions](#download-actions) and download the associated files into the mentioned `path` within a directory with the same name as the download action(i.e. `/var/tmp/ota-file/update_firmware`). +Here is one more example config to illustrate how uplink can be configured: +``` +[downloader] +actions = [{ name = "update_firmware", timeout = 3600 }] // sets the timeout to 1hr, allowing for uplink to wait for a whole hour before timing out the download, useful for large downloads over slower networks +path = "/data/local/downloads" // downloads the file into a location commonly seen in Android systems, i.e. /data/local/downloads/update_firmware +``` + +## Download Actions +Download actions contain a payload in JSON text format: +``` +{ + "url": "https://firmware...", // URL to download file from + "content-length": 123456, // Size of file in bytes + "file-name": "example.txt", // Name of the file post download + "checksum": "abc123...", // Checksum that can be used to verify the download was successful + ... +} +``` + +P.S. [Bytebeam OTA updates](https://bytebeam.io/docs/triggering-ota-update) are an example of Download Action diff --git a/examples/android/README.md b/examples/android/README.md new file mode 100644 index 000000000..408d39f61 --- /dev/null +++ b/examples/android/README.md @@ -0,0 +1,3 @@ +## Uplink example setup for android + +The `payload` directory has an example uplink setup that can be installed in an android device. \ No newline at end of file diff --git a/examples/android/VERSION.txt b/examples/android/VERSION.txt new file mode 100644 index 000000000..bcfacb597 --- /dev/null +++ b/examples/android/VERSION.txt @@ -0,0 +1 @@ +r2-2023-10-13-NOWD-AOSP \ No newline at end of file diff --git a/examples/android/bin/logrotate b/examples/android/bin/logrotate new file mode 100755 index 000000000..d4a93921c Binary files /dev/null and b/examples/android/bin/logrotate differ diff --git a/examples/android/bin/uplink b/examples/android/bin/uplink new file mode 100755 index 000000000..edb085aa4 Binary files /dev/null and b/examples/android/bin/uplink differ diff --git a/examples/android/env.sh b/examples/android/env.sh new file mode 100644 index 000000000..1b9700b5c --- /dev/null +++ b/examples/android/env.sh @@ -0,0 +1,2 @@ +export DATA_DIR=/data/local/tmp/uplink +export UPLINK_LOG_LEVEL=-v \ No newline at end of file diff --git a/examples/android/etc/uplink.config.toml b/examples/android/etc/uplink.config.toml new file mode 100644 index 000000000..c57e8557a --- /dev/null +++ b/examples/android/etc/uplink.config.toml @@ -0,0 +1,39 @@ +processes = [] +action_redirections = { "update_firmware" = "install_update", "send_script" = "run_script" } +script_runner = [{ name = "run_script" }] +persistence_path = "/data/local/tmp/uplink/persistence" + +[tcpapps.app] +port = 8031 +#actions = [{ name = "install_update" }] + +[tcpapps.app2] +port = 8032 + +[system_stats] +enabled = true +process_names = ["/data/local/uplinkmodule/bin/uplink", "com.foobnix.pro.pdf.reader"] +update_period = 2 +stream_size = 1 + +[ota_installer] +path="/data/local/tmp/uplink/installer" +actions=[{name="install_update", timeout=310}] +uplink_port=8032 + +[downloader] +actions = [{ name = "update_firmware" }, { name = "send_script" }] +path = "/data/local/tmp/uplink/downloader" + +[streams.device_shadow] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/device_shadow/jsonarray" +batch_size = 128 +flush_period = 30 + +[logging] +tags = ["*"] +min_level = 4 + +[console] +enabled = true +port = 9328 diff --git a/examples/android/service.sh b/examples/android/service.sh new file mode 100755 index 000000000..3a5a20d68 --- /dev/null +++ b/examples/android/service.sh @@ -0,0 +1,11 @@ +#!/system/bin/sh + +set -x + +export MODULE_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +. $MODULE_DIR/env.sh +mkdir -p $DATA_DIR + +cd $DATA_DIR || exit + +$MODULE_DIR/bin/uplink -a $DATA_DIR/device.json -c $MODULE_DIR/etc/uplink.config.toml $UPLINK_LOG_LEVEL 2>&1 | $MODULE_DIR/bin/logrotate --max-size 10000000 --backup-files-count 30 --output $DATA_DIR/out.log \ No newline at end of file diff --git a/examples/demo.cpp b/examples/demo.cpp new file mode 100644 index 000000000..9d69b85fa --- /dev/null +++ b/examples/demo.cpp @@ -0,0 +1,130 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +optional readUntilNewline(int socketFD) { + std::vector buffer; + char c; + while (true) { + int res = read(socketFD, &c, 1); + if (res == 0) { + // socket is in blocking mode, read return value 0 means socket has closed + return {}; + } + if (c == '\n') { + break; + } + buffer.push_back(c); + } + return string(buffer.begin(), buffer.end()); +} + +ssize_t writeToSocket(int socketFD, const string &data) { + return send(socketFD, data.c_str(), data.length(), MSG_NOSIGNAL); +} + +long current_time_millis() { + struct timeval tv; + gettimeofday(&tv, NULL); + long long millis = + ((long long)tv.tv_sec) * 1000 + ((long long)tv.tv_usec) / 1000; + return millis; +} + +int connectToUplink(int port) { + struct addrinfo hints, *res; + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + if (getaddrinfo("127.0.0.1", std::to_string(port).c_str(), &hints, &res) != 0) { + cout << "getaddrinfo failed for port " << port << endl; + return 0; + } + + int socketFD = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + + fcntl(socketFD, F_SETFL, O_NONBLOCK); + + fd_set fdset; + struct timeval tv; + FD_ZERO(&fdset); + FD_SET(socketFD, &fdset); + tv.tv_sec = 0; + tv.tv_usec = 250000; // 250 ms connection timeout + + ::connect(socketFD, res->ai_addr, res->ai_addrlen); + + if (select(socketFD + 1, nullptr, &fdset, nullptr, &tv) == 1) { + int so_error; + socklen_t len = sizeof so_error; + + getsockopt(socketFD, SOL_SOCKET, SO_ERROR, &so_error, &len); + + if (so_error == 0) { + freeaddrinfo(res); + int flags = fcntl(socketFD, F_GETFL, 0); + flags = flags & ~O_NONBLOCK; + fcntl(socketFD, F_SETFL, flags); + return socketFD; + } else { + cout << "getsockopt returned error " << so_error << endl; + return 0; + } + } + + cout << "select failed for port " << port << endl; + freeaddrinfo(res); + return 0; +} + +int main() { + int socketFD = connectToUplink(8031); + std::mutex clientLock; + if (socketFD == 0) { + cout << "couldn't connect" << endl; + return 1; + } + + std::thread actionResponder([&] () { + while (true) { + optional lineO = readUntilNewline(socketFD); + if (!lineO.has_value()) { + break; + } + string line = lineO.value(); + clientLock.lock(); + cout << line << endl; + // push action_status for this action + clientLock.unlock(); + } + cout << "stopping responder thread" << endl; + }); + + int sequence = 1; + while (true) { + sleep(1); + ostringstream oss; + oss << "{\"stream\": \"device_shadow\", \"sequence\": " << sequence++ << ", \"timestamp\": " << current_time_millis() << ", \"a\": 1, \"b\": 2 }" << endl; + clientLock.lock(); + if (writeToSocket(socketFD, oss.str()) == -1) { + break; + } + clientLock.unlock(); + } + actionResponder.join(); + cout << "stopping pusher thread" << endl; + return 0; +} \ No newline at end of file diff --git a/examples/demo.go b/examples/demo.go index c07a951b4..c3399ac17 100644 --- a/examples/demo.go +++ b/examples/demo.go @@ -19,7 +19,6 @@ type ActionStatus struct { type Action struct { Id string `json:"action_id"` - Kind string `json:"timestamp"` Name string `json:"name"` Payload string `json:"payload"` } diff --git a/qa-scripts/.env b/qa-scripts/.env new file mode 100644 index 000000000..430822ddc --- /dev/null +++ b/qa-scripts/.env @@ -0,0 +1,5 @@ +CONSOLED_DOMAIN= +BYTEBEAM_TENANT_ID= +BYTEBEAM_API_KEY= +DEVICE_ID= + diff --git a/qa-scripts/README.md b/qa-scripts/README.md new file mode 100644 index 000000000..2eb6a4e7d --- /dev/null +++ b/qa-scripts/README.md @@ -0,0 +1,32 @@ +# QA + +The scripts to automate setting up QA testing environ are as such: +- `setup.sh`: Create two container proxy system for simulating toxic internet with uplink on one side. +- `basic.sh`: Run uplink with built-in config to verify basic setup and operation. +- `streams.sh`: Runs uplink with certains streams configured to push data onto network faster than normal, observe logs to verify required behavior. +- `persistence.sh`: Runs uplink and provides guidance on how to trigger network toxics to trigger uplink into persistance mode, observe logs to verify required behavior. +- `actions.sh`: Runs uplink configured to accept actions, with guidance on how to trigger the action and observe logs to verify required behavior. +- `downloader.sh`: Runs uplink configured to accept download actions that can then be slowed down with toxics as guided to observe and verify required behavior. +- `cleanup.sh`: Pull down what was created in `setup.sh`. + +## Miscellaneous +### Tunshell +Start uplink in the simulator container: +```sh +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -vvv -m uplink::collector::tunshell -m uplink::base::bridge +``` +Trigger multiple tunshell sessions from platform to ensure that connections are established and even with another session or action(like "send_file") in progress a new session is allowed to connect. + +### Device Shadow +Start uplink in the simulator container: +```sh +printf "$(cat << EOF +[device_shadow] +interval = 10 +EOF +)" > devices/shadow.toml +docker cp devices/shadow.toml simulator:/usr/share/bytebeam/uplink/devices/shadow.toml + +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/shadow.toml -vvv -m uplink::collector::device_shadow +``` +Ensure the logs are timed 10s between diff --git a/qa-scripts/actions.sh b/qa-scripts/actions.sh new file mode 100755 index 000000000..dd7e0af25 --- /dev/null +++ b/qa-scripts/actions.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -xe + +source qa-scripts/.env + +printf "$(cat << EOF +persistence_path = "/var/tmp/persistence" +action_redirections = { "send_file" = "load_file", "send_script" = "run_script" } +script_runner = [{ name = "run_script" }] + +[downloader] +actions = [{ name = "send_file" },{ name = "send_script"}] +path = "/var/tmp/downloads" + +[simulator] +actions = ["load_file"] +gps_paths = "./paths/" + +[tcpapps.blackhole] +port = 7891 +actions = [{ name = "no_response", timeout = 100 }, { name = "restart_response", timeout = 1800 }] +EOF +)" > devices/actions.toml +docker cp devices/actions.toml simulator:/usr/share/bytebeam/uplink/devices/actions.toml + +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/actions.toml -vv -m uplink::base::bridge + +# Trigger send_file action by first uploading the file(similar for send_file): +# FILE_UUID=$(curl --location "https://$CONSOLED_DOMAIN/api/v1/file" --header "x-bytebeam-tenant: demo" --header "x-bytebeam-api-key: $BYTEBEAM_KEY" --form 'file=@"/path/to/file.txt"' --form 'fileName="file.txt"' | jq .id | tr -d '"') +# Push send_file action to the designated device: +# FILE_SIZE=$(wc -c /path/to/file.txt | cut -d ' ' -f1) curl --location "https://$CONSOLED_DOMAIN/api/v1/actions" --header "x-bytebeam-tenant: demo" --header "x-bytebeam-api-key: $BYTEBEAM_KEY" --header 'Content-Type: application/json' --data "{ \"device_ids\": [\"$DEVICE_ID\"], \"search_type\": \"default\", \"search_key\": \"\", \"search_query\": \"string\", \"action\": \"send_file\", \"params\": { \"id\": \"$FILE_UUID\", \"url\":\"https://firmware.$CONSOLED_DOMAIN/api/v1/file/$FILE_UUID/artifact\", \"file_name\":\"file.txt\", \"content-length\":$FILE_SIZE }}" +# Trigger another action when first action is in execution on the device to see actions being rejected +# Stop uplink when an action is still in execution and check persistence directory for current_action file +# docker exec -it simulator tree /var/tmp/persistence +# Restart uplink and connect with example python to send respose to persisted action(restart_response action) +# Do the same with another action(no_response action) see if on timeout a failure response is created when the action handler doesn't send any response \ No newline at end of file diff --git a/qa-scripts/basics.sh b/qa-scripts/basics.sh new file mode 100755 index 000000000..698d453c6 --- /dev/null +++ b/qa-scripts/basics.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -xe + +source qa-scripts/.env + +printf "$(cat << EOF +[console] +enabled = true +port = 3333 +EOF +)" > devices/basics.toml +docker cp devices/basics.toml simulator:/usr/share/bytebeam/uplink/devices/basics.toml + +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/basics.toml -vv + +# from separate terminal run the following to trigger minimum log level change(show debug logs of rumqttc) +# docker exec -it simulator curl -X POST -H "Content-Type: text/plain" -d "uplink=info,rumqtt=debug" http://localhost:3333/logs diff --git a/qa-scripts/cleanup.sh b/qa-scripts/cleanup.sh new file mode 100755 index 000000000..28b01d6b7 --- /dev/null +++ b/qa-scripts/cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -xe + +docker stop simulator noxious +docker network rm qa-jail diff --git a/qa-scripts/downloader.sh b/qa-scripts/downloader.sh new file mode 100755 index 000000000..c0f6375a1 --- /dev/null +++ b/qa-scripts/downloader.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -xe + +source qa-scripts/.env + +# printf "$(cat << EOF +# [downloader] +# actions = [{ name = "send_file", timeout = 1800 }] +# path = "/var/tmp/downloads" +# EOF +# )" > devices/downloader.toml +# docker cp devices/downloader.toml simulator:/usr/share/bytebeam/uplink/devices/downloader.toml + +# docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/downloader.toml -vv -m uplink::base::bridge -m uplink::collector::downloader + +# Trigger send_file action by first uploading file(choose a large file ~1GB): +# FILE_UUID=$(curl --location "https://$CONSOLED_DOMAIN/api/v1/file" --header "x-bytebeam-tenant: demo" --header "x-bytebeam-api-key: $BYTEBEAM_KEY" --form 'file=@"/path/to/file.txt"' --form 'fileName="file.txt"' | jq .id | tr -d '"') +# Push send_file to the designated device: +# FILE_SIZE=$(wc -c /path/to/file.txt | cut -d ' ' -f1) curl --location "https://$CONSOLED_DOMAIN/api/v1/actions" --header "x-bytebeam-tenant: demo" --header "x-bytebeam-api-key: $BYTEBEAM_KEY" --header 'Content-Type: application/json' --data "{ \"device_ids\": [\"$DEVICE_ID\"], \"search_type\": \"default\", \"search_key\": \"\", \"search_query\": \"string\", \"action\": \"send_file\", \"params\": { \"id\": \"$FILE_UUID\", \"url\":\"https://firmware.$CONSOLED_DOMAIN/api/v1/file/$FILE_UUID/artifact\", \"file_name\":\"file.txt\", \"content-length\":$FILE_SIZE }}" +# Slow down/stop the https network for downloader +# toxiproxy-cli toxic add -n slow -t latency -a latency=100 --downstream https +# toxiproxy-cli delete https +# Re-establish the https network +# toxiproxy-cli new https --listen 0.0.0.0:443 --upstream firmware.stage.bytebeam.io:443 +# See logs of the download erroring out in between, restarting from where it had failed + +# Restart uplink with partition attached that is smaller than the file to be downloaded +docker stop simulator +NOXIOUS_IP=$(docker inspect noxious | jq '.[].NetworkSettings.Networks."qa-jail".IPAddress' | tr -d '"') +docker run --name simulator \ + --rm -d \ + --network=qa-jail \ + --add-host "$CONSOLED_DOMAIN:$NOXIOUS_IP" \ + --add-host "firmware.$CONSOLED_DOMAIN:$NOXIOUS_IP" \ + -v /mnt/downloads:/var/tmp/downloads \ + -e CONSOLED_DOMAIN=$CONSOLED_DOMAIN \ + -e BYTEBEAM_API_KEY=$BYTEBEAM_API_KEY \ + -it bytebeamio/simulator + +docker exec -it simulator sv stop /etc/runit/uplink +docker exec -it simulator /usr/share/bytebeam/uplink/simulator.sh download_auth_config $DEVICE_ID + +printf "$(cat << EOF +[downloader] +actions = [{ name = "update_firmware" }, { name = "send_file" }, { name = "send_script" }] +EOF +)" > devices/downloader.toml +docker cp devices/downloader.toml simulator:/usr/share/bytebeam/uplink/devices/downloader.toml + +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/downloader.toml -vv -m uplink::base::bridge -m uplink::collector::downloader +# Trigger a large enough download to trigger the disk check to fail \ No newline at end of file diff --git a/qa-scripts/persistence.sh b/qa-scripts/persistence.sh new file mode 100755 index 000000000..55b932faa --- /dev/null +++ b/qa-scripts/persistence.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -xe + +source qa-scripts/.env + +printf "$(cat << EOF +persistence_path = "/var/tmp/persistence" + +[simulator] +actions = [] +gps_paths = "./paths/" + +[streams.motor] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/gps/jsonarray" + +[streams.bms] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/bms/jsonarray" +persistence = { max_file_size = 0 } + +[streams.imu] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/imu/jsonarray" +persistence = { max_file_count = 3, max_file_size = 1024 } +EOF +)" > devices/persistence.toml +docker cp devices/persistence.toml simulator:/usr/share/bytebeam/uplink/devices/persistence.toml + +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/persistence.toml -vv -m uplink::base::serializer -m storage + +# Slow down mqtts +# toxiproxy-cli toxic add -n slow -t latency -a latency=100 --downstream mqtts +# Look at logs for persistence into disk in slow mode, catchup mode +# Disrupt mqtts, check logs for slow mode data loss on in-memory buffer overflow, etc. +# toxiproxy-cli delete mqtts +# Verify persistence of packets onto disk +# docker exec -it simulator tree /var/tmp/persistence +# Bring back network, check logs for back to normal mode, check platform for appropriate data retention/loss with timestamp gaps +# toxiproxy-cli new mqtts --listen 0.0.0.0:8883 --upstream $CONSOLED_DOMAIN:8883 \ No newline at end of file diff --git a/qa-scripts/setup.sh b/qa-scripts/setup.sh new file mode 100755 index 000000000..f3e1cc3f8 --- /dev/null +++ b/qa-scripts/setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -xe + +source qa-scripts/.env + +docker build -t bytebeamio/simulator . +docker network create qa-jail +docker run --name noxious \ + --rm -d \ + -p 8474:8474 \ + --network=qa-jail \ + oguzbilgener/noxious +NOXIOUS_IP=$(docker inspect noxious | jq '.[].NetworkSettings.Networks."qa-jail".IPAddress' | tr -d '"') + +toxiproxy-cli new mqtts --listen 0.0.0.0:8883 --upstream $CONSOLED_DOMAIN:8883 +toxiproxy-cli new https --listen 0.0.0.0:443 --upstream firmware.$CONSOLED_DOMAIN:443 + +docker run --name simulator \ + --rm -d \ + --network=qa-jail \ + --add-host "$CONSOLED_DOMAIN:$NOXIOUS_IP" \ + --add-host "firmware.$CONSOLED_DOMAIN:$NOXIOUS_IP" \ + -e CONSOLED_DOMAIN=$CONSOLED_DOMAIN \ + -e BYTEBEAM_TENANT_ID=$BYTEBEAM_TENANT_ID \ + -e BYTEBEAM_API_KEY=$BYTEBEAM_API_KEY \ + -it bytebeamio/simulator + +docker exec -it simulator sv stop /etc/runit/uplink +docker exec -it simulator /usr/share/bytebeam/uplink/simulator.sh download_auth_config $DEVICE_ID diff --git a/qa-scripts/streams.sh b/qa-scripts/streams.sh new file mode 100755 index 000000000..5606c2675 --- /dev/null +++ b/qa-scripts/streams.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -xe + +source qa-scripts/.env + +printf "$(cat << EOF +[simulator] +actions = [] +gps_paths = "./paths/" + +[streams.bms] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/bms/jsonarray" +flush_period = 2 + +[streams.imu] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/imu/jsonarray" +batch_size = 10 +EOF +)" > devices/streams.toml +docker cp devices/streams.toml simulator:/usr/share/bytebeam/uplink/devices/streams.toml + +docker exec -it simulator uplink -a /usr/share/bytebeam/uplink/devices/device_$DEVICE_ID.json -c /usr/share/bytebeam/uplink/devices/streams.toml -vv -m uplink::base::bridge diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..7c94e9fe9 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,12 @@ +[toolchain] +channel = "1.78.0" +targets = [ + "x86_64-linux-android", + "i686-linux-android", + "aarch64-linux-android", + "armv7-linux-androideabi" +] +profile = "minimal" +components = [ + "clippy" +] diff --git a/simulator.sh b/simulator.sh index 8b2fad37e..240a51f71 100755 --- a/simulator.sh +++ b/simulator.sh @@ -42,6 +42,30 @@ persistence_path = \"/var/tmp/persistence/$id\" max_file_size = 104857600 max_file_count = 3 +[streams.gps] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/gps/jsonarray" +persistence = { max_file_size = 0 } + +[streams.bms] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/bms/jsonarray" +persistence = { max_file_size = 0 } + +[streams.imu] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/imu/jsonarray" +persistence = { max_file_size = 0 } + +[streams.motor] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/motor/jsonarray" +persistence = { max_file_size = 0 } + +[streams.peripheral_state] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/peripheral_state/jsonarray" +persistence = { max_file_size = 0 } + +[streams.device_shadow] +topic = "/tenants/{tenant_id}/devices/{device_id}/events/device_shadow/jsonarray" +persistence = { max_file_size = 0 } + [simulator] gps_paths = "./paths" actions= [{ name = \"load_file\" }, { name = \"install_firmware\" }, { name = \"update_config\" }, { name = \"unlock\" }, { name = \"lock\" }] @@ -59,7 +83,7 @@ download_auth_config() { echo "Downloading config: $url" mkdir -p devices curl --location $url \ - --header 'x-bytebeam-tenant: demo' \ + --header "x-bytebeam-tenant: $BYTEBEAM_TENANT_ID" \ --header "x-bytebeam-api-key: $BYTEBEAM_API_KEY" > devices/device_$id.json } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 4543ec18f..ba212fad3 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -5,13 +5,13 @@ authors = ["tekjar "] edition = "2021" [dependencies] -bytes = "1" -log = "0.4" +bytes = { workspace = true } +log = { workspace = true } seahash = "4" -thiserror = "1" +thiserror = { workspace = true } [dev-dependencies] -tempdir = "0.3" -rand = "0.7" -mqttbytes = "0.1" +tempdir = { workspace = true } +rand = { workspace = true } +rumqttc = { workspace = true } pretty_env_logger = "0.4" diff --git a/storage/src/lib.rs b/storage/src/lib.rs index acaa6fef8..ea8522ee0 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,10 +1,10 @@ use bytes::{Buf, BufMut, BytesMut}; -use log::{self, error, info, warn}; +use log::{debug, error, info, warn}; use seahash::hash; use std::collections::VecDeque; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Read, Write}; +use std::fs::{self, OpenOptions}; +use std::io::{self, copy, Write}; use std::mem; use std::path::{Path, PathBuf}; @@ -16,9 +16,12 @@ pub enum Error { NotBackup, #[error("Corrupted backup file")] CorruptedFile, + #[error("Empty write buffer")] + NoWrites, } pub struct Storage { + name: String, /// maximum allowed file size max_file_size: usize, /// current open file @@ -30,8 +33,9 @@ pub struct Storage { } impl Storage { - pub fn new(max_file_size: usize) -> Storage { + pub fn new(name: impl Into, max_file_size: usize) -> Storage { Storage { + name: name.into(), max_file_size, current_write_file: BytesMut::with_capacity(max_file_size * 2), current_read_file: BytesMut::with_capacity(max_file_size * 2), @@ -68,6 +72,13 @@ impl Storage { } } + pub fn disk_utilized(&self) -> usize { + match &self.persistence { + Some(p) => p.bytes_occupied, + None => 0, + } + } + pub fn inmemory_read_size(&self) -> usize { self.current_read_file.len() } @@ -83,22 +94,34 @@ impl Storage { return Ok(None); } + self.flush() + } + + /// Force flush the contents of write buffer onto disk + pub fn flush(&mut self) -> Result, Error> { + if self.current_write_file.is_empty() { + return Err(Error::NoWrites); + } match &mut self.persistence { Some(persistence) => { - let hash = hash(&self.current_write_file[..]); - let mut next_file = persistence.open_next_write_file()?; - info!("Flushing data to disk!! {:?}", next_file.path); + let NextFile { mut file, deleted } = persistence.open_next_write_file()?; + info!("Flushing data to disk for stoarge: {}; path = {:?}", self.name, file.path()); + file.write(&mut self.current_write_file)?; - next_file.file.write_all(&hash.to_be_bytes())?; - next_file.file.write_all(&self.current_write_file[..])?; - next_file.file.flush()?; + // 8 is the number of bytes the hash(u64) occupies + persistence.bytes_occupied += 8 + self.current_write_file.len(); self.current_write_file.clear(); - Ok(next_file.deleted) + + Ok(deleted) } None => { // TODO(RT): Make sure that disk files starts with id 1 to represent in memory file // with id 0 self.current_write_file.clear(); + warn!( + "Persistence disabled for storage: {}. Deleted in-memory buffer on overflow", + self.name + ); Ok(Some(0)) } } @@ -116,9 +139,12 @@ impl Storage { } if let Some(persistence) = &mut self.persistence { - // Remove read file on completion - if let Some(id) = persistence.current_read_file_id.take() { - persistence.remove(id)?; + // Remove read file on completion in destructive-read mode + let read_is_destructive = !persistence.non_destructive_read; + let read_file_id = persistence.current_read_file_id.take(); + if let Some(id) = read_is_destructive.then_some(read_file_id).flatten() { + let deleted_file = persistence.remove(id)?; + debug!("Completed reading a persistence file, deleting it; storage = {}, path = {deleted_file:?}", self.name); } // Swap read buffer with write buffer to read data in inmemory write @@ -183,9 +209,91 @@ fn get_file_ids(path: &Path) -> Result, Error> { Ok(file_ids) } -struct NextFile { - path: PathBuf, - file: File, +/// A handle to describe a persistence file on disk +pub struct PersistenceFile<'a> { + /// Path to the persistence directory + dir: &'a Path, + /// Name of the file e.g. `backup@1` + file_name: String, +} + +impl<'a> PersistenceFile<'a> { + pub fn new(dir: &'a Path, file_name: String) -> Result { + Ok(Self { dir, file_name }) + } + + /// Path of persistence file when stored on disk + pub fn path(&self) -> PathBuf { + self.dir.join(&self.file_name) + } + + // Moves the corrupt persistence file into special directory + fn handle_corrupt_file(&self) -> Result<(), Error> { + let path_src = self.path(); + let dest_dir = self.dir.join("corrupted"); + fs::create_dir_all(&dest_dir)?; + let path_dest = dest_dir.join(&self.file_name); + + warn!("Moving corrupted file from {path_src:?} to {path_dest:?}"); + fs::rename(path_src, path_dest)?; + + Ok(()) + } + + /// Read contents of the persistence file from disk into buffer in memory + pub fn read(&mut self, buf: &mut BytesMut) -> Result<(), Error> { + let path = self.path(); + let mut file = OpenOptions::new().read(true).open(path)?; + + // Initialize buffer and load next read file + buf.clear(); + copy(&mut file, &mut buf.writer())?; + + // Verify with checksum + if buf.len() < 8 { + self.handle_corrupt_file()?; + return Err(Error::CorruptedFile); + } + + let expected_hash = buf.get_u64(); + let actual_hash = hash(&buf[..]); + if actual_hash != expected_hash { + self.handle_corrupt_file()?; + return Err(Error::CorruptedFile); + } + + Ok(()) + } + + /// Write contents of buffer from memory onto the persistence file in disk + pub fn write(&mut self, buf: &mut BytesMut) -> Result<(), Error> { + let path = self.path(); + let mut file = OpenOptions::new().write(true).create(true).truncate(true).open(path)?; + + let hash = hash(&buf[..]); + file.write_all(&hash.to_be_bytes())?; + file.write_all(&buf[..])?; + file.flush()?; + + Ok(()) + } + + /// Deletes the persistence file from disk + pub fn delete(&mut self) -> Result { + let path = self.path(); + + // Query the fs to track size of removed persistence file + let metadata = fs::metadata(&path)?; + let bytes_occupied = metadata.len(); + + fs::remove_file(&path)?; + + Ok(bytes_occupied) + } +} + +struct NextFile<'a> { + file: PersistenceFile<'a>, deleted: Option, } @@ -201,6 +309,8 @@ struct Persistence { // /// Deleted file id // deleted: Option, non_destructive_read: bool, + /// Disk space(in bytes) currently occupied by persistence files + bytes_occupied: usize, } impl Persistence { @@ -209,6 +319,13 @@ impl Persistence { let backlog_files = get_file_ids(&path)?; info!("List of file ids loaded from disk: {backlog_files:?}"); + let bytes_occupied = backlog_files.iter().fold(0, |acc, id| { + let mut file = PathBuf::from(&path); + let file_name = format!("backup@{id}"); + file.push(file_name); + fs::metadata(&file).unwrap().len() as usize + acc + }); + Ok(Persistence { path, max_file_count, @@ -216,55 +333,26 @@ impl Persistence { current_read_file_id: None, // deleted: None, non_destructive_read: false, + bytes_occupied, }) } - fn path(&self, id: u64) -> Result { - let file_name = format!("backup@{id}"); - let path = self.path.join(file_name); - - Ok(path) - } - - /// Removes a file with provided id - fn remove(&self, id: u64) -> Result<(), Error> { - if self.non_destructive_read { - return Ok(()); - } - - let path = self.path(id)?; - fs::remove_file(path)?; - - Ok(()) - } + /// Removes a persistence file with provided id + fn remove(&mut self, id: u64) -> Result { + let file_name = format!("backup@{}", id); + let mut file = PersistenceFile::new(&self.path, file_name)?; + let path = file.path(); - /// Move corrupt file to special directory - fn handle_corrupt_file(&self) -> Result<(), Error> { - let id = self.current_read_file_id.expect("There is supposed to be a file here"); - let path_src = self.path(id)?; - let dest_dir = self.path.join("corrupted"); - fs::create_dir_all(&dest_dir)?; - - let file_name = path_src.file_name().expect("The file name should exist"); - let path_dest = dest_dir.join(file_name); - - warn!("Moving corrupted file from {path_src:?} to {path_dest:?}"); - fs::rename(path_src, path_dest)?; + self.bytes_occupied -= file.delete()? as usize; - Ok(()) + Ok(path) } /// Opens file to flush current inmemory write buffer to disk. /// Also handles retention of previous files on disk fn open_next_write_file(&mut self) -> Result { let next_file_id = self.backlog_files.back().map_or(0, |id| id + 1); - let file_name = format!("backup@{next_file_id}"); - let next_file_path = self.path.join(file_name); - let next_file = OpenOptions::new().write(true).create(true).open(&next_file_path)?; - self.backlog_files.push_back(next_file_id); - - let mut next = NextFile { path: next_file_path, file: next_file, deleted: None }; let mut backlog_files_count = self.backlog_files.len(); // File being read is also to be considered @@ -272,57 +360,40 @@ impl Persistence { backlog_files_count += 1 } - // Return next file details if backlog is within limits - if backlog_files_count <= self.max_file_count { - return Ok(next); - } + // Delete earliest file if backlog limits crossed + let deleted = if backlog_files_count > self.max_file_count { + // Remove file being read, or first in backlog + // NOTE: keeps read buffer unchanged + let id = match self.current_read_file_id.take() { + Some(id) => id, + _ => self.backlog_files.pop_front().unwrap(), + }; + + if !self.non_destructive_read { + let deleted_file = self.remove(id)?; + warn!("file limit reached. deleting backup@{}; path = {deleted_file:?}", id); + } - // Remove file being read, or first in backlog - // NOTE: keeps read buffer unchanged - let id = match self.current_read_file_id.take() { - Some(id) => id, - _ => self.backlog_files.pop_front().unwrap(), + Some(id) + } else { + None }; - warn!("file limit reached. deleting backup@{}", id); - - next.deleted = Some(id); - self.remove(id)?; - - Ok(next) + let file_name = format!("backup@{}", next_file_id); + Ok(NextFile { file: PersistenceFile::new(&self.path, file_name)?, deleted }) } + /// Load the next persistence file to be read into memory fn load_next_read_file(&mut self, current_read_file: &mut BytesMut) -> Result<(), Error> { // Len always > 0 because of above if. Doesn't panic let id = self.backlog_files.pop_front().unwrap(); - let next_file_path = self.path(id)?; - - let mut file = OpenOptions::new().read(true).open(&next_file_path)?; + let file_name = format!("backup@{}", id); + let mut file = PersistenceFile::new(&self.path, file_name)?; // Load file into memory and store its id for deleting in the future - let metadata = fs::metadata(&next_file_path)?; - - // Initialize next read file with 0s - current_read_file.clear(); - let init = vec![0u8; metadata.len() as usize]; - current_read_file.put_slice(&init); - - file.read_exact(&mut current_read_file[..])?; + file.read(current_read_file)?; self.current_read_file_id = Some(id); - // Verify with checksum - if current_read_file.len() < 8 { - self.handle_corrupt_file()?; - return Err(Error::CorruptedFile); - } - - let expected_hash = current_read_file.get_u64(); - let actual_hash = hash(¤t_read_file[..]); - if actual_hash != expected_hash { - self.handle_corrupt_file()?; - return Err(Error::CorruptedFile); - } - Ok(()) } } @@ -330,7 +401,7 @@ impl Persistence { #[cfg(test)] mod test { use super::*; - use mqttbytes::*; + use rumqttc::*; use tempdir::TempDir; fn init_backup_folders() -> TempDir { @@ -360,7 +431,7 @@ mod test { break; } - match read(storage.reader(), 1048).unwrap() { + match Packet::read(storage.reader(), 1048).unwrap() { Packet::Publish(p) => publishes.push(p), packet => unreachable!("{:?}", packet), } @@ -373,7 +444,7 @@ mod test { fn flush_creates_new_file_after_size_limit() { // 1036 is the size of a publish message with topic = "hello", qos = 1, payload = 1024 bytes let backup = init_backup_folders(); - let mut storage = Storage::new(10 * 1036); + let mut storage = Storage::new("test", 10 * 1036); storage.set_persistence(backup.path(), 10).unwrap(); // 2 files on disk and a partially filled in memory buffer @@ -390,7 +461,7 @@ mod test { #[test] fn old_file_is_deleted_after_limit() { let backup = init_backup_folders(); - let mut storage = Storage::new(10 * 1036); + let mut storage = Storage::new("test", 10 * 1036); storage.set_persistence(backup.path(), 10).unwrap(); // 11 files created. 10 on disk @@ -410,7 +481,7 @@ mod test { #[test] fn reload_loads_correct_file_into_memory() { let backup = init_backup_folders(); - let mut storage = Storage::new(10 * 1036); + let mut storage = Storage::new("test", 10 * 1036); storage.set_persistence(backup.path(), 10).unwrap(); // 10 files on disk @@ -428,7 +499,7 @@ mod test { #[test] fn reload_loads_partially_written_write_buffer_correctly() { let backup = init_backup_folders(); - let mut storage = Storage::new(10 * 1036); + let mut storage = Storage::new("test", 10 * 1036); storage.set_persistence(backup.path(), 10).unwrap(); // 10 files on disk and partially filled current write buffer @@ -447,7 +518,7 @@ mod test { #[test] fn ensure_file_remove_on_read_completion_only() { let backup = init_backup_folders(); - let mut storage = Storage::new(10 * 1036); + let mut storage = Storage::new("test", 10 * 1036); storage.set_persistence(backup.path(), 10).unwrap(); // 10 files on disk and partially filled current write buffer, 10 publishes per file write_n_publishes(&mut storage, 105); @@ -481,7 +552,7 @@ mod test { #[test] fn ensure_files_including_read_removed_post_flush_on_overflow() { let backup = init_backup_folders(); - let mut storage = Storage::new(10 * 1036); + let mut storage = Storage::new("test", 10 * 1036); storage.set_persistence(backup.path(), 10).unwrap(); // 10 files on disk and partially filled current write buffer, 10 publishes per file write_n_publishes(&mut storage, 105); diff --git a/tools/actions/main.go b/tools/actions/main.go index 709f13e57..844ca4839 100644 --- a/tools/actions/main.go +++ b/tools/actions/main.go @@ -14,15 +14,13 @@ import ( type Action struct { ID string `json:"action_id"` - Kind string `json:"kind"` Command string `json:"name"` Payload string `json:"payload"` } -func NewAction(id, kind, command, payload string) *Action { +func NewAction(id, command, payload string) *Action { action := Action{ ID: id, - Kind: kind, Command: command, Payload: payload, } @@ -75,34 +73,29 @@ func createAction(name string) *Action { fmt.Println("action =", name, "id =", id) switch name { case "update_firmware": - kind := "process" command := "tools/ota" payload := `{"hello": "world"}` - action := NewAction(id, kind, command, payload) + action := NewAction(id, command, payload) return action case "stop_collector": - kind := "control" command := name payload := `{"hello": "world"}` - action := NewAction(id, kind, command, payload) + action := NewAction(id, command, payload) return action case "start_collector": - kind := "control" command := name payload := `{"args": ["simulator"]}` - action := NewAction(id, kind, command, payload) + action := NewAction(id, command, payload) return action case "stop_collector_stream": - kind := "control" command := name payload := `{"args": ["simulator", "gps"]}` - action := NewAction(id, kind, command, payload) + action := NewAction(id, command, payload) return action case "start_collector_strea": - kind := "control" command := name payload := `{"args": ["simulator", "gps"]}` - action := NewAction(id, kind, command, payload) + action := NewAction(id, command, payload) return action default: fmt.Println("Invalid action") diff --git a/tools/deserialize-backup/src/main.rs b/tools/deserialize-backup/src/main.rs index 2f129709b..c86caf21c 100644 --- a/tools/deserialize-backup/src/main.rs +++ b/tools/deserialize-backup/src/main.rs @@ -115,14 +115,18 @@ fn main() -> Result<(), Error> { let dirs = std::fs::read_dir(commandline.directory)?; for dir in dirs { let dir = dir?; + if dir.metadata()?.is_file() { + continue; + } + + let path = dir.path(); + let stream_name = dir.path().into_iter().last().unwrap().to_string_lossy().to_string(); // NOTE: max_file_size and max_file_count should not matter when reading non-destructively - let mut storage = storage::Storage::new(1048576); - storage.set_persistence(dir.path(), 3)?; + let mut storage = storage::Storage::new(&stream_name, 1048576); + storage.set_persistence(path, 3)?; storage.set_non_destructive_read(true); - let stream = streams - .entry(dir.path().into_iter().last().unwrap().to_string_lossy().to_string()) - .or_default(); + let stream = streams.entry(stream_name).or_default(); 'outer: loop { loop { match storage.reload_on_eof() { diff --git a/tools/system-stats/Cargo.lock b/tools/system-stats/Cargo.lock new file mode 100644 index 000000000..654de981c --- /dev/null +++ b/tools/system-stats/Cargo.lock @@ -0,0 +1,841 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crossbeam-deque" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "deranged" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "simplelog" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" +dependencies = [ + "log", + "termcolor", + "time", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.26.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "system-stats" +version = "0.1.0" +dependencies = [ + "futures-util", + "log", + "serde", + "serde_json", + "simplelog", + "structopt", + "sysinfo", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "time" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +dependencies = [ + "deranged", + "itoa", + "libc", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] + +[[package]] +name = "tokio" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +dependencies = [ + "backtrace", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "slab", + "tokio", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/tools/system-stats/Cargo.toml b/tools/system-stats/Cargo.toml new file mode 100644 index 000000000..f52357769 --- /dev/null +++ b/tools/system-stats/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "system-stats" +version = "0.1.0" +edition = "2021" +authors = ["Devdutt Shenoi "] + +[dependencies] +futures-util = { version = "0.3", features = ["sink"] } +log = "0.4" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0" +simplelog = "0.12.0" +structopt = "0.3" +sysinfo = "0.26" +thiserror = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "net"] } +tokio-stream = "0.1" +tokio-util = { version = "0.7", features = ["codec", "time"] } \ No newline at end of file diff --git a/tools/system-stats/src/lib.rs b/tools/system-stats/src/lib.rs new file mode 100644 index 000000000..200a10d9b --- /dev/null +++ b/tools/system-stats/src/lib.rs @@ -0,0 +1,636 @@ +use futures_util::SinkExt; +use log::error; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sysinfo::{ + ComponentExt, CpuExt, DiskExt, NetworkData, NetworkExt, PidExt, ProcessExt, SystemExt, +}; +use tokio::{net::TcpStream, time::interval}; +use tokio_util::codec::{Framed, LinesCodec, LinesCodecError}; + +use std::{ + collections::HashMap, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Io error {0}")] + Io(#[from] std::io::Error), + #[error("Lines codec error {0}")] + Codec(#[from] LinesCodecError), + #[error("Serde error {0}")] + Json(#[from] serde_json::error::Error), +} + +#[derive(Debug, Serialize)] +pub struct Payload { + pub stream: String, + pub sequence: u32, + pub timestamp: u64, + #[serde(flatten)] + pub payload: Value, +} + +fn clock() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64 +} + +type Pid = u32; + +#[derive(Debug, Default, Serialize, Clone)] +pub struct System { + sequence: u32, + timestamp: u64, + kernel_version: String, + uptime: u64, + no_processes: usize, + /// Average load within one minute. + load_avg_one: f64, + /// Average load within five minutes. + load_avg_five: f64, + /// Average load within fifteen minutes. + load_avg_fifteen: f64, + total_memory: u64, + available_memory: u64, + used_memory: u64, +} + +impl System { + fn init(sys: &sysinfo::System) -> System { + System { + kernel_version: match sys.kernel_version() { + Some(kv) => kv, + None => String::default(), + }, + total_memory: sys.total_memory(), + ..Default::default() + } + } + + fn update(&mut self, sys: &sysinfo::System, timestamp: u64) { + self.sequence += 1; + self.timestamp = timestamp; + self.uptime = sys.uptime(); + self.no_processes = sys.processes().len(); + let sysinfo::LoadAvg { one, five, fifteen } = sys.load_average(); + self.load_avg_one = one; + self.load_avg_five = five; + self.load_avg_fifteen = fifteen; + self.available_memory = sys.available_memory(); + self.used_memory = self.total_memory - self.available_memory; + } +} + +impl From<&System> for Payload { + fn from(value: &System) -> Self { + let System { + sequence, + timestamp, + kernel_version, + uptime, + no_processes, + load_avg_one, + load_avg_five, + load_avg_fifteen, + total_memory, + available_memory, + used_memory, + } = value; + + Payload { + stream: "uplink".to_owned(), + sequence: *sequence, + timestamp: *timestamp, + payload: json!({ + "kernel_version": kernel_version, + "uptime": uptime, + "no_processes": no_processes, + "load_avg_one": load_avg_one, + "load_avg_five": load_avg_five, + "load_avg_fifteen": load_avg_fifteen, + "total_memory": total_memory, + "available_memory": available_memory, + "used_memory": used_memory, + }), + } + } +} + +struct SystemStats { + stat: System, +} + +impl SystemStats { + fn push(&mut self, sys: &sysinfo::System, timestamp: u64) -> Payload { + self.stat.update(sys, timestamp); + + (&self.stat).into() + } +} + +#[derive(Debug, Serialize, Clone)] +struct Network { + sequence: u32, + timestamp: u64, + name: String, + incoming_data_rate: f64, + outgoing_data_rate: f64, + #[serde(skip_serializing)] + timer: Instant, +} + +impl Network { + fn init(name: String) -> Self { + Network { + sequence: 0, + timestamp: 0, + name, + incoming_data_rate: 0.0, + outgoing_data_rate: 0.0, + timer: Instant::now(), + } + } + + /// Update metrics values for network usage over time + fn update(&mut self, data: &NetworkData, timestamp: u64, sequence: u32) { + let update_period = self.timer.elapsed().as_secs_f64(); + // TODO: check if these calculations are correct + self.incoming_data_rate = data.total_received() as f64 / update_period; + self.outgoing_data_rate = data.total_transmitted() as f64 / update_period; + self.timestamp = timestamp; + self.sequence = sequence; + } +} + +impl From<&mut Network> for Payload { + fn from(value: &mut Network) -> Self { + let Network { sequence, timestamp, name, incoming_data_rate, outgoing_data_rate, .. } = + value; + + Payload { + stream: "uplink_network_stats".to_owned(), + sequence: *sequence, + timestamp: *timestamp, + payload: json!({ + "name": name, + "incoming_data_rate": incoming_data_rate, + "outgoing_data_rate": outgoing_data_rate, + }), + } + } +} + +struct NetworkStats { + sequence: u32, + map: HashMap, +} + +impl NetworkStats { + fn push( + &mut self, + net_name: String, + net_data: &sysinfo::NetworkData, + timestamp: u64, + ) -> Payload { + self.sequence += 1; + let net = self.map.entry(net_name.clone()).or_insert_with(|| Network::init(net_name)); + net.update(net_data, timestamp, self.sequence); + + net.into() + } +} + +#[derive(Debug, Serialize, Default, Clone)] +struct Disk { + sequence: u32, + timestamp: u64, + name: String, + total: u64, + available: u64, + used: u64, +} + +impl Disk { + fn init(name: String, disk: &sysinfo::Disk) -> Self { + Disk { name, total: disk.total_space(), ..Default::default() } + } + + fn update(&mut self, disk: &sysinfo::Disk, timestamp: u64, sequence: u32) { + self.total = disk.total_space(); + self.available = disk.available_space(); + self.used = self.total - self.available; + self.timestamp = timestamp; + self.sequence = sequence; + } +} + +impl From<&mut Disk> for Payload { + fn from(value: &mut Disk) -> Self { + let Disk { sequence, timestamp, name, total, available, used } = value; + + Payload { + stream: "uplink_disk_stats".to_owned(), + sequence: *sequence, + timestamp: *timestamp, + payload: json!({ + "name": name, + "total": total, + "available": available, + "used": used, + }), + } + } +} + +struct DiskStats { + sequence: u32, + map: HashMap, +} + +impl DiskStats { + fn push(&mut self, disk_data: &sysinfo::Disk, timestamp: u64) -> Payload { + self.sequence += 1; + let disk_name = disk_data.name().to_string_lossy().to_string(); + let disk = + self.map.entry(disk_name.clone()).or_insert_with(|| Disk::init(disk_name, disk_data)); + disk.update(disk_data, timestamp, self.sequence); + + disk.into() + } +} + +#[derive(Debug, Default, Serialize, Clone)] +struct Processor { + sequence: u32, + timestamp: u64, + name: String, + frequency: u64, + usage: f32, +} + +impl Processor { + fn init(name: String) -> Self { + Processor { name, ..Default::default() } + } + + fn update(&mut self, proc: &sysinfo::Cpu, timestamp: u64, sequence: u32) { + self.frequency = proc.frequency(); + self.usage = proc.cpu_usage(); + self.timestamp = timestamp; + self.sequence = sequence; + } +} + +impl From<&mut Processor> for Payload { + fn from(value: &mut Processor) -> Self { + let Processor { sequence, timestamp, name, frequency, usage } = value; + + Payload { + stream: "uplink_processor_stats".to_owned(), + sequence: *sequence, + timestamp: *timestamp, + payload: json!({ + "name": name, + "frequency": frequency, + "usage": usage, + }), + } + } +} + +struct ProcessorStats { + sequence: u32, + map: HashMap, +} + +impl ProcessorStats { + fn push(&mut self, proc_data: &sysinfo::Cpu, timestamp: u64) -> Payload { + let proc_name = proc_data.name().to_string(); + self.sequence += 1; + let proc = self.map.entry(proc_name.clone()).or_insert_with(|| Processor::init(proc_name)); + proc.update(proc_data, timestamp, self.sequence); + + proc.into() + } +} + +#[derive(Debug, Default, Serialize, Clone)] +struct Component { + sequence: u32, + timestamp: u64, + label: String, + temperature: f32, +} + +impl Component { + fn init(label: String) -> Self { + Component { label, ..Default::default() } + } + + fn update(&mut self, comp: &sysinfo::Component, timestamp: u64, sequence: u32) { + self.temperature = comp.temperature(); + self.timestamp = timestamp; + self.sequence = sequence; + } +} + +impl From<&mut Component> for Payload { + fn from(value: &mut Component) -> Self { + let Component { sequence, timestamp, label, temperature } = value; + + Payload { + stream: "uplink_component_stats".to_owned(), + sequence: *sequence, + timestamp: *timestamp, + payload: json!({ + "label": label, "temperature": temperature + }), + } + } +} + +struct ComponentStats { + sequence: u32, + map: HashMap, +} + +impl ComponentStats { + fn push(&mut self, comp_data: &sysinfo::Component, timestamp: u64) -> Payload { + let comp_label = comp_data.label().to_string(); + self.sequence += 1; + let comp = + self.map.entry(comp_label.clone()).or_insert_with(|| Component::init(comp_label)); + comp.update(comp_data, timestamp, self.sequence); + + comp.into() + } +} + +#[derive(Debug, Default, Serialize, Clone)] +struct Process { + sequence: u32, + timestamp: u64, + pid: Pid, + name: String, + cpu_usage: f32, + mem_usage: u64, + disk_total_written_bytes: u64, + disk_written_bytes: u64, + disk_total_read_bytes: u64, + disk_read_bytes: u64, + start_time: u64, +} + +impl Process { + fn init(pid: Pid, name: String, start_time: u64) -> Self { + Process { pid, name, start_time, ..Default::default() } + } + + fn update(&mut self, proc: &sysinfo::Process, timestamp: u64, sequence: u32) { + let sysinfo::DiskUsage { total_written_bytes, written_bytes, total_read_bytes, read_bytes } = + proc.disk_usage(); + self.disk_total_written_bytes = total_written_bytes; + self.disk_written_bytes = written_bytes; + self.disk_total_read_bytes = total_read_bytes; + self.disk_read_bytes = read_bytes; + self.cpu_usage = proc.cpu_usage(); + self.mem_usage = proc.memory(); + self.timestamp = timestamp; + self.sequence = sequence; + } +} + +impl From<&mut Process> for Payload { + fn from(value: &mut Process) -> Self { + let Process { + sequence, + timestamp, + pid, + name, + cpu_usage, + mem_usage, + disk_total_written_bytes, + disk_written_bytes, + disk_total_read_bytes, + disk_read_bytes, + start_time, + } = value; + + Payload { + stream: "uplink_process_stats".to_owned(), + sequence: *sequence, + timestamp: *timestamp, + payload: json!({ + "pid": pid, + "name": name, + "cpu_usage": cpu_usage, + "mem_usage": mem_usage, + "disk_total_written_bytes": disk_total_written_bytes, + "disk_written_bytes": disk_written_bytes, + "disk_total_read_bytes": disk_total_read_bytes, + "disk_read_bytes": disk_read_bytes, + "start_time": start_time, + }), + } + } +} + +struct ProcessStats { + sequence: u32, + map: HashMap, +} + +impl ProcessStats { + fn push( + &mut self, + id: Pid, + proc_data: &sysinfo::Process, + name: String, + timestamp: u64, + ) -> Payload { + self.sequence += 1; + let proc = + self.map.entry(id).or_insert_with(|| Process::init(id, name, proc_data.start_time())); + proc.update(proc_data, timestamp, self.sequence); + + proc.into() + } +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct Config { + pub process_names: Vec, + pub update_period: u64, +} + +/// Collects and forward system information such as kernel version, memory and disk space usage, +/// information regarding running processes, network and processor usage, etc to an IoT platform. +pub struct StatCollector { + /// Handle to sysinfo struct containing system information. + sys: sysinfo::System, + /// System information values to be serialized. + system: SystemStats, + /// Information about running processes. + processes: ProcessStats, + /// Individual Processor information. + processors: ProcessorStats, + /// Information regarding individual Network interfaces. + networks: NetworkStats, + /// Information regarding individual Disks. + disks: DiskStats, + /// Temperature information from individual components. + components: ComponentStats, + /// System stats configuration. + config: Config, + /// Handle to.send stats as payload onto bridge + client: Framed, +} + +impl StatCollector { + /// Create and initialize a stat collector + pub fn new(config: Config, client: Framed) -> Self { + let mut sys = sysinfo::System::new(); + sys.refresh_disks_list(); + sys.refresh_networks_list(); + sys.refresh_memory(); + sys.refresh_cpu(); + sys.refresh_components(); + + let mut map = HashMap::new(); + for disk_data in sys.disks() { + let disk_name = disk_data.name().to_string_lossy().to_string(); + map.insert(disk_name.clone(), Disk::init(disk_name, disk_data)); + } + let disks = DiskStats { sequence: 0, map }; + + let mut map = HashMap::new(); + for (net_name, _) in sys.networks() { + map.insert(net_name.to_owned(), Network::init(net_name.to_owned())); + } + let networks = NetworkStats { sequence: 0, map }; + + let mut map = HashMap::new(); + for proc in sys.cpus().iter() { + let proc_name = proc.name().to_owned(); + map.insert(proc_name.clone(), Processor::init(proc_name)); + } + let processors = ProcessorStats { sequence: 0, map }; + + let processes = ProcessStats { sequence: 0, map: HashMap::new() }; + let components = ComponentStats { sequence: 0, map: HashMap::new() }; + + let system = SystemStats { stat: System::init(&sys) }; + + StatCollector { + sys, + system, + config, + processes, + disks, + networks, + processors, + components, + client, + } + } + + /// Stat collector execution loop, sleeps for the duation of `config.stats.update_period` in seconds. + /// Update system information values and increment sequence numbers, while.sending to specific data streams. + pub async fn start(mut self) -> Result<(), Error> { + let mut interval = interval(Duration::from_secs(self.config.update_period)); + loop { + interval.tick().await; + + self.update_memory_stats().await?; + self.update_disk_stats().await?; + self.update_network_stats().await?; + self.update_cpu_stats().await?; + self.update_component_stats().await?; + self.update_process_stats().await?; + } + } + + // Refresh memory stats + async fn update_memory_stats(&mut self) -> Result<(), Error> { + self.sys.refresh_memory(); + let timestamp = clock(); + let payload = self.system.push(&self.sys, timestamp); + let payload = serde_json::to_string(&payload)?; + self.client.send(payload).await?; + + Ok(()) + } + + // Refresh disk stats + async fn update_disk_stats(&mut self) -> Result<(), Error> { + self.sys.refresh_disks(); + let timestamp = clock(); + for disk_data in self.sys.disks() { + let payload = self.disks.push(disk_data, timestamp); + let payload = serde_json::to_string(&payload)?; + self.client.send(payload).await?; + } + + Ok(()) + } + + // Refresh network byte rate stats + async fn update_network_stats(&mut self) -> Result<(), Error> { + self.sys.refresh_networks(); + let timestamp = clock(); + for (net_name, net_data) in self.sys.networks() { + let payload = self.networks.push(net_name.to_owned(), net_data, timestamp); + let payload = serde_json::to_string(&payload)?; + self.client.send(payload).await?; + } + + Ok(()) + } + + // Refresh processor stats + async fn update_cpu_stats(&mut self) -> Result<(), Error> { + self.sys.refresh_cpu(); + let timestamp = clock(); + for proc_data in self.sys.cpus().iter() { + let payload = self.processors.push(proc_data, timestamp); + let payload = serde_json::to_string(&payload)?; + self.client.send(payload).await?; + } + + Ok(()) + } + + // Refresh component stats + async fn update_component_stats(&mut self) -> Result<(), Error> { + self.sys.refresh_components(); + let timestamp = clock(); + for comp_data in self.sys.components().iter() { + let payload = self.components.push(comp_data, timestamp); + let payload = serde_json::to_string(&payload)?; + self.client.send(payload).await?; + } + + Ok(()) + } + + // Refresh processes info + // NOTE: This can be further optimized by storing pids of interested processes + // at init and only collecting process information for them instead of iterating + // over all running processes as is being done now. + async fn update_process_stats(&mut self) -> Result<(), Error> { + self.sys.refresh_processes(); + let timestamp = clock(); + for (&id, p) in self.sys.processes() { + let name = p.cmd().get(0).map(|s| s.to_string()).unwrap_or(p.name().to_string()); + + if self.config.process_names.contains(&name) { + let payload = self.processes.push(id.as_u32(), p, name, timestamp); + let payload = serde_json::to_string(&payload)?; + self.client.send(payload).await?; + } + } + + Ok(()) + } +} diff --git a/tools/system-stats/src/main.rs b/tools/system-stats/src/main.rs new file mode 100644 index 000000000..131d03932 --- /dev/null +++ b/tools/system-stats/src/main.rs @@ -0,0 +1,71 @@ +use std::time::Duration; + +use log::{error, LevelFilter}; +use simplelog::{ + ColorChoice, CombinedLogger, ConfigBuilder, LevelPadding, TermLogger, TerminalMode, +}; +use structopt::StructOpt; +use system_stats::{Config, StatCollector}; +use tokio::net::TcpStream; +use tokio_util::codec::{Framed, LinesCodec}; + +#[derive(StructOpt, Debug)] +#[structopt(name = "simulator", about = "simulates a demo device")] +pub struct CommandLine { + /// uplink port + #[structopt(short = "p", help = "uplink port")] + pub port: u16, + /// log level (v: info, vv: debug, vvv: trace) + #[structopt(short = "v", long = "verbose", parse(from_occurrences))] + pub verbose: u8, + /// name of processes to be monitored + #[structopt(short = "P", help = "processes")] + pub process_names: Vec, + /// time between updates + #[structopt(short = "t", help = "update period", default_value = "30")] + pub update_period: u64, +} + +#[tokio::main] +async fn main() { + let CommandLine { process_names, update_period, port, .. } = init(); + + let addr = format!("localhost:{}", port); + + loop { + let Ok(stream) = TcpStream::connect(&addr).await else { + error!("Uplink is not running, will reconnect after sleeping"); + std::thread::sleep(Duration::from_secs(update_period)); + continue; + }; + let client = Framed::new(stream, LinesCodec::new()); + let config = Config { process_names: process_names.clone(), update_period }; + + let collector = StatCollector::new(config, client); + if let Err(e) = collector.start().await { + error!("Error forwarding stats: {e}"); + } + } +} + +fn init() -> CommandLine { + let commandline: CommandLine = StructOpt::from_args(); + let level = match commandline.verbose { + 0 => LevelFilter::Warn, + 1 => LevelFilter::Info, + 2 => LevelFilter::Debug, + _ => LevelFilter::Trace, + }; + + let mut config = ConfigBuilder::new(); + config + .set_location_level(LevelFilter::Off) + .set_target_level(LevelFilter::Error) + .set_thread_level(LevelFilter::Error) + .set_level_padding(LevelPadding::Right); + + let loggers = TermLogger::new(level, config.build(), TerminalMode::Mixed, ColorChoice::Auto); + CombinedLogger::init(vec![loggers]).unwrap(); + + commandline +} diff --git a/tools/tunshell/src/main.rs b/tools/tunshell/src/main.rs index 284865e90..02a6795b3 100644 --- a/tools/tunshell/src/main.rs +++ b/tools/tunshell/src/main.rs @@ -24,7 +24,6 @@ struct Config { #[derive(Serialize)] struct Action { id: String, - kind: String, name: String, payload: String, } @@ -69,7 +68,6 @@ fn main() { let action = Action { id: "tunshell".to_string(), name: "launch_shell".to_string(), - kind: "launch_shell".to_string(), payload: serde_json::to_string(&Keys { session: target_key, relay: "eu.relay.tunshell.com".to_string(), diff --git a/tools/utils/Cargo.toml b/tools/utils/Cargo.toml index 70b7732f9..bc707d157 100644 --- a/tools/utils/Cargo.toml +++ b/tools/utils/Cargo.toml @@ -20,11 +20,11 @@ name = "wait_and_send" path = "src/wait_and_send.rs" [dependencies] -uplink = { path = "../../uplink" } -tokio = { version = "1", features = ["full"] } -futures-util = "0.3" -serde = { version = "1", features = ["derive"] } -tokio-util = { version = "0.7", features = ["codec", "time"] } -tokio-stream = "0.1" -serde_json = "1.0" -flume = "0.10" \ No newline at end of file +flume = { workspace = true } +futures-util = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tokio-util = { workspace = true } +uplink = { path = "../../uplink" } \ No newline at end of file diff --git a/tools/utils/src/push_to_uplink.rs b/tools/utils/src/push_to_uplink.rs index 4d292286c..4e882de76 100644 --- a/tools/utils/src/push_to_uplink.rs +++ b/tools/utils/src/push_to_uplink.rs @@ -17,7 +17,7 @@ async fn main() { fn argv_to_payload(pairs: &[String]) -> Value { // nlici - let stream = pairs.get(0).unwrap(); + let stream = pairs.first().unwrap(); let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64; let kv_count = pairs.len() - 1; assert_eq!(kv_count % 2, 0); diff --git a/uplink/Cargo.toml b/uplink/Cargo.toml index 23fcb38a7..f7bc2e5a5 100644 --- a/uplink/Cargo.toml +++ b/uplink/Cargo.toml @@ -1,19 +1,20 @@ [package] name = "uplink" -version = "2.5.0" +version = "2.12.2" authors = ["tekjar "] edition = "2021" [dependencies] -bytes = "1" -flume = "0.10" -rumqttc = { git = "https://github.com/bytebeamio/rumqtt" } -serde = { version = "1", features = ["derive"] } -serde_json = "1.0" -thiserror = "1" -tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.7", features = ["codec", "time"] } -tokio-stream = "0.1" +bytes = { workspace = true } +flume = { workspace = true } +rumqttc = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_with = "3.3.0" +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tokio-util = { workspace = true } # binary utils anyhow = "1" @@ -28,21 +29,25 @@ pretty-bytes = "0.2.2" storage = { path = "../storage" } # logging -log = "0.4" +log = { workspace = true } regex = "1.7.1" tracing = { version="0.1", features=["log"] } -tracing-subscriber = { version="=0.3.14", features=["env-filter"] } +tracing-subscriber = { version="0.3.18", features=["env-filter"] } -# collectors +# built-in collectors # tunshell tokio-compat-02 = "0.2.0" tunshell-client = { git = "https://github.com/bytebeamio/tunshell.git", branch = "android_patch" } # simulator fake = { version = "2.5.0", features = ["derive"] } -rand = "0.8" +rand = { workspace = true } # downloader -futures-util = "0.3" +fs2 = "0.4" +futures-util = { workspace = true } +hex = "0.4" +human_bytes = "0.4" reqwest = { version = "0.11", default-features = false, features = ["stream", "rustls-tls"] } +rsa = { version = "0.9.6", features = ["sha2"] } # systemstats sysinfo = "0.26" # logcat @@ -50,7 +55,6 @@ time = "0.3" lazy_static = "1.4.0" # installer tar = "0.4" - # device_shadow surge-ping = "0.8" @@ -65,4 +69,4 @@ signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] } vergen = { version = "7", features = ["git", "build", "time"] } [dev-dependencies] -tempdir = "0.3" \ No newline at end of file +tempdir = { workspace = true } diff --git a/uplink/src/base/actions.rs b/uplink/src/base/actions.rs index 46bf6c791..1433a1eab 100644 --- a/uplink/src/base/actions.rs +++ b/uplink/src/base/actions.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use serde_json::json; +use tokio::time::Instant; use crate::{Payload, Point}; @@ -13,12 +13,13 @@ pub struct Action { // action id #[serde(alias = "id")] pub action_id: String, - // determines if action is a process - pub kind: String, // action name pub name: String, // action payload. json. can be args/payload. depends on the invoked command pub payload: String, + // Instant at which action must be timedout + #[serde(skip)] + pub deadline: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -94,33 +95,17 @@ impl ActionResponse { self } - pub fn as_payload(&self) -> Payload { - Payload::from(self) - } - pub fn from_payload(payload: &Payload) -> Result { let intermediate = serde_json::to_value(payload)?; serde_json::from_value(intermediate) } } -impl From<&ActionResponse> for Payload { - fn from(resp: &ActionResponse) -> Self { - Self { - stream: "action_status".to_owned(), - sequence: resp.sequence, - timestamp: resp.timestamp, - payload: json!({ - "id": resp.action_id, - "state": resp.state, - "progress": resp.progress, - "errors": resp.errors - }), - } +impl Point for ActionResponse { + fn stream_name(&self) -> &str { + "action_status" } -} -impl Point for ActionResponse { fn sequence(&self) -> u32 { self.sequence } diff --git a/uplink/src/base/bridge/actions_lane.rs b/uplink/src/base/bridge/actions_lane.rs new file mode 100644 index 000000000..ffb9083a7 --- /dev/null +++ b/uplink/src/base/bridge/actions_lane.rs @@ -0,0 +1,941 @@ +use flume::{bounded, Receiver, RecvError, Sender, TrySendError}; +use log::{debug, error, info, warn}; +use serde::{Deserialize, Serialize}; +use tokio::select; +use tokio::time::{self, interval, Instant, Sleep}; + +use std::collections::HashSet; +use std::fs; +use std::path::PathBuf; +use std::{collections::HashMap, fmt::Debug, pin::Pin, sync::Arc, time::Duration}; + +use super::streams::Streams; +use super::{ActionBridgeShutdown, Package, StreamMetrics}; +use crate::config::ActionRoute; +use crate::{Action, ActionResponse, Config}; + +const TUNSHELL_ACTION: &str = "launch_shell"; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Receiver error {0}")] + Recv(#[from] RecvError), + #[error("Io error {0}")] + Io(#[from] std::io::Error), + #[error("Serde error {0}")] + Serde(#[from] serde_json::Error), + #[error("Action receiver busy or down")] + UnresponsiveReceiver, + #[error("No route for action {0}")] + NoRoute(String), + #[error("Action timedout")] + ActionTimeout, + #[error("Another action is currently being processed")] + Busy, + #[error("Action Route clash: \"{0}\"")] + ActionRouteClash(String), +} + +struct RedirectionError(Action); + +pub struct ActionsBridge { + /// All configuration + config: Arc, + /// Tx handle to give to apps + status_tx: Sender, + /// Rx to receive action status from apps + status_rx: Receiver, + /// Actions incoming from backend + actions_rx: Receiver, + /// Contains stream to send ActionResponses on + streams: Streams, + /// Apps registered with the bridge + /// NOTE: Sometimes action_routes could overlap, the latest route + /// to be registered will be used in such a circumstance. + action_routes: HashMap, + /// Action redirections + action_redirections: HashMap, + /// Current action that is being processed + current_action: Option, + parallel_actions: HashSet, + ctrl_rx: Receiver, + ctrl_tx: Sender, + shutdown_handle: Sender<()>, +} + +impl ActionsBridge { + pub fn new( + config: Arc, + package_tx: Sender>, + actions_rx: Receiver, + shutdown_handle: Sender<()>, + metrics_tx: Sender, + ) -> Self { + let (status_tx, status_rx) = bounded(10); + let action_redirections = config.action_redirections.clone(); + let (ctrl_tx, ctrl_rx) = bounded(1); + + let mut streams_config = HashMap::new(); + let mut action_status = config.action_status.clone(); + if action_status.batch_size > 1 { + warn!("Buffer size of `action_status` stream restricted to 1") + } + action_status.batch_size = 1; + streams_config.insert("action_status".to_owned(), action_status); + let mut streams = Streams::new(config.clone(), package_tx, metrics_tx); + streams.config_streams(streams_config); + + Self { + status_tx, + status_rx, + config, + actions_rx, + streams, + action_routes: HashMap::with_capacity(10), + action_redirections, + current_action: None, + parallel_actions: HashSet::new(), + shutdown_handle, + ctrl_rx, + ctrl_tx, + } + } + + pub fn register_action_route( + &mut self, + ActionRoute { name, timeout: duration }: ActionRoute, + actions_tx: Sender, + ) -> Result<(), Error> { + let action_router = ActionRouter { actions_tx, duration }; + if self.action_routes.insert(name.clone(), action_router).is_some() { + return Err(Error::ActionRouteClash(name)); + } + + Ok(()) + } + + pub fn register_action_routes, V: IntoIterator>( + &mut self, + routes: V, + actions_tx: Sender, + ) -> Result<(), Error> { + for route in routes { + self.register_action_route(route.into(), actions_tx.clone())?; + } + + Ok(()) + } + + /// Handle to send action status messages from connected application + pub fn status_tx(&self) -> StatusTx { + StatusTx { inner: self.status_tx.clone() } + } + + /// Handle to send action lane control messages + pub fn ctrl_tx(&self) -> CtrlTx { + CtrlTx { inner: self.ctrl_tx.clone() } + } + + fn clear_current_action(&mut self) { + self.current_action.take(); + } + + pub async fn start(&mut self) -> Result<(), Error> { + let mut metrics_timeout = interval(self.config.stream_metrics.timeout); + let mut end: Pin> = Box::pin(time::sleep(Duration::from_secs(u64::MAX))); + self.load_saved_action()?; + + loop { + select! { + action = self.actions_rx.recv_async() => { + let action = action?; + self.handle_action(action).await; + } + response = self.status_rx.recv_async() => { + let response = response?; + self.forward_action_response(response).await; + } + _ = &mut self.current_action.as_mut().map(|a| &mut a.timeout).unwrap_or(&mut end) => { + let action = self.current_action.take().unwrap(); + error!("Timeout waiting for action response. Action ID = {}", action.id); + self.forward_action_error(action.action, Error::ActionTimeout).await; + + // Remove action because it timedout + self.clear_current_action() + } + // Flush streams that timeout + Some(timedout_stream) = self.streams.stream_timeouts.next(), if self.streams.stream_timeouts.has_pending() => { + debug!("Flushing stream = {}", timedout_stream); + if let Err(e) = self.streams.flush_stream(&timedout_stream).await { + error!("Failed to flush stream = {}. Error = {}", timedout_stream, e); + } + } + // Flush all metrics when timed out + _ = metrics_timeout.tick() => { + if let Err(e) = self.streams.check_and_flush_metrics() { + debug!("Failed to flush stream metrics. Error = {}", e); + } + } + // Handle a shutdown signal + _ = self.ctrl_rx.recv_async() => { + if let Err(e) = self.save_current_action() { + error!("Failed to save current action: {e}"); + } + // NOTE: there might be events still waiting for recv on bridge_rx + self.shutdown_handle.send(()).unwrap(); + + return Ok(()) + } + } + } + } + + async fn handle_action(&mut self, action: Action) { + let action_id = action.action_id.clone(); + // Reactlabs setup processes logs generated by uplink + info!("Received action: {:?}", action); + + if let Some(current_action) = &self.current_action { + if action.name != TUNSHELL_ACTION { + warn!( + "Another action is currently occupying uplink; action_id = {}", + current_action.id + ); + self.forward_action_error(action, Error::Busy).await; + return; + } + } + + // NOTE: Don't do any blocking operations here + // TODO: Remove blocking here. Audit all blocking functions here + let error = match self.try_route_action(action.clone()) { + Ok(_) => { + let response = ActionResponse::progress(&action_id, "Received", 0); + self.forward_action_response(response).await; + return; + } + Err(e) => e, + }; + + // Remove action because it couldn't be routed + self.clear_current_action(); + + // Ignore sending failure status to backend. This makes + // backend retry action. + // + // TODO: Do we need this? Shouldn't backend have an easy way to + // retry failed actions in bulk? + if self.config.ignore_actions_if_no_clients { + error!("No clients connected, ignoring action = {:?}", action_id); + return; + } + + error!("Failed to route action to app. Error = {:?}", error); + self.forward_action_error(action, error).await; + } + + /// Save current action information in persistence + fn save_current_action(&mut self) -> Result<(), Error> { + let current_action = match self.current_action.take() { + Some(c) => c, + None => return Ok(()), + }; + let mut path = self.config.persistence_path.clone(); + path.push("current_action"); + info!("Storing current action in persistence; path: {}", path.display()); + current_action.write_to_disk(path)?; + + Ok(()) + } + + /// Load a saved action from persistence, performed on startup + fn load_saved_action(&mut self) -> Result<(), Error> { + let mut path = self.config.persistence_path.clone(); + path.push("current_action"); + + if path.is_file() { + let current_action = CurrentAction::read_from_disk(path)?; + info!("Loading saved action from persistence; action_id: {}", current_action.id); + self.current_action = Some(current_action) + } + + Ok(()) + } + + /// Handle received actions + fn try_route_action(&mut self, action: Action) -> Result<(), Error> { + let route = self + .action_routes + .get(&action.name) + .ok_or_else(|| Error::NoRoute(action.name.clone()))?; + + let deadline = route.try_send(action.clone()).map_err(|_| Error::UnresponsiveReceiver)?; + // current action left unchanged in case of new tunshell action + if action.name == TUNSHELL_ACTION { + self.parallel_actions.insert(action.action_id); + return Ok(()); + } + + self.current_action = Some(CurrentAction::new(action, deadline)); + + Ok(()) + } + + async fn forward_action_response(&mut self, mut response: ActionResponse) { + if self.parallel_actions.contains(&response.action_id) { + self.forward_parallel_action_response(response).await; + + return; + } + + let inflight_action = match &mut self.current_action { + Some(v) => v, + None => { + error!("Action timed out already/not present, ignoring response: {:?}", response); + return; + } + }; + + if *inflight_action.id != response.action_id { + error!("response id({}) != active action({})", response.action_id, inflight_action.id); + return; + } + + info!("Action response = {:?}", response); + self.streams.forward(response.clone()).await; + + if response.is_completed() || response.is_failed() { + self.clear_current_action(); + return; + } + + // Forward actions included in the config to the appropriate forward route, when + // they have reached 100% progress but haven't been marked as "Completed"/"Finished". + if response.is_done() { + let mut action = inflight_action.action.clone(); + + if let Some(a) = response.done_response.take() { + action = a; + } + + if let Err(RedirectionError(action)) = self.redirect_action(action).await { + // NOTE: send success reponse for actions that don't have redirections configured + warn!("Action redirection is not configured for: {:?}", action); + let response = ActionResponse::success(&action.action_id); + self.streams.forward(response).await; + + self.clear_current_action(); + } + } + } + + async fn redirect_action(&mut self, mut action: Action) -> Result<(), RedirectionError> { + let fwd_name = self + .action_redirections + .get(&action.name) + .ok_or_else(|| RedirectionError(action.clone()))?; + + debug!( + "Redirecting action: {} ~> {}; action_id = {}", + action.name, fwd_name, action.action_id, + ); + + fwd_name.clone_into(&mut action.name); + + if let Err(e) = self.try_route_action(action.clone()) { + error!("Failed to route action to app. Error = {:?}", e); + self.forward_action_error(action, e).await; + + // Remove action because it couldn't be forwarded + self.clear_current_action() + } + + Ok(()) + } + + async fn forward_parallel_action_response(&mut self, response: ActionResponse) { + info!("Action response = {:?}", response); + if response.is_completed() || response.is_failed() { + self.parallel_actions.remove(&response.action_id); + } + + self.streams.forward(response).await; + } + + async fn forward_action_error(&mut self, action: Action, error: Error) { + let response = ActionResponse::failure(&action.action_id, error.to_string()); + + self.streams.forward(response).await; + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct SaveAction { + pub id: String, + pub action: Action, + pub timeout: Duration, +} + +struct CurrentAction { + pub id: String, + pub action: Action, + pub timeout: Pin>, +} + +impl CurrentAction { + pub fn new(action: Action, deadline: Instant) -> CurrentAction { + CurrentAction { + id: action.action_id.clone(), + action, + timeout: Box::pin(time::sleep_until(deadline)), + } + } + + pub fn write_to_disk(self, path: PathBuf) -> Result<(), Error> { + let timeout = self.timeout.as_ref().deadline() - Instant::now(); + let save_action = SaveAction { id: self.id, action: self.action, timeout }; + let json = serde_json::to_string(&save_action)?; + fs::write(path, json)?; + + Ok(()) + } + + pub fn read_from_disk(path: PathBuf) -> Result { + let read = fs::read(&path)?; + let json: SaveAction = serde_json::from_slice(&read)?; + fs::remove_file(path)?; + + Ok(CurrentAction { + id: json.id, + action: json.action, + timeout: Box::pin(time::sleep(json.timeout)), + }) + } +} + +#[derive(Debug)] +pub struct ActionRouter { + pub(crate) actions_tx: Sender, + duration: Duration, +} + +impl ActionRouter { + #[allow(clippy::result_large_err)] + /// Forwards action to the appropriate application and returns the instance in time at which it should be timedout if incomplete + pub fn try_send(&self, mut action: Action) -> Result> { + let deadline = Instant::now() + self.duration; + action.deadline = Some(deadline); + self.actions_tx.try_send(action)?; + + Ok(deadline) + } +} + +/// Handle for apps to send action status to bridge +#[derive(Debug, Clone)] +pub struct StatusTx { + pub(crate) inner: Sender, +} + +impl StatusTx { + pub async fn send_action_response(&self, response: ActionResponse) { + self.inner.send_async(response).await.unwrap() + } +} + +/// Handle to send control messages to action lane +#[derive(Debug, Clone)] +pub struct CtrlTx { + pub(crate) inner: Sender, +} + +impl CtrlTx { + /// Triggers shutdown of `bridge::actions_lane` + pub async fn trigger_shutdown(&self) { + self.inner.send_async(ActionBridgeShutdown).await.unwrap() + } +} + +#[cfg(test)] +mod tests { + use tokio::runtime::Runtime; + + use crate::config::{StreamConfig, StreamMetricsConfig}; + + use super::*; + + fn default_config() -> Config { + Config { + stream_metrics: StreamMetricsConfig { + enabled: false, + timeout: Duration::from_secs(10), + ..Default::default() + }, + action_status: StreamConfig { + flush_period: Duration::from_secs(2), + ..Default::default() + }, + ..Default::default() + } + } + + fn create_bridge( + config: Arc, + ) -> (ActionsBridge, Sender, Receiver>) { + let (data_tx, data_rx) = bounded(10); + let (actions_tx, actions_rx) = bounded(10); + let (shutdown_handle, _) = bounded(1); + let (metrics_tx, _) = bounded(1); + let bridge = ActionsBridge::new(config, data_tx, actions_rx, shutdown_handle, metrics_tx); + + (bridge, actions_tx, data_rx) + } + + fn spawn_bridge(mut bridge: ActionsBridge) { + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + rt.block_on(async { bridge.start().await.unwrap() }); + }); + } + + struct Responses { + rx: Receiver>, + responses: Vec, + } + + impl Responses { + fn next(&mut self) -> ActionResponse { + if self.responses.is_empty() { + let status = self.rx.recv().unwrap().serialize().unwrap(); + self.responses = serde_json::from_slice(&status).unwrap(); + } + + self.responses.remove(0) + } + } + + #[tokio::test] + async fn timeout_on_diff_routes() { + let tmpdir = tempdir::TempDir::new("bridge").unwrap(); + std::env::set_current_dir(&tmpdir).unwrap(); + let config = Arc::new(default_config()); + let (mut bridge, actions_tx, data_rx) = create_bridge(config); + let route_1 = ActionRoute { name: "route_1".to_string(), timeout: Duration::from_secs(10) }; + + let (route_tx, route_1_rx) = bounded(1); + bridge.register_action_route(route_1, route_tx).unwrap(); + + let (route_tx, route_2_rx) = bounded(1); + let route_2 = ActionRoute { name: "route_2".to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(route_2, route_tx).unwrap(); + + spawn_bridge(bridge); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + loop { + select! { + action = route_1_rx.recv_async() => { + let action = action.unwrap(); + assert_eq!(action.action_id, "1".to_owned()); + } + + action = route_2_rx.recv_async() => { + let action = action.unwrap(); + assert_eq!(action.action_id, "2".to_owned()); + } + } + } + }); + }); + + std::thread::sleep(Duration::from_secs(1)); + + let action_1 = Action { + action_id: "1".to_string(), + name: "route_1".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action_1).unwrap(); + + let mut responses = Responses { rx: data_rx, responses: vec![] }; + + let status = responses.next(); + assert_eq!(status.state, "Received".to_owned()); + let start = status.timestamp; + + let status = responses.next(); + // verify response is timeout failure + assert!(status.is_failed()); + assert_eq!(status.action_id, "1".to_owned()); + assert_eq!(status.errors, ["Action timedout"]); + let elapsed = status.timestamp - start; + // verify timeout in 10s + assert_eq!(elapsed / 1000, 10); + + let action_2 = Action { + action_id: "2".to_string(), + name: "route_2".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action_2).unwrap(); + + let status = responses.next(); + assert_eq!(status.state, "Received".to_owned()); + let start = status.timestamp; + + let status = responses.next(); + // verify response is timeout failure + assert!(status.is_failed()); + assert_eq!(status.action_id, "2".to_owned()); + assert_eq!(status.errors, ["Action timedout"]); + let elapsed = status.timestamp - start; + // verify timeout in 30s + assert_eq!(elapsed / 1000, 30); + } + + #[tokio::test] + async fn recv_action_while_current_action_exists() { + let tmpdir = tempdir::TempDir::new("bridge").unwrap(); + std::env::set_current_dir(&tmpdir).unwrap(); + let config = Arc::new(default_config()); + let (mut bridge, actions_tx, data_rx) = create_bridge(config); + + let test_route = ActionRoute { name: "test".to_string(), timeout: Duration::from_secs(30) }; + + let (route_tx, action_rx) = bounded(1); + bridge.register_action_route(test_route, route_tx).unwrap(); + + spawn_bridge(bridge); + + std::thread::spawn(move || loop { + let action = action_rx.recv().unwrap(); + assert_eq!(action.action_id, "1".to_owned()); + }); + + std::thread::sleep(Duration::from_secs(1)); + + let action_1 = Action { + action_id: "1".to_string(), + name: "test".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action_1).unwrap(); + + let mut responses = Responses { rx: data_rx, responses: vec![] }; + + let status = responses.next(); + assert_eq!(status.action_id, "1".to_owned()); + assert_eq!(status.state, "Received".to_owned()); + + let action_2 = Action { + action_id: "2".to_string(), + name: "test".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action_2).unwrap(); + + let status = responses.next(); + // verify response is uplink occupied failure + assert!(status.is_failed()); + assert_eq!(status.action_id, "2".to_owned()); + assert_eq!(status.errors, ["Another action is currently being processed"]); + } + + #[tokio::test] + async fn complete_response_on_no_redirection() { + let tmpdir = tempdir::TempDir::new("bridge").unwrap(); + std::env::set_current_dir(&tmpdir).unwrap(); + let config = Arc::new(default_config()); + let (mut bridge, actions_tx, data_rx) = create_bridge(config); + + let test_route = ActionRoute { name: "test".to_string(), timeout: Duration::from_secs(30) }; + + let (route_tx, action_rx) = bounded(1); + bridge.register_action_route(test_route, route_tx).unwrap(); + let bridge_tx = bridge.status_tx(); + + spawn_bridge(bridge); + + std::thread::spawn(move || loop { + let action = action_rx.recv().unwrap(); + assert_eq!(action.action_id, "1".to_owned()); + std::thread::sleep(Duration::from_secs(1)); + let response = ActionResponse::progress("1", "Tested", 100); + Runtime::new().unwrap().block_on(bridge_tx.send_action_response(response)); + }); + + std::thread::sleep(Duration::from_secs(1)); + + let action = Action { + action_id: "1".to_string(), + name: "test".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action).unwrap(); + + let mut responses = Responses { rx: data_rx, responses: vec![] }; + + let status = responses.next(); + assert_eq!(status.state, "Received".to_owned()); + + let status = responses.next(); + assert!(status.is_done()); + assert_eq!(status.state, "Tested"); + + let status = responses.next(); + assert!(status.is_completed()); + } + + #[tokio::test] + async fn no_complete_response_between_redirection() { + let tmpdir = tempdir::TempDir::new("bridge").unwrap(); + std::env::set_current_dir(&tmpdir).unwrap(); + let mut config = default_config(); + config.action_redirections.insert("test".to_string(), "redirect".to_string()); + let (mut bridge, actions_tx, data_rx) = create_bridge(Arc::new(config)); + let bridge_tx_1 = bridge.status_tx(); + let bridge_tx_2 = bridge.status_tx(); + + let (route_tx, action_rx_1) = bounded(1); + let test_route = ActionRoute { name: "test".to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(test_route, route_tx).unwrap(); + + let (route_tx, action_rx_2) = bounded(1); + let redirect_route = + ActionRoute { name: "redirect".to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(redirect_route, route_tx).unwrap(); + + spawn_bridge(bridge); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let action = action_rx_1.recv().unwrap(); + assert_eq!(action.action_id, "1".to_owned()); + std::thread::sleep(Duration::from_secs(1)); + let response = ActionResponse::progress("1", "Tested", 100); + rt.block_on(bridge_tx_1.send_action_response(response)); + }); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let action = action_rx_2.recv().unwrap(); + assert_eq!(action.action_id, "1".to_owned()); + let response = ActionResponse::progress("1", "Redirected", 0); + rt.block_on(bridge_tx_2.send_action_response(response)); + std::thread::sleep(Duration::from_secs(1)); + let response = ActionResponse::success("1"); + rt.block_on(bridge_tx_2.send_action_response(response)); + }); + + std::thread::sleep(Duration::from_secs(1)); + + let action = Action { + action_id: "1".to_string(), + name: "test".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action).unwrap(); + + let mut responses = Responses { rx: data_rx, responses: vec![] }; + + let status = responses.next(); + assert_eq!(status.state, "Received".to_owned()); + + let status = responses.next(); + assert!(status.is_done()); + assert_eq!(status.state, "Tested"); + + let status = responses.next(); + assert!(!status.is_completed()); + assert_eq!(status.state, "Redirected"); + + let status = responses.next(); + assert!(status.is_completed()); + } + + #[tokio::test] + async fn accept_regular_actions_during_tunshell() { + let tmpdir = tempdir::TempDir::new("bridge").unwrap(); + std::env::set_current_dir(&tmpdir).unwrap(); + let config = default_config(); + let (mut bridge, actions_tx, data_rx) = create_bridge(Arc::new(config)); + let bridge_tx_1 = bridge.status_tx(); + let bridge_tx_2 = bridge.status_tx(); + + let (route_tx, action_rx_1) = bounded(1); + let tunshell_route = + ActionRoute { name: TUNSHELL_ACTION.to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(tunshell_route, route_tx).unwrap(); + + let (route_tx, action_rx_2) = bounded(1); + let test_route = ActionRoute { name: "test".to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(test_route, route_tx).unwrap(); + + spawn_bridge(bridge); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let action = action_rx_1.recv().unwrap(); + assert_eq!(action.action_id, "1"); + let response = ActionResponse::progress(&action.action_id, "Launched", 0); + rt.block_on(bridge_tx_1.send_action_response(response)); + std::thread::sleep(Duration::from_secs(3)); + let response = ActionResponse::success(&action.action_id); + rt.block_on(bridge_tx_1.send_action_response(response)); + }); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let action = action_rx_2.recv().unwrap(); + assert_eq!(action.action_id, "2"); + let response = ActionResponse::progress(&action.action_id, "Running", 0); + rt.block_on(bridge_tx_2.send_action_response(response)); + std::thread::sleep(Duration::from_secs(1)); + let response = ActionResponse::success(&action.action_id); + rt.block_on(bridge_tx_2.send_action_response(response)); + }); + + std::thread::sleep(Duration::from_secs(1)); + + let action = Action { + action_id: "1".to_string(), + name: "launch_shell".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action).unwrap(); + + std::thread::sleep(Duration::from_secs(1)); + + let action = Action { + action_id: "2".to_string(), + name: "test".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action).unwrap(); + + let mut responses = Responses { rx: data_rx, responses: vec![] }; + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "1"); + assert_eq!(state, "Received"); + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "1"); + assert_eq!(state, "Launched"); + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "2"); + assert_eq!(state, "Received"); + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "2"); + assert_eq!(state, "Running"); + + let status = responses.next(); + assert_eq!(status.action_id, "2"); + assert!(status.is_completed()); + + let status = responses.next(); + assert_eq!(status.action_id, "1"); + assert!(status.is_completed()); + } + + #[tokio::test] + async fn accept_tunshell_during_regular_action() { + let tmpdir = tempdir::TempDir::new("bridge").unwrap(); + std::env::set_current_dir(&tmpdir).unwrap(); + let config = default_config(); + let (mut bridge, actions_tx, data_rx) = create_bridge(Arc::new(config)); + let bridge_tx_1 = bridge.status_tx(); + let bridge_tx_2 = bridge.status_tx(); + + let (route_tx, action_rx_1) = bounded(1); + let test_route = ActionRoute { name: "test".to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(test_route, route_tx).unwrap(); + + let (route_tx, action_rx_2) = bounded(1); + let tunshell_route = + ActionRoute { name: TUNSHELL_ACTION.to_string(), timeout: Duration::from_secs(30) }; + bridge.register_action_route(tunshell_route, route_tx).unwrap(); + + spawn_bridge(bridge); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let action = action_rx_1.recv().unwrap(); + assert_eq!(action.action_id, "1"); + let response = ActionResponse::progress(&action.action_id, "Running", 0); + rt.block_on(bridge_tx_1.send_action_response(response)); + std::thread::sleep(Duration::from_secs(3)); + let response = ActionResponse::success(&action.action_id); + rt.block_on(bridge_tx_1.send_action_response(response)); + }); + + std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let action = action_rx_2.recv().unwrap(); + assert_eq!(action.action_id, "2"); + let response = ActionResponse::progress(&action.action_id, "Launched", 0); + rt.block_on(bridge_tx_2.send_action_response(response)); + std::thread::sleep(Duration::from_secs(1)); + let response = ActionResponse::success(&action.action_id); + rt.block_on(bridge_tx_2.send_action_response(response)); + }); + + std::thread::sleep(Duration::from_secs(1)); + + let action = Action { + action_id: "1".to_string(), + name: "test".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action).unwrap(); + + std::thread::sleep(Duration::from_secs(1)); + + let action = Action { + action_id: "2".to_string(), + name: "launch_shell".to_string(), + payload: "test".to_string(), + deadline: None, + }; + actions_tx.send(action).unwrap(); + + let mut responses = Responses { rx: data_rx, responses: vec![] }; + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "1"); + assert_eq!(state, "Received"); + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "1"); + assert_eq!(state, "Running"); + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "2"); + assert_eq!(state, "Received"); + + let ActionResponse { action_id, state, .. } = responses.next(); + assert_eq!(action_id, "2"); + assert_eq!(state, "Launched"); + + let status = responses.next(); + assert_eq!(status.action_id, "2"); + assert!(status.is_completed()); + + let status = responses.next(); + assert_eq!(status.action_id, "1"); + assert!(status.is_completed()); + } +} diff --git a/uplink/src/base/bridge/data_lane.rs b/uplink/src/base/bridge/data_lane.rs new file mode 100644 index 000000000..3101d897b --- /dev/null +++ b/uplink/src/base/bridge/data_lane.rs @@ -0,0 +1,115 @@ +use std::sync::Arc; + +use flume::{bounded, Receiver, RecvError, Sender}; +use log::{debug, error}; +use tokio::{select, time::interval}; + +use crate::Config; + +use super::{streams::Streams, DataBridgeShutdown, Package, Payload, StreamMetrics}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Receiver error {0}")] + Recv(#[from] RecvError), +} + +pub struct DataBridge { + /// All configuration + config: Arc, + /// Tx handle to give to apps + data_tx: Sender, + /// Rx to receive data from apps + data_rx: Receiver, + /// Handle to send data over streams + streams: Streams, + ctrl_rx: Receiver, + ctrl_tx: Sender, +} + +impl DataBridge { + pub fn new( + config: Arc, + package_tx: Sender>, + metrics_tx: Sender, + ) -> Self { + let (data_tx, data_rx) = bounded(10); + let (ctrl_tx, ctrl_rx) = bounded(1); + + let mut streams = Streams::new(config.clone(), package_tx, metrics_tx); + streams.config_streams(config.streams.clone()); + + Self { data_tx, data_rx, config, streams, ctrl_rx, ctrl_tx } + } + + /// Handle to send data points from source application + pub fn data_tx(&self) -> DataTx { + DataTx { inner: self.data_tx.clone() } + } + + /// Handle to send data lane control message + pub fn ctrl_tx(&self) -> CtrlTx { + CtrlTx { inner: self.ctrl_tx.clone() } + } + + pub async fn start(&mut self) -> Result<(), Error> { + let mut metrics_timeout = interval(self.config.stream_metrics.timeout); + + loop { + select! { + data = self.data_rx.recv_async() => { + let data = data?; + self.streams.forward(data).await; + } + // Flush streams that timeout + Some(timedout_stream) = self.streams.stream_timeouts.next(), if self.streams.stream_timeouts.has_pending() => { + debug!("Flushing stream = {}", timedout_stream); + if let Err(e) = self.streams.flush_stream(&timedout_stream).await { + error!("Failed to flush stream = {}. Error = {}", timedout_stream, e); + } + } + // Flush all metrics when timed out + _ = metrics_timeout.tick() => { + if let Err(e) = self.streams.check_and_flush_metrics() { + debug!("Failed to flush stream metrics. Error = {}", e); + } + } + // Handle a shutdown signal + _ = self.ctrl_rx.recv_async() => { + self.streams.flush_all().await; + + return Ok(()) + } + } + } + } +} + +/// Handle for apps to send action status to bridge +#[derive(Debug, Clone)] +pub struct DataTx { + pub(crate) inner: Sender, +} + +impl DataTx { + pub async fn send_payload(&self, payload: Payload) { + self.inner.send_async(payload).await.unwrap() + } + + pub fn send_payload_sync(&self, payload: Payload) { + self.inner.send(payload).unwrap() + } +} + +/// Handle to send control messages to data lane +#[derive(Debug, Clone)] +pub struct CtrlTx { + pub(crate) inner: Sender, +} + +impl CtrlTx { + /// Triggers shutdown of `bridge::data_lane` + pub async fn trigger_shutdown(&self) { + self.inner.send_async(DataBridgeShutdown).await.unwrap() + } +} diff --git a/uplink/src/base/bridge/mod.rs b/uplink/src/base/bridge/mod.rs index 4c50cae9f..118775113 100644 --- a/uplink/src/base/bridge/mod.rs +++ b/uplink/src/base/bridge/mod.rs @@ -1,54 +1,34 @@ -use flume::{bounded, Receiver, RecvError, Sender, TrySendError}; -use log::{debug, error, info, warn}; +use flume::{bounded, Receiver, Sender}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tokio::select; -use tokio::time::{self, interval, Instant, Sleep}; -use std::collections::HashSet; -use std::fs; -use std::path::PathBuf; -use std::{collections::HashMap, fmt::Debug, pin::Pin, sync::Arc, time::Duration}; +use std::{fmt::Debug, sync::Arc}; +mod actions_lane; +mod data_lane; mod delaymap; mod metrics; pub(crate) mod stream; mod streams; -use super::Compression; -use crate::base::ActionRoute; +use actions_lane::{ActionsBridge, Error}; +pub use actions_lane::{CtrlTx as ActionsLaneCtrlTx, StatusTx}; +use data_lane::DataBridge; +pub use data_lane::{CtrlTx as DataLaneCtrlTx, DataTx}; + +use crate::config::{ActionRoute, StreamConfig}; use crate::{Action, ActionResponse, Config}; pub use metrics::StreamMetrics; -use streams::Streams; - -const TUNSHELL_ACTION: &str = "launch_shell"; -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Receiver error {0}")] - Recv(#[from] RecvError), - #[error("Io error {0}")] - Io(#[from] std::io::Error), - #[error("Serde error {0}")] - Serde(#[from] serde_json::Error), - #[error("Action receiver busy or down")] - UnresponsiveReceiver, - #[error("No route for action {0}")] - NoRoute(String), - #[error("Action timedout")] - ActionTimeout, - #[error("Another action is currently being processed")] - Busy, -} - -pub trait Point: Send + Debug { +pub trait Point: Send + Debug + Serialize + 'static { + fn stream_name(&self) -> &str; fn sequence(&self) -> u32; fn timestamp(&self) -> u64; } pub trait Package: Send + Debug { - fn topic(&self) -> Arc; - fn stream(&self) -> Arc; + fn stream_config(&self) -> Arc; + fn stream_name(&self) -> Arc; // TODO: Implement a generic Return type that can wrap // around custom serialization error types. fn serialize(&self) -> serde_json::Result>; @@ -58,7 +38,6 @@ pub trait Package: Send + Debug { fn is_empty(&self) -> bool { self.len() == 0 } - fn compression(&self) -> Compression; } // TODO Don't do any deserialization on payload. Read it a Vec which is in turn a json @@ -74,6 +53,10 @@ pub struct Payload { } impl Point for Payload { + fn stream_name(&self) -> &str { + &self.stream + } + fn sequence(&self) -> u32 { self.sequence } @@ -83,45 +66,15 @@ impl Point for Payload { } } -#[derive(Debug)] -pub enum Event { - /// App name and handle for brige to send actions to the app - RegisterActionRoute(String, ActionRouter), - /// Data sent by the app - Data(Payload), - /// Sometime apps can choose to directly send action response instead - /// sending in `Payload` form - ActionResponse(ActionResponse), -} +/// Commands that can be used to remotely trigger action_lane shutdown +pub(crate) struct ActionBridgeShutdown; -/// Commands that can be used to remotely control bridge -pub(crate) enum BridgeCtrl { - Shutdown, -} +/// Commands that can be used to remotely trigger data_lane shutdown +pub(crate) struct DataBridgeShutdown; pub struct Bridge { - /// All configuration - config: Arc, - /// Tx handle to give to apps - bridge_tx: Sender, - /// Rx to receive events from apps - bridge_rx: Receiver, - /// Handle to send data over streams - streams: Streams, - /// Actions incoming from backend - actions_rx: Receiver, - /// Apps registered with the bridge - /// NOTE: Sometimes action_routes could overlap, the latest route - /// to be registered will be used in such a circumstance. - action_routes: HashMap, - /// Action redirections - action_redirections: HashMap, - /// Current action that is being processed - current_action: Option, - parallel_actions: HashSet, - ctrl_rx: Receiver, - ctrl_tx: Sender, - shutdown_handle: Sender<()>, + pub(crate) data: DataBridge, + pub(crate) actions: ActionsBridge, } impl Bridge { @@ -131,801 +84,56 @@ impl Bridge { metrics_tx: Sender, actions_rx: Receiver, shutdown_handle: Sender<()>, - ) -> Bridge { - let (bridge_tx, bridge_rx) = bounded(10); - let action_redirections = config.action_redirections.clone(); - let (ctrl_tx, ctrl_rx) = bounded(1); - let streams = Streams::new(config.clone(), package_tx, metrics_tx); - - Bridge { - bridge_tx, - bridge_rx, - config, - actions_rx, - action_routes: HashMap::with_capacity(10), - action_redirections, - current_action: None, - parallel_actions: HashSet::new(), - shutdown_handle, - ctrl_rx, - ctrl_tx, - streams, - } - } - - pub fn tx(&mut self) -> BridgeTx { - BridgeTx { events_tx: self.bridge_tx.clone(), shutdown_handle: self.ctrl_tx.clone() } - } - - fn clear_current_action(&mut self) { - self.current_action.take(); - } - - pub async fn start(&mut self) -> Result<(), Error> { - let mut metrics_timeout = interval(Duration::from_secs(self.config.stream_metrics.timeout)); - let mut end: Pin> = Box::pin(time::sleep(Duration::from_secs(u64::MAX))); - self.load_saved_action()?; - - loop { - select! { - action = self.actions_rx.recv_async() => { - let action = action?; - let action_id = action.action_id.clone(); - // Reactlabs setup processes logs generated by uplink - info!("Received action: {:?}", action); - - if let Some(current_action) = &self.current_action { - if action.name != TUNSHELL_ACTION { - warn!("Another action is currently occupying uplink; action_id = {}", current_action.id); - self.forward_action_error(action, Error::Busy).await; - continue - } - } - - // NOTE: Don't do any blocking operations here - // TODO: Remove blocking here. Audit all blocking functions here - let error = match self.try_route_action(action.clone()) { - Ok(_) => { - let response = ActionResponse::progress(&action_id, "Received", 0); - self.forward_action_response(response).await; - continue; - } - Err(e) => e, - }; - // Ignore sending failure status to backend. This makes - // backend retry action. - // - // TODO: Do we need this? Shouldn't backend have an easy way to - // retry failed actions in bulk? - if self.config.ignore_actions_if_no_clients { - error!("No clients connected, ignoring action = {:?}", action_id); - self.current_action = None; - continue; - } - - error!("Failed to route action to app. Error = {:?}", error); - self.forward_action_error(action, error).await; - - // Remove action because it couldn't be routed - self.clear_current_action() - } - event = self.bridge_rx.recv_async() => { - let event = event?; - match event { - Event::RegisterActionRoute(name, tx) => { - self.action_routes.insert(name, tx); - } - Event::Data(v) => { - self.streams.forward(v).await; - } - Event::ActionResponse(response) => { - self.forward_action_response(response).await; - } - } - } - _ = &mut self.current_action.as_mut().map(|a| &mut a.timeout).unwrap_or(&mut end) => { - let action = self.current_action.take().unwrap(); - error!("Timeout waiting for action response. Action ID = {}", action.id); - self.forward_action_error(action.action, Error::ActionTimeout).await; - - // Remove action because it timedout - self.clear_current_action() - } - // Flush streams that timeout - Some(timedout_stream) = self.streams.stream_timeouts.next(), if self.streams.stream_timeouts.has_pending() => { - debug!("Flushing stream = {}", timedout_stream); - if let Err(e) = self.streams.flush_stream(&timedout_stream).await { - error!("Failed to flush stream = {}. Error = {}", timedout_stream, e); - } - } - // Flush all metrics when timed out - _ = metrics_timeout.tick() => { - if let Err(e) = self.streams.check_and_flush_metrics() { - debug!("Failed to flush stream metrics. Error = {}", e); - } - } - // Handle a shutdown signal - _ = self.ctrl_rx.recv_async() => { - if let Err(e) = self.save_current_action() { - error!("Failed to save current action: {e}"); - } - self.streams.flush_all().await; - // NOTE: there might be events still waiting for recv on bridge_rx - self.shutdown_handle.send(()).unwrap(); - - return Ok(()) - } - } - } - } - - /// Save current action information in persistence - fn save_current_action(&mut self) -> Result<(), Error> { - let current_action = match self.current_action.take() { - Some(c) => c, - None => return Ok(()), - }; - let mut path = self.config.persistence_path.clone(); - path.push("current_action"); - info!("Storing current action in persistence; path: {}", path.display()); - current_action.write_to_disk(path)?; - - Ok(()) - } - - /// Load a saved action from persistence, performed on startup - fn load_saved_action(&mut self) -> Result<(), Error> { - let mut path = self.config.persistence_path.clone(); - path.push("current_action"); - - if path.is_file() { - let current_action = CurrentAction::read_from_disk(path)?; - info!("Loading saved action from persistence; action_id: {}", current_action.id); - self.current_action = Some(current_action) - } - - Ok(()) - } - - /// Handle received actions - fn try_route_action(&mut self, action: Action) -> Result<(), Error> { - match self.action_routes.get(&action.name) { - Some(route) => { - let duration = - route.try_send(action.clone()).map_err(|_| Error::UnresponsiveReceiver)?; - // current action left unchanged in case of forwarded action bein - if action.name == TUNSHELL_ACTION { - self.parallel_actions.insert(action.action_id); - return Ok(()); - } - - self.current_action = Some(CurrentAction::new(action, duration)); - - Ok(()) - } - None => Err(Error::NoRoute(action.name)), - } - } - - async fn forward_action_response(&mut self, response: ActionResponse) { - if self.parallel_actions.contains(&response.action_id) { - self.forward_parallel_action_response(response).await; - - return; - } - - let inflight_action = match &mut self.current_action { - Some(v) => v, - None => { - error!("Action timed out already/not present, ignoring response: {:?}", response); - return; - } - }; - - if *inflight_action.id != response.action_id { - error!("response id({}) != active action({})", response.action_id, inflight_action.id); - return; - } - - info!("Action response = {:?}", response); - self.streams.forward(response.as_payload()).await; - - if response.is_completed() || response.is_failed() { - self.clear_current_action(); - return; - } - - // Forward actions included in the config to the appropriate forward route, when - // they have reached 100% progress but haven't been marked as "Completed"/"Finished". - if response.is_done() { - let fwd_name = match self.action_redirections.get(&inflight_action.action.name) { - Some(n) => n, - None => { - // NOTE: send success reponse for actions that don't have redirections configured - warn!("Action redirection for {} not configured", inflight_action.action.name); - let response = ActionResponse::success(&inflight_action.id); - self.streams.forward(response.as_payload()).await; - - self.clear_current_action(); - return; - } - }; - - if let Some(action) = response.done_response { - inflight_action.action = action; - } - - let mut fwd_action = inflight_action.action.clone(); - fwd_action.name = fwd_name.to_owned(); - - if let Err(e) = self.try_route_action(fwd_action.clone()) { - error!("Failed to route action to app. Error = {:?}", e); - self.forward_action_error(fwd_action, e).await; - - // Remove action because it couldn't be forwarded - self.clear_current_action() - } - } - } - - async fn forward_parallel_action_response(&mut self, response: ActionResponse) { - info!("Action response = {:?}", response); - self.streams.forward(response.as_payload()).await; - - if response.is_completed() || response.is_failed() { - self.parallel_actions.remove(&response.action_id); - } - } - - async fn forward_action_error(&mut self, action: Action, error: Error) { - let response = ActionResponse::failure(&action.action_id, error.to_string()); - - self.streams.forward(response.as_payload()).await; + ) -> Self { + let data = DataBridge::new(config.clone(), package_tx.clone(), metrics_tx.clone()); + let actions = + ActionsBridge::new(config, package_tx, actions_rx, shutdown_handle, metrics_tx); + Self { data, actions } } -} - -#[derive(Debug, Deserialize, Serialize)] -struct SaveAction { - pub id: String, - pub action: Action, - pub timeout: Duration, -} -struct CurrentAction { - pub id: String, - pub action: Action, - pub timeout: Pin>, -} - -impl CurrentAction { - pub fn new(action: Action, duration: Duration) -> CurrentAction { - CurrentAction { - id: action.action_id.clone(), - action, - timeout: Box::pin(time::sleep(duration)), - } + /// Handle to send data/action status messages + pub fn bridge_tx(&self) -> BridgeTx { + BridgeTx { data_tx: self.data.data_tx(), status_tx: self.actions.status_tx() } } - pub fn write_to_disk(self, path: PathBuf) -> Result<(), Error> { - let timeout = self.timeout.as_ref().deadline() - Instant::now(); - let save_action = SaveAction { id: self.id, action: self.action, timeout }; - let json = serde_json::to_string(&save_action)?; - fs::write(path, json)?; - - Ok(()) + pub(crate) fn ctrl_tx(&self) -> (actions_lane::CtrlTx, data_lane::CtrlTx) { + (self.actions.ctrl_tx(), self.data.ctrl_tx()) } - pub fn read_from_disk(path: PathBuf) -> Result { - let read = fs::read(&path)?; - let json: SaveAction = serde_json::from_slice(&read)?; - fs::remove_file(path)?; + pub fn register_action_route(&mut self, route: ActionRoute) -> Result, Error> { + let (actions_tx, actions_rx) = bounded(1); + self.actions.register_action_route(route, actions_tx)?; - Ok(CurrentAction { - id: json.id, - action: json.action, - timeout: Box::pin(time::sleep(json.timeout)), - }) + Ok(actions_rx) } -} -#[derive(Debug)] -pub struct ActionRouter { - pub(crate) actions_tx: Sender, - duration: Duration, -} - -impl ActionRouter { - #[allow(clippy::result_large_err)] - pub fn try_send(&self, action: Action) -> Result> { - self.actions_tx.try_send(action)?; + pub fn register_action_routes, V: IntoIterator>( + &mut self, + routes: V, + ) -> Result, Error> { + let (actions_tx, actions_rx) = bounded(1); + self.actions.register_action_routes(routes, actions_tx)?; - Ok(self.duration) + Ok(actions_rx) } } #[derive(Debug, Clone)] pub struct BridgeTx { - // Handle for apps to send events to bridge - pub(crate) events_tx: Sender, - pub(crate) shutdown_handle: Sender, + pub data_tx: DataTx, + pub status_tx: StatusTx, } impl BridgeTx { - pub async fn register_action_route(&self, route: ActionRoute) -> Receiver { - let (actions_tx, actions_rx) = bounded(1); - let ActionRoute { name, timeout } = route; - let duration = Duration::from_secs(timeout); - let action_router = ActionRouter { actions_tx, duration }; - let event = Event::RegisterActionRoute(name, action_router); - - // Bridge should always be up and hence unwrap is ok - self.events_tx.send_async(event).await.unwrap(); - actions_rx - } - - pub async fn register_action_routes, V: IntoIterator>( - &self, - routes: V, - ) -> Option> { - let routes: Vec = routes.into_iter().map(|n| n.into()).collect(); - if routes.is_empty() { - return None; - } - - let (actions_tx, actions_rx) = bounded(1); - - for route in routes { - let ActionRoute { name, timeout } = route; - let duration = Duration::from_secs(timeout); - let action_router = ActionRouter { actions_tx: actions_tx.clone(), duration }; - let event = Event::RegisterActionRoute(name, action_router); - // Bridge should always be up and hence unwrap is ok - self.events_tx.send_async(event).await.unwrap(); - } - - Some(actions_rx) - } - pub async fn send_payload(&self, payload: Payload) { - let event = Event::Data(payload); - self.events_tx.send_async(event).await.unwrap() + self.data_tx.send_payload(payload).await } pub fn send_payload_sync(&self, payload: Payload) { - let event = Event::Data(payload); - self.events_tx.send(event).unwrap() + self.data_tx.send_payload_sync(payload) } pub async fn send_action_response(&self, response: ActionResponse) { - let event = Event::ActionResponse(response); - self.events_tx.send_async(event).await.unwrap() - } - - pub async fn trigger_shutdown(&self) { - self.shutdown_handle.send_async(BridgeCtrl::Shutdown).await.unwrap() - } -} - -#[cfg(test)] -mod tests { - use std::{sync::Arc, time::Duration}; - - use flume::{bounded, Receiver, Sender}; - use tokio::{runtime::Runtime, select}; - - use crate::{ - base::{ActionRoute, StreamMetricsConfig}, - Action, ActionResponse, Config, - }; - - use super::*; - - fn default_config() -> Config { - Config { - stream_metrics: StreamMetricsConfig { - enabled: false, - timeout: 10, - ..Default::default() - }, - ..Default::default() - } - } - - fn start_bridge(config: Arc) -> (BridgeTx, Sender, Receiver>) { - let (package_tx, package_rx) = bounded(10); - let (metrics_tx, _) = bounded(10); - let (actions_tx, actions_rx) = bounded(10); - let (shutdown_handle, _) = bounded(1); - - let mut bridge = Bridge::new(config, package_tx, metrics_tx, actions_rx, shutdown_handle); - let bridge_tx = bridge.tx(); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - rt.block_on(async { bridge.start().await.unwrap() }); - }); - - (bridge_tx, actions_tx, package_rx) - } - - fn recv_response(package_rx: &Receiver>) -> ActionResponse { - let status = package_rx.recv().unwrap().serialize().unwrap(); - let status: Vec = serde_json::from_slice(&status).unwrap(); - status[0].clone() - } - - #[tokio::test] - async fn timeout_on_diff_routes() { - let tmpdir = tempdir::TempDir::new("bridge").unwrap(); - std::env::set_current_dir(&tmpdir).unwrap(); - let config = Arc::new(default_config()); - let (bridge_tx, actions_tx, package_rx) = start_bridge(config); - let route_1 = ActionRoute { name: "route_1".to_string(), timeout: 10 }; - let route_1_rx = bridge_tx.register_action_route(route_1).await; - - let route_2 = ActionRoute { name: "route_2".to_string(), timeout: 30 }; - let route_2_rx = bridge_tx.register_action_route(route_2).await; - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - loop { - select! { - action = route_1_rx.recv_async() => { - let action = action.unwrap(); - assert_eq!(action.action_id, "1".to_owned()); - } - - action = route_2_rx.recv_async() => { - let action = action.unwrap(); - assert_eq!(action.action_id, "2".to_owned()); - } - } - } - }); - }); - - std::thread::sleep(Duration::from_secs(1)); - - let action_1 = Action { - action_id: "1".to_string(), - kind: "test".to_string(), - name: "route_1".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action_1).unwrap(); - - let status = recv_response(&package_rx); - assert_eq!(status.state, "Received".to_owned()); - let start = status.timestamp; - - let status = recv_response(&package_rx); - // verify response is timeout failure - assert!(status.is_failed()); - assert_eq!(status.action_id, "1".to_owned()); - assert_eq!(status.errors, ["Action timedout"]); - let elapsed = status.timestamp - start; - // verify timeout in 10s - assert_eq!(elapsed / 1000, 10); - - let action_2 = Action { - action_id: "2".to_string(), - kind: "test".to_string(), - name: "route_2".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action_2).unwrap(); - - let status = recv_response(&package_rx); - assert_eq!(status.state, "Received".to_owned()); - let start = status.timestamp; - - let status = recv_response(&package_rx); - // verify response is timeout failure - assert!(status.is_failed()); - assert_eq!(status.action_id, "2".to_owned()); - assert_eq!(status.errors, ["Action timedout"]); - let elapsed = status.timestamp - start; - // verify timeout in 30s - assert_eq!(elapsed / 1000, 30); - } - - #[tokio::test] - async fn recv_action_while_current_action_exists() { - let tmpdir = tempdir::TempDir::new("bridge").unwrap(); - std::env::set_current_dir(&tmpdir).unwrap(); - let config = Arc::new(default_config()); - let (bridge_tx, actions_tx, package_rx) = start_bridge(config); - - let test_route = ActionRoute { name: "test".to_string(), timeout: 30 }; - let action_rx = bridge_tx.register_action_route(test_route).await; - - std::thread::spawn(move || loop { - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "1".to_owned()); - }); - - std::thread::sleep(Duration::from_secs(1)); - - let action_1 = Action { - action_id: "1".to_string(), - kind: "test".to_string(), - name: "test".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action_1).unwrap(); - - let status = recv_response(&package_rx); - assert_eq!(status.action_id, "1".to_owned()); - assert_eq!(status.state, "Received".to_owned()); - - let action_2 = Action { - action_id: "2".to_string(), - kind: "test".to_string(), - name: "test".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action_2).unwrap(); - - let status = recv_response(&package_rx); - // verify response is uplink occupied failure - assert!(status.is_failed()); - assert_eq!(status.action_id, "2".to_owned()); - assert_eq!(status.errors, ["Another action is currently being processed"]); - } - - #[tokio::test] - async fn complete_response_on_no_redirection() { - let tmpdir = tempdir::TempDir::new("bridge").unwrap(); - std::env::set_current_dir(&tmpdir).unwrap(); - let config = Arc::new(default_config()); - let (bridge_tx, actions_tx, package_rx) = start_bridge(config); - - let test_route = ActionRoute { name: "test".to_string(), timeout: 30 }; - let action_rx = bridge_tx.register_action_route(test_route).await; - - std::thread::spawn(move || loop { - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "1".to_owned()); - std::thread::sleep(Duration::from_secs(1)); - let response = ActionResponse::progress("1", "Tested", 100); - Runtime::new().unwrap().block_on(bridge_tx.send_action_response(response)); - }); - - std::thread::sleep(Duration::from_secs(1)); - - let action = Action { - action_id: "1".to_string(), - kind: "test".to_string(), - name: "test".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action).unwrap(); - - let status = recv_response(&package_rx); - assert_eq!(status.state, "Received".to_owned()); - - let status = recv_response(&package_rx); - assert!(status.is_done()); - assert_eq!(status.state, "Tested"); - - let status = recv_response(&package_rx); - assert!(status.is_completed()); - } - - #[tokio::test] - async fn no_complete_response_between_redirection() { - let tmpdir = tempdir::TempDir::new("bridge").unwrap(); - std::env::set_current_dir(&tmpdir).unwrap(); - let mut config = default_config(); - config.action_redirections.insert("test".to_string(), "redirect".to_string()); - let (bridge_tx, actions_tx, package_rx) = start_bridge(Arc::new(config)); - let bridge_tx_clone = bridge_tx.clone(); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - let test_route = ActionRoute { name: "test".to_string(), timeout: 30 }; - let action_rx = rt.block_on(bridge_tx.register_action_route(test_route)); - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "1".to_owned()); - std::thread::sleep(Duration::from_secs(1)); - let response = ActionResponse::progress("1", "Tested", 100); - rt.block_on(bridge_tx.send_action_response(response)); - }); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - let test_route = ActionRoute { name: "redirect".to_string(), timeout: 30 }; - let action_rx = rt.block_on(bridge_tx_clone.register_action_route(test_route)); - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "1".to_owned()); - let response = ActionResponse::progress("1", "Redirected", 0); - rt.block_on(bridge_tx_clone.send_action_response(response)); - std::thread::sleep(Duration::from_secs(1)); - let response = ActionResponse::success("1"); - rt.block_on(bridge_tx_clone.send_action_response(response)); - }); - - std::thread::sleep(Duration::from_secs(1)); - - let action = Action { - action_id: "1".to_string(), - kind: "test".to_string(), - name: "test".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action).unwrap(); - - let status = recv_response(&package_rx); - assert_eq!(status.state, "Received".to_owned()); - - let status = recv_response(&package_rx); - assert!(status.is_done()); - assert_eq!(status.state, "Tested"); - - let status = recv_response(&package_rx); - assert!(!status.is_completed()); - assert_eq!(status.state, "Redirected"); - - let status = recv_response(&package_rx); - assert!(status.is_completed()); - } - - #[tokio::test] - async fn accept_regular_actions_during_tunshell() { - let tmpdir = tempdir::TempDir::new("bridge").unwrap(); - std::env::set_current_dir(&tmpdir).unwrap(); - let config = default_config(); - let (bridge_tx, actions_tx, package_rx) = start_bridge(Arc::new(config)); - let bridge_tx_clone = bridge_tx.clone(); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - let tunshell_route = ActionRoute { name: TUNSHELL_ACTION.to_string(), timeout: 30 }; - let action_rx = rt.block_on(bridge_tx.register_action_route(tunshell_route)); - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "1"); - let response = ActionResponse::progress(&action.action_id, "Launched", 0); - rt.block_on(bridge_tx.send_action_response(response)); - std::thread::sleep(Duration::from_secs(3)); - let response = ActionResponse::success(&action.action_id); - rt.block_on(bridge_tx.send_action_response(response)); - }); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - let test_route = ActionRoute { name: "test".to_string(), timeout: 30 }; - let action_rx = rt.block_on(bridge_tx_clone.register_action_route(test_route)); - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "2"); - let response = ActionResponse::progress(&action.action_id, "Running", 0); - rt.block_on(bridge_tx_clone.send_action_response(response)); - std::thread::sleep(Duration::from_secs(1)); - let response = ActionResponse::success(&action.action_id); - rt.block_on(bridge_tx_clone.send_action_response(response)); - }); - - std::thread::sleep(Duration::from_secs(1)); - - let action = Action { - action_id: "1".to_string(), - kind: "tunshell".to_string(), - name: "launch_shell".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action).unwrap(); - - std::thread::sleep(Duration::from_secs(1)); - - let action = Action { - action_id: "2".to_string(), - kind: "test".to_string(), - name: "test".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action).unwrap(); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "1"); - assert_eq!(state, "Received"); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "1"); - assert_eq!(state, "Launched"); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "2"); - assert_eq!(state, "Received"); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "2"); - assert_eq!(state, "Running"); - - let status = recv_response(&package_rx); - assert_eq!(status.action_id, "2"); - assert!(status.is_completed()); - - let status = recv_response(&package_rx); - assert_eq!(status.action_id, "1"); - assert!(status.is_completed()); - } - - #[tokio::test] - async fn accept_tunshell_during_regular_action() { - let tmpdir = tempdir::TempDir::new("bridge").unwrap(); - std::env::set_current_dir(&tmpdir).unwrap(); - let config = default_config(); - let (bridge_tx, actions_tx, package_rx) = start_bridge(Arc::new(config)); - let bridge_tx_clone = bridge_tx.clone(); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - let test_route = ActionRoute { name: "test".to_string(), timeout: 30 }; - let action_rx = rt.block_on(bridge_tx_clone.register_action_route(test_route)); - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "1"); - let response = ActionResponse::progress(&action.action_id, "Running", 0); - rt.block_on(bridge_tx_clone.send_action_response(response)); - std::thread::sleep(Duration::from_secs(3)); - let response = ActionResponse::success(&action.action_id); - rt.block_on(bridge_tx_clone.send_action_response(response)); - }); - - std::thread::spawn(move || { - let rt = Runtime::new().unwrap(); - let test_route = ActionRoute { name: TUNSHELL_ACTION.to_string(), timeout: 30 }; - let action_rx = rt.block_on(bridge_tx.register_action_route(test_route)); - let action = action_rx.recv().unwrap(); - assert_eq!(action.action_id, "2"); - let response = ActionResponse::progress(&action.action_id, "Launched", 0); - rt.block_on(bridge_tx.send_action_response(response)); - std::thread::sleep(Duration::from_secs(1)); - let response = ActionResponse::success(&action.action_id); - rt.block_on(bridge_tx.send_action_response(response)); - }); - - std::thread::sleep(Duration::from_secs(1)); - - let action = Action { - action_id: "1".to_string(), - kind: "test".to_string(), - name: "test".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action).unwrap(); - - std::thread::sleep(Duration::from_secs(1)); - - let action = Action { - action_id: "2".to_string(), - kind: "tunshell".to_string(), - name: "launch_shell".to_string(), - payload: "test".to_string(), - }; - actions_tx.send(action).unwrap(); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "1"); - assert_eq!(state, "Received"); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "1"); - assert_eq!(state, "Running"); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "2"); - assert_eq!(state, "Received"); - - let ActionResponse { action_id, state, .. } = recv_response(&package_rx); - assert_eq!(action_id, "2"); - assert_eq!(state, "Launched"); - - let status = recv_response(&package_rx); - assert_eq!(status.action_id, "2"); - assert!(status.is_completed()); - - let status = recv_response(&package_rx); - assert_eq!(status.action_id, "1"); - assert!(status.is_completed()); + self.status_tx.send_action_response(response).await } } diff --git a/uplink/src/base/bridge/stream.rs b/uplink/src/base/bridge/stream.rs index 2cd45dca6..e77ecbc68 100644 --- a/uplink/src/base/bridge/stream.rs +++ b/uplink/src/base/bridge/stream.rs @@ -4,9 +4,8 @@ use flume::{SendError, Sender}; use log::{debug, trace}; use serde::Serialize; -use crate::base::{Compression, StreamConfig, DEFAULT_TIMEOUT}; - use super::{Package, Point, StreamMetrics}; +use crate::config::StreamConfig; /// Signals status of stream buffer #[derive(Debug)] @@ -22,84 +21,52 @@ pub enum Error { Send(#[from] SendError>), } -pub const MAX_BUFFER_SIZE: usize = 100; +pub const MAX_BATCH_SIZE: usize = 100; #[derive(Debug)] pub struct Stream { pub name: Arc, - pub max_buffer_size: usize, - pub flush_period: Duration, - topic: Arc, + pub config: Arc, last_sequence: u32, last_timestamp: u64, buffer: Buffer, tx: Sender>, pub metrics: StreamMetrics, - compression: Compression, } impl Stream where - T: Point + Debug + Send + 'static, + T: Point, Buffer: Package, { pub fn new( - stream: impl Into, - topic: impl Into, - max_buffer_size: usize, + stream_name: impl Into, + stream_config: StreamConfig, tx: Sender>, - compression: Compression, ) -> Stream { - let name = Arc::new(stream.into()); - let topic = Arc::new(topic.into()); - let buffer = Buffer::new(name.clone(), topic.clone(), compression); - let flush_period = Duration::from_secs(DEFAULT_TIMEOUT); - let metrics = StreamMetrics::new(&name, max_buffer_size); + let name = Arc::new(stream_name.into()); + let config = Arc::new(stream_config); + let buffer = Buffer::new(name.clone(), config.clone()); + let metrics = StreamMetrics::new(&name, config.batch_size); - Stream { - name, - max_buffer_size, - flush_period, - topic, - last_sequence: 0, - last_timestamp: 0, - buffer, - tx, - metrics, - compression, - } - } - - pub fn with_config( - name: &String, - config: &StreamConfig, - tx: Sender>, - ) -> Stream { - let mut stream = Stream::new(name, &config.topic, config.buf_size, tx, config.compression); - stream.flush_period = Duration::from_secs(config.flush_period); - stream + Stream { name, config, last_sequence: 0, last_timestamp: 0, buffer, tx, metrics } } pub fn dynamic( - stream: impl Into, + stream_name: impl Into, project_id: impl Into, device_id: impl Into, - max_buffer_size: usize, tx: Sender>, ) -> Stream { - let stream = stream.into(); + let stream_name = stream_name.into(); let project_id = project_id.into(); let device_id = device_id.into(); - let topic = String::from("/tenants/") - + &project_id - + "/devices/" - + &device_id - + "/events/" - + &stream - + "/jsonarray"; + let topic = + format!("/tenants/{project_id}/devices/{device_id}/events/{stream_name}/jsonarray"); + let config = StreamConfig { topic, ..Default::default() }; - Stream::new(stream, topic, max_buffer_size, tx, Compression::Disabled) + Stream::new(stream_name, config, tx) } fn add(&mut self, data: T) -> Result>, Error> { @@ -126,8 +93,8 @@ where self.last_sequence = current_sequence; self.last_timestamp = current_timestamp; - // if max_buffer_size is breached, flush - let buf = if self.buffer.buffer.len() >= self.max_buffer_size { + // if max_bATCH_size is breached, flush + let buf = if self.buffer.buffer.len() >= self.config.batch_size { self.metrics.add_batch(); Some(self.take_buffer()) } else { @@ -140,10 +107,10 @@ where // Returns buffer content, replacing with empty buffer in-place fn take_buffer(&mut self) -> Buffer { let name = self.name.clone(); - let topic = self.topic.clone(); - trace!("Flushing stream name: {}, topic: {}", name, topic); + let config = self.config.clone(); + trace!("Flushing stream name: {}, topic: {}", name, config.topic); - mem::replace(&mut self.buffer, Buffer::new(name, topic, self.compression)) + mem::replace(&mut self.buffer, Buffer::new(name, config)) } /// Triggers flush and async channel send if not empty @@ -166,7 +133,7 @@ where self.len() == 0 } - /// Fill buffer with data and trigger async channel send on breaching max_buf_size. + /// Fill buffer with data and trigger async channel send on breaching max_batch_size. /// Returns [`StreamStatus`]. pub async fn fill(&mut self, data: T) -> Result { if let Some(buf) = self.add(data)? { @@ -175,7 +142,7 @@ where } let status = match self.len() { - 1 => StreamStatus::Init(self.flush_period), + 1 => StreamStatus::Init(self.config.flush_period), len => StreamStatus::Partial(len), }; @@ -183,7 +150,7 @@ where } #[cfg(test)] - /// Push data into buffer and trigger sync channel send on max_buf_size. + /// Push data into buffer and trigger sync channel send on max_batch_size. /// Returns [`StreamStatus`]. pub fn push(&mut self, data: T) -> Result { if let Some(buf) = self.add(data)? { @@ -192,7 +159,7 @@ where } let status = match self.len() { - 1 => StreamStatus::Init(self.flush_period), + 1 => StreamStatus::Init(self.config.flush_period), len => StreamStatus::Partial(len), }; @@ -210,23 +177,21 @@ where /// Buffer doesn't put any restriction on type of `T` #[derive(Debug)] pub struct Buffer { - pub stream: Arc, - pub topic: Arc, + pub stream_name: Arc, + pub stream_config: Arc, pub buffer: Vec, pub anomalies: String, pub anomaly_count: usize, - pub compression: Compression, } impl Buffer { - pub fn new(stream: Arc, topic: Arc, compression: Compression) -> Buffer { + pub fn new(stream_name: Arc, stream_config: Arc) -> Buffer { Buffer { - stream, - topic, - buffer: vec![], + buffer: Vec::with_capacity(stream_config.batch_size), + stream_name, + stream_config, anomalies: String::with_capacity(100), anomaly_count: 0, - compression, } } @@ -236,11 +201,7 @@ impl Buffer { return; } - let error = String::from(self.stream.as_ref()) - + ".sequence: " - + &last.to_string() - + ", " - + ¤t.to_string(); + let error = format!("{}.sequence: {last}, {current}", self.stream_name); self.anomalies.push_str(&error) } @@ -250,7 +211,7 @@ impl Buffer { return; } - let error = "timestamp: ".to_owned() + &last.to_string() + ", " + ¤t.to_string(); + let error = format!("timestamp: {last}, {current}"); self.anomalies.push_str(&error) } @@ -265,15 +226,15 @@ impl Buffer { impl Package for Buffer where - T: Debug + Send + Point, + T: Point, Vec: Serialize, { - fn topic(&self) -> Arc { - self.topic.clone() + fn stream_config(&self) -> Arc { + self.stream_config.clone() } - fn stream(&self) -> Arc { - self.stream.clone() + fn stream_name(&self) -> Arc { + self.stream_name.clone() } fn serialize(&self) -> serde_json::Result> { @@ -291,29 +252,18 @@ where fn latency(&self) -> u64 { 0 } - - fn compression(&self) -> Compression { - self.compression - } } impl Clone for Stream { fn clone(&self) -> Self { Stream { name: self.name.clone(), - flush_period: self.flush_period, - max_buffer_size: self.max_buffer_size, - topic: self.topic.clone(), + config: self.config.clone(), last_sequence: 0, last_timestamp: 0, - buffer: Buffer::new( - self.buffer.stream.clone(), - self.buffer.topic.clone(), - self.compression, - ), - metrics: StreamMetrics::new(&self.name, self.max_buffer_size), + buffer: Buffer::new(self.buffer.stream_name.clone(), self.buffer.stream_config.clone()), + metrics: StreamMetrics::new(&self.name, self.config.batch_size), tx: self.tx.clone(), - compression: self.compression, } } } diff --git a/uplink/src/base/bridge/streams.rs b/uplink/src/base/bridge/streams.rs index 0ca422c01..fae1e2afc 100644 --- a/uplink/src/base/bridge/streams.rs +++ b/uplink/src/base/bridge/streams.rs @@ -4,47 +4,41 @@ use std::sync::Arc; use flume::Sender; use log::{error, info, trace}; -use super::stream::{self, StreamStatus, MAX_BUFFER_SIZE}; -use super::StreamMetrics; -use crate::{Config, Package, Payload, Stream}; +use super::stream::{self, StreamStatus}; +use super::{Point, StreamMetrics}; +use crate::config::StreamConfig; +use crate::{Config, Package, Stream}; use super::delaymap::DelayMap; const MAX_STREAM_COUNT: usize = 1000; -pub struct Streams { +pub struct Streams { config: Arc, data_tx: Sender>, metrics_tx: Sender, - map: HashMap>, + map: HashMap>, pub stream_timeouts: DelayMap, - pub metrics_timeouts: DelayMap, } -impl Streams { +impl Streams { pub fn new( config: Arc, data_tx: Sender>, metrics_tx: Sender, ) -> Self { - let mut map = HashMap::new(); - for (name, stream) in &config.streams { - let stream = Stream::with_config(name, stream, data_tx.clone()); - map.insert(name.to_owned(), stream); - } + Self { config, data_tx, metrics_tx, map: HashMap::new(), stream_timeouts: DelayMap::new() } + } - Self { - config, - data_tx, - metrics_tx, - map, - stream_timeouts: DelayMap::new(), - metrics_timeouts: DelayMap::new(), + pub fn config_streams(&mut self, streams_config: HashMap) { + for (name, stream) in streams_config { + let stream = Stream::new(&name, stream, self.data_tx.clone()); + self.map.insert(name.to_owned(), stream); } } - pub async fn forward(&mut self, data: Payload) { - let stream_name = data.stream.to_owned(); + pub async fn forward(&mut self, data: T) { + let stream_name = data.stream_name().to_string(); let stream = match self.map.get_mut(&stream_name) { Some(partition) => partition, @@ -60,7 +54,6 @@ impl Streams { &stream_name, &self.config.project_id, &self.config.device_id, - MAX_BUFFER_SIZE, self.data_tx.clone(), ); @@ -68,7 +61,7 @@ impl Streams { } }; - let max_stream_size = stream.max_buffer_size; + let max_stream_size = stream.config.batch_size; let state = match stream.fill(data).await { Ok(s) => s, Err(e) => { diff --git a/uplink/src/base/mod.rs b/uplink/src/base/mod.rs index ba9110154..74887842c 100644 --- a/uplink/src/base/mod.rs +++ b/uplink/src/base/mod.rs @@ -1,16 +1,12 @@ -use std::env::{current_dir, var}; -use std::path::PathBuf; +use std::fmt::Debug; use std::time::{SystemTime, UNIX_EPOCH}; -use std::{collections::HashMap, fmt::Debug}; -use serde::{Deserialize, Serialize}; +use tokio::join; -#[cfg(target_os = "linux")] -use crate::collector::journalctl::JournalCtlConfig; -#[cfg(target_os = "android")] -use crate::collector::logcat::LogcatConfig; - -use self::bridge::stream::MAX_BUFFER_SIZE; +use self::bridge::{ActionsLaneCtrlTx, DataLaneCtrlTx}; +use self::mqtt::CtrlTx as MqttCtrlTx; +use self::serializer::CtrlTx as SerializerCtrlTx; +use crate::collector::downloader::CtrlTx as DownloaderCtrlTx; pub mod actions; pub mod bridge; @@ -18,301 +14,30 @@ pub mod monitor; pub mod mqtt; pub mod serializer; -pub const DEFAULT_TIMEOUT: u64 = 60; - -#[inline] -fn default_timeout() -> u64 { - DEFAULT_TIMEOUT -} - -#[inline] -fn max_buf_size() -> usize { - MAX_BUFFER_SIZE -} - -fn default_file_size() -> usize { - 10485760 // 10MB -} - -fn default_persistence_path() -> PathBuf { - let mut path = current_dir().expect("Couldn't figure out current directory"); - path.push(".persistence"); - path -} - -// Automatically assigns port 5050 for default main app, if left unconfigured -fn default_tcpapps() -> HashMap { - let mut apps = HashMap::new(); - apps.insert("main".to_string(), AppConfig { port: 5050, actions: vec![] }); - - apps -} - -fn default_clickhouse_host() -> String { - "localhost".to_string() -} - -fn default_clickhouse_port() -> u16 { - 8123 -} - -fn default_clickhouse_username() -> String { - var("CLICKHOUSE_USERNAME").expect("The env variable CLICKHOUSE_USERNAME is not set") -} - -fn default_clickhouse_password() -> String { - var("CLICKHOUSE_PASSWORD").expect("The env variable CLICKHOUSE_PASSWORD is not set") -} - pub fn clock() -> u128 { SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() } -#[derive(Debug, Clone, Copy, Deserialize, Serialize, Default)] -pub enum Compression { - #[default] - Disabled, - Lz4, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct StreamConfig { - pub topic: String, - #[serde(default = "max_buf_size")] - pub buf_size: usize, - #[serde(default = "default_timeout")] - /// Duration(in seconds) that bridge collector waits from - /// receiving first element, before the stream gets flushed. - pub flush_period: u64, - #[serde(default)] - pub compression: Compression, - #[serde(default)] - pub persistence: Persistence, -} - -impl Default for StreamConfig { - fn default() -> Self { - Self { - topic: "".to_string(), - buf_size: MAX_BUFFER_SIZE, - flush_period: default_timeout(), - compression: Compression::Disabled, - persistence: Persistence::default(), - } - } -} - -#[derive(Debug, Clone, Deserialize)] -pub struct Persistence { - #[serde(default = "default_file_size")] - pub max_file_size: usize, - #[serde(default)] - pub max_file_count: usize, -} - -impl Default for Persistence { - fn default() -> Self { - Persistence { max_file_size: default_file_size(), max_file_count: 0 } +/// Send control messages to the various components in uplink. Currently this is +/// used only to trigger uplink shutdown. Shutdown signals are sent to all +/// components simultaneously with a join. +#[derive(Debug, Clone)] +pub struct CtrlTx { + pub actions_lane: ActionsLaneCtrlTx, + pub data_lane: DataLaneCtrlTx, + pub mqtt: MqttCtrlTx, + pub serializer: SerializerCtrlTx, + pub downloader: DownloaderCtrlTx, +} + +impl CtrlTx { + pub async fn trigger_shutdown(&self) { + join!( + self.actions_lane.trigger_shutdown(), + self.data_lane.trigger_shutdown(), + self.mqtt.trigger_shutdown(), + self.serializer.trigger_shutdown(), + self.downloader.trigger_shutdown() + ); } } - -#[derive(Debug, Clone, Deserialize)] -pub struct Authentication { - pub ca_certificate: String, - pub device_certificate: String, - pub device_private_key: String, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct Stats { - pub enabled: bool, - pub process_names: Vec, - pub update_period: u64, - pub stream_size: Option, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct SimulatorConfig { - /// path to directory containing files with gps paths to be used in simulation - pub gps_paths: String, - /// actions that are to be routed to simulator - pub actions: Vec, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct DownloaderConfig { - pub path: String, - #[serde(default)] - pub actions: Vec, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct InstallerConfig { - pub path: String, - #[serde(default)] - pub actions: Vec, - pub uplink_port: u16, -} - -#[derive(Debug, Clone, Deserialize, Serialize, Default)] -pub struct StreamMetricsConfig { - pub enabled: bool, - pub topic: String, - pub blacklist: Vec, - pub timeout: u64, -} - -#[derive(Debug, Clone, Deserialize, Serialize, Default)] -pub struct SerializerMetricsConfig { - pub enabled: bool, - pub topic: String, - pub timeout: u64, -} - -#[derive(Debug, Clone, Deserialize, Serialize, Default)] -pub struct MqttMetricsConfig { - pub enabled: bool, - pub topic: String, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct AppConfig { - pub port: u16, - #[serde(default)] - pub actions: Vec, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct ConsoleConfig { - pub enabled: bool, - pub port: u16, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct MqttConfig { - pub max_packet_size: usize, - pub max_inflight: u16, - pub keep_alive: u64, - pub network_timeout: u64, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct ActionRoute { - pub name: String, - #[serde(default = "default_timeout")] - pub timeout: u64, -} - -impl From<&ActionRoute> for ActionRoute { - fn from(value: &ActionRoute) -> Self { - value.clone() - } -} - -#[derive(Clone, Debug, Deserialize)] -pub struct DeviceShadowConfig { - pub interval: u64, -} - -impl Default for DeviceShadowConfig { - fn default() -> Self { - Self { interval: DEFAULT_TIMEOUT } - } -} - -fn default_true() -> bool { - true -} - -#[derive(Clone, Debug, Deserialize)] -pub struct LogReaderConfig { - pub path: String, - pub stream_name: String, - pub log_template: String, - pub timestamp_template: String, - #[serde(default = "default_true")] - pub multi_line: bool -} - -#[derive(Clone, Debug, Deserialize)] -pub struct PrometheusConfig { - pub endpoint: String, - pub interval: u64, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct QueryLogConfig { - pub stream: String, - pub where_clause: String, - pub interval: u64, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct ClickhouseConfig { - #[serde(default = "default_clickhouse_host")] - pub host: String, - #[serde(default = "default_clickhouse_port")] - pub port: u16, - #[serde(default = "default_clickhouse_username")] - pub username: String, - #[serde(default = "default_clickhouse_password")] - pub password: String, - pub query_log: Option, -} - -impl Default for ClickhouseConfig { - fn default() -> Self { - Self { - host: default_clickhouse_host(), - port: default_clickhouse_port(), - username: default_clickhouse_username(), - password: default_clickhouse_password(), - query_log: None, - } - } -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct Config { - pub project_id: String, - pub device_id: String, - pub broker: String, - pub port: u16, - #[serde(default)] - pub console: ConsoleConfig, - pub authentication: Option, - #[serde(default = "default_tcpapps")] - pub tcpapps: HashMap, - pub mqtt: MqttConfig, - #[serde(default)] - pub processes: Vec, - #[serde(default)] - pub script_runner: Vec, - #[serde(skip)] - pub actions_subscription: String, - pub streams: HashMap, - #[serde(default = "default_persistence_path")] - pub persistence_path: PathBuf, - pub action_status: StreamConfig, - pub stream_metrics: StreamMetricsConfig, - pub serializer_metrics: SerializerMetricsConfig, - pub mqtt_metrics: MqttMetricsConfig, - pub downloader: DownloaderConfig, - pub system_stats: Stats, - pub simulator: Option, - pub ota_installer: Option, - #[serde(default)] - pub log_reader: HashMap, - pub prometheus: Option, - pub clickhouse: Option, - #[serde(default)] - pub device_shadow: DeviceShadowConfig, - #[serde(default)] - pub action_redirections: HashMap, - #[serde(default)] - pub ignore_actions_if_no_clients: bool, - #[cfg(target_os = "linux")] - pub logging: Option, - #[cfg(target_os = "android")] - pub logging: Option, -} diff --git a/uplink/src/base/monitor/mod.rs b/uplink/src/base/monitor/mod.rs index 662402a9a..d3bf75318 100644 --- a/uplink/src/base/monitor/mod.rs +++ b/uplink/src/base/monitor/mod.rs @@ -5,9 +5,9 @@ use flume::{Receiver, RecvError}; use rumqttc::{AsyncClient, ClientError, QoS, Request}; use tokio::select; -use crate::base::bridge::StreamMetrics; use crate::Config; +use super::bridge::StreamMetrics; use super::mqtt::MqttMetrics; use super::serializer::SerializerMetrics; @@ -38,8 +38,10 @@ impl Monitor { pub async fn start(&self) -> Result<(), Error> { let stream_metrics_config = self.config.stream_metrics.clone(); - let stream_metrics_topic = stream_metrics_config.topic; - let mut stream_metrics = Vec::with_capacity(10); + let bridge_stream_metrics_topic = stream_metrics_config.bridge_topic; + let mut bridge_stream_metrics = Vec::with_capacity(10); + let serializer_stream_metrics_topic = stream_metrics_config.serializer_topic; + let mut serializer_stream_metrics = Vec::with_capacity(10); let serializer_metrics_config = self.config.serializer_metrics.clone(); let serializer_metrics_topic = serializer_metrics_config.topic; @@ -58,18 +60,31 @@ impl Monitor { continue; } - stream_metrics.push(o); - let v = serde_json::to_string(&stream_metrics).unwrap(); + bridge_stream_metrics.push(o); + let v = serde_json::to_string(&bridge_stream_metrics).unwrap(); - stream_metrics.clear(); - self.client.publish(&stream_metrics_topic, QoS::AtLeastOnce, false, v).await.unwrap(); + bridge_stream_metrics.clear(); + self.client.publish(&bridge_stream_metrics_topic, QoS::AtLeastOnce, false, v).await.unwrap(); } o = self.serializer_metrics_rx.recv_async() => { let o = o?; - serializer_metrics.push(o); - let v = serde_json::to_string(&serializer_metrics).unwrap(); - serializer_metrics.clear(); - self.client.publish(&serializer_metrics_topic, QoS::AtLeastOnce, false, v).await.unwrap(); + match o { + SerializerMetrics::Main(o) => { + serializer_metrics.push(o); + let v = serde_json::to_string(&serializer_metrics).unwrap(); + serializer_metrics.clear(); + self.client.publish(&serializer_metrics_topic, QoS::AtLeastOnce, false, v).await.unwrap(); + } + SerializerMetrics::Stream(o) => { + if stream_metrics_config.blacklist.contains(&o.stream) { + continue; + } + serializer_stream_metrics.push(o); + let v = serde_json::to_string(&serializer_stream_metrics).unwrap(); + serializer_stream_metrics.clear(); + self.client.publish(&serializer_stream_metrics_topic, QoS::AtLeastOnce, false, v).await.unwrap(); + } + } } o = self.mqtt_metrics_rx.recv_async() => { let o = o?; diff --git a/uplink/src/base/mqtt/mod.rs b/uplink/src/base/mqtt/mod.rs index 07026aad4..12f4f3d83 100644 --- a/uplink/src/base/mqtt/mod.rs +++ b/uplink/src/base/mqtt/mod.rs @@ -1,8 +1,10 @@ -use flume::{Sender, TrySendError}; +use bytes::BytesMut; +use flume::{bounded, Receiver, Sender, TrySendError}; use log::{debug, error, info}; +use storage::PersistenceFile; use thiserror::Error; -use tokio::task; use tokio::time::Duration; +use tokio::{select, task}; use std::fs::File; use std::io::Read; @@ -10,8 +12,8 @@ use std::path::Path; use crate::{Action, Config}; use rumqttc::{ - AsyncClient, ConnectionError, Event, EventLoop, Incoming, Key, MqttOptions, Publish, QoS, - TlsConfiguration, Transport, + AsyncClient, ConnectionError, Event, EventLoop, Incoming, MqttOptions, Packet, Publish, QoS, + Request, TlsConfiguration, Transport, }; use std::sync::Arc; @@ -25,6 +27,12 @@ pub enum Error { Serde(#[from] serde_json::Error), #[error("TrySend error {0}")] TrySend(Box>), + #[error("Io error {0}")] + Io(#[from] std::io::Error), + #[error("Mqtt error {0}")] + Mqtt(#[from] rumqttc::mqttbytes::Error), + #[error("Storage error {0}")] + Storage(#[from] storage::Error), } impl From> for Error { @@ -47,6 +55,9 @@ pub struct Mqtt { metrics: MqttMetrics, /// Metrics tx metrics_tx: Sender, + /// Control handles + ctrl_rx: Receiver, + ctrl_tx: Sender, } impl Mqtt { @@ -57,8 +68,9 @@ impl Mqtt { ) -> Mqtt { // create a new eventloop and reuse it during every reconnection let options = mqttoptions(&config); - let (client, mut eventloop) = AsyncClient::new(options, 10); + let (client, mut eventloop) = AsyncClient::new(options, 0); eventloop.network_options.set_connection_timeout(config.mqtt.network_timeout); + let (ctrl_tx, ctrl_rx) = bounded(1); Mqtt { config, @@ -67,6 +79,8 @@ impl Mqtt { native_actions_tx: actions_tx, metrics: MqttMetrics::new(), metrics_tx, + ctrl_tx, + ctrl_rx, } } @@ -75,65 +89,152 @@ impl Mqtt { self.client.clone() } + pub fn ctrl_tx(&self) -> CtrlTx { + CtrlTx { inner: self.ctrl_tx.clone() } + } + + /// Shutdown eventloop and write inflight publish packets to disk + pub fn persist_inflight(&mut self) -> Result<(), Error> { + self.eventloop.clean(); + let publishes: Vec<&Publish> = self + .eventloop + .pending + .iter() + .filter_map(|request| match request { + Request::Publish(publish) => Some(publish), + _ => None, + }) + .collect(); + + if publishes.is_empty() { + return Ok(()); + } + + let mut file = PersistenceFile::new(&self.config.persistence_path, "inflight".to_string())?; + let mut buf = BytesMut::new(); + + for publish in publishes { + publish.write(&mut buf)?; + } + + file.write(&mut buf)?; + debug!("Pending publishes written to disk: {}", file.path().display()); + + Ok(()) + } + + /// Checks for and loads data pending in persistence/inflight file + /// once done, deletes the file, while writing incoming data into storage. + fn reload_from_inflight_file(&mut self) -> Result<(), Error> { + // Read contents of inflight file into an in-memory buffer + let mut file = PersistenceFile::new(&self.config.persistence_path, "inflight".to_string())?; + let path = file.path(); + if !path.is_file() { + return Ok(()); + } + let mut buf = BytesMut::new(); + file.read(&mut buf)?; + + let max_packet_size = self.config.mqtt.max_packet_size; + loop { + // NOTE: This can fail when packet sizes > max_payload_size in config are written to disk. + match Packet::read(&mut buf, max_packet_size) { + Ok(Packet::Publish(publish)) => { + self.eventloop.pending.push_back(Request::Publish(publish)) + } + Ok(packet) => unreachable!("Unexpected packet: {:?}", packet), + Err(rumqttc::Error::InsufficientBytes(_)) => break, + Err(e) => { + error!("Error reading from file: {e}"); + break; + } + } + } + + info!("Pending publishes read from disk; removing file: {}", path.display()); + file.delete()?; + + Ok(()) + } + /// Poll eventloop to receive packets from broker pub async fn start(mut self) { + if let Err(e) = self.reload_from_inflight_file() { + error!("Error recovering data from inflight file: {e}"); + } + loop { - match self.eventloop.poll().await { - Ok(Event::Incoming(Incoming::ConnAck(connack))) => { - info!("Connected to broker. Session present = {}", connack.session_present); - let subscription = self.config.actions_subscription.clone(); - let client = self.client(); - - self.metrics.add_connection(); - - // This can potentially block when client from other threads - // have already filled the channel due to bad network. So we spawn - task::spawn(async move { - match client.subscribe(&subscription, QoS::AtLeastOnce).await { - Ok(..) => info!("Subscribe -> {:?}", subscription), - Err(e) => error!("Failed to send subscription. Error = {:?}", e), + select! { + event = self.eventloop.poll() => { + match event { + Ok(Event::Incoming(Incoming::ConnAck(connack))) => { + info!("Connected to broker. Session present = {}", connack.session_present); + let subscription = self.config.actions_subscription.clone(); + let client = self.client(); + + self.metrics.add_connection(); + + // This can potentially block when client from other threads + // have already filled the channel due to bad network. So we spawn + task::spawn(async move { + match client.subscribe(&subscription, QoS::AtLeastOnce).await { + Ok(..) => info!("Subscribe -> {:?}", subscription), + Err(e) => error!("Failed to send subscription. Error = {:?}", e), + } + }); } - }); - } - Ok(Event::Incoming(Incoming::Publish(p))) => { - self.metrics.add_action(); - if let Err(e) = self.handle_incoming_publish(p) { - error!("Incoming publish handle failed. Error = {:?}", e); - } - } - Ok(Event::Incoming(packet)) => { - debug!("Incoming = {:?}", packet); - match packet { - rumqttc::Packet::PubAck(_) => self.metrics.add_puback(), - rumqttc::Packet::PingResp => { - self.metrics.add_pingresp(); - let inflight = self.eventloop.state.inflight(); - self.metrics.update_inflight(inflight); - if let Err(e) = self.check_and_flush_metrics() { - error!("Failed to flush MQTT metrics. Erro = {:?}", e); + Ok(Event::Incoming(Incoming::Publish(p))) => { + self.metrics.add_action(); + if let Err(e) = self.handle_incoming_publish(p) { + error!("Incoming publish handle failed. Error = {:?}", e); } } - _ => {} - } - } - Ok(Event::Outgoing(packet)) => { - debug!("Outgoing = {:?}", packet); - match packet { - rumqttc::Outgoing::Publish(_) => self.metrics.add_publish(), - rumqttc::Outgoing::PingReq => { - self.metrics.add_pingreq(); + Ok(Event::Incoming(packet)) => { + debug!("Incoming = {:?}", packet); + match packet { + rumqttc::Packet::PubAck(_) => self.metrics.add_puback(), + rumqttc::Packet::PingResp => { + self.metrics.add_pingresp(); + let inflight = self.eventloop.state.inflight(); + self.metrics.update_inflight(inflight); + if let Err(e) = self.check_and_flush_metrics() { + error!("Failed to flush MQTT metrics. Erro = {:?}", e); + } + } + _ => {} + } + } + Ok(Event::Outgoing(packet)) => { + debug!("Outgoing = {:?}", packet); + match packet { + rumqttc::Outgoing::Publish(_) => self.metrics.add_publish(), + rumqttc::Outgoing::PingReq => { + self.metrics.add_pingreq(); + } + _ => {} + } + } + Err(e) => { + self.metrics.add_reconnection(); + self.check_disconnection_metrics(e); + tokio::time::sleep(Duration::from_secs(3)).await; + continue; } - _ => {} } - } - Err(e) => { - self.metrics.add_reconnection(); - self.check_disconnection_metrics(e); - tokio::time::sleep(Duration::from_secs(3)).await; - continue; + }, + Ok(MqttShutdown) = self.ctrl_rx.recv_async() => { + break; } } } + + // TODO: when uplink uses last-wills to handle unexpected disconnections, try to disconnect from + // mqtt connection, before force persisting in-flight publishes to disk. Timedout in a second. + // But otherwise sending a disconnect to broker is unnecessary. + + if let Err(e) = self.persist_inflight() { + error!("Couldn't persist inflight messages. Error = {:?}", e); + } } fn handle_incoming_publish(&mut self, publish: Publish) -> Result<(), Error> { @@ -195,7 +296,7 @@ fn mqttoptions(config: &Config) -> MqttOptions { let transport = Transport::Tls(TlsConfiguration::Simple { ca, alpn: None, - client_auth: Some((device_certificate, Key::RSA(device_private_key))), + client_auth: Some((device_certificate, device_private_key)), }); mqttoptions.set_transport(transport); @@ -216,3 +317,19 @@ fn _get_certs(key_path: &Path, ca_path: &Path) -> (Vec, Vec) { (key, ca) } + +/// Command to remotely trigger `Mqtt` shutdown +pub(crate) struct MqttShutdown; + +/// Handle to send control messages to `Mqtt` +#[derive(Debug, Clone)] +pub struct CtrlTx { + pub(crate) inner: Sender, +} + +impl CtrlTx { + /// Triggers shutdown of `Mqtt` + pub async fn trigger_shutdown(&self) { + self.inner.send_async(MqttShutdown).await.unwrap() + } +} diff --git a/uplink/src/base/serializer/metrics.rs b/uplink/src/base/serializer/metrics.rs index 2844aeadd..12eea18aa 100644 --- a/uplink/src/base/serializer/metrics.rs +++ b/uplink/src/base/serializer/metrics.rs @@ -1,10 +1,13 @@ +use std::time::Duration; + use serde::Serialize; +use serde_with::{serde_as, DurationSecondsWithFrac}; use crate::base::clock; /// Metrics information relating to the operation of the `Serializer`, all values are reset on metrics flush #[derive(Debug, Serialize, Clone)] -pub struct SerializerMetrics { +pub struct Metrics { timestamp: u128, sequence: u32, /// One of **Catchup**, **Normal**, **Slow** or **Crash** @@ -17,6 +20,8 @@ pub struct SerializerMetrics { pub read_memory: usize, /// Number of files that have been written to disk pub disk_files: usize, + /// Disk size currently occupied by persistence files + pub disk_utilized: usize, /// Nuber of persistence files that had to deleted before being consumed pub lost_segments: usize, /// Number of errors faced during serializer operation @@ -25,9 +30,9 @@ pub struct SerializerMetrics { pub sent_size: usize, } -impl SerializerMetrics { +impl Metrics { pub fn new(mode: &str) -> Self { - SerializerMetrics { + Metrics { timestamp: clock(), sequence: 1, mode: mode.to_owned(), @@ -35,6 +40,7 @@ impl SerializerMetrics { write_memory: 0, read_memory: 0, disk_files: 0, + disk_utilized: 0, lost_segments: 0, errors: 0, sent_size: 0, @@ -42,7 +48,7 @@ impl SerializerMetrics { } pub fn set_mode(&mut self, name: &str) { - self.mode = name.to_owned(); + name.clone_into(&mut self.mode); } pub fn batches(&self) -> usize { @@ -68,6 +74,10 @@ impl SerializerMetrics { self.disk_files = count; } + pub fn set_disk_utilized(&mut self, bytes: usize) { + self.disk_utilized = bytes; + } + pub fn increment_errors(&mut self) { self.errors += 1; } @@ -92,3 +102,81 @@ impl SerializerMetrics { self.errors = 0; } } + +#[serde_as] +#[derive(Debug, Serialize, Clone)] +pub struct StreamMetrics { + pub timestamp: u128, + pub sequence: u32, + pub stream: String, + pub serialized_data_size: usize, + pub compressed_data_size: usize, + #[serde(skip)] + pub serializations: u32, + #[serde_as(as = "DurationSecondsWithFrac")] + pub total_serialization_time: Duration, + #[serde_as(as = "DurationSecondsWithFrac")] + pub avg_serialization_time: Duration, + #[serde(skip)] + pub compressions: u32, + #[serde_as(as = "DurationSecondsWithFrac")] + pub total_compression_time: Duration, + #[serde_as(as = "DurationSecondsWithFrac")] + pub avg_compression_time: Duration, +} + +impl StreamMetrics { + pub fn new(name: &str) -> Self { + StreamMetrics { + stream: name.to_owned(), + timestamp: clock(), + sequence: 1, + serialized_data_size: 0, + compressed_data_size: 0, + serializations: 0, + total_serialization_time: Duration::ZERO, + avg_serialization_time: Duration::ZERO, + compressions: 0, + total_compression_time: Duration::ZERO, + avg_compression_time: Duration::ZERO, + } + } + + pub fn add_serialized_sizes(&mut self, data_size: usize, compressed_data_size: Option) { + self.serialized_data_size += data_size; + self.compressed_data_size += compressed_data_size.unwrap_or(data_size); + } + + pub fn add_serialization_time(&mut self, serialization_time: Duration) { + self.serializations += 1; + self.total_serialization_time += serialization_time; + } + + pub fn add_compression_time(&mut self, compression_time: Duration) { + self.compressions += 1; + self.total_compression_time += compression_time; + } + + // Should be called before serializing metrics to ensure averages are computed. + // Averages aren't calculated for ever `add_*` call to save on costs. + pub fn prepare_snapshot(&mut self) { + self.avg_serialization_time = self + .total_serialization_time + .checked_div(self.serializations) + .unwrap_or(Duration::ZERO); + self.avg_compression_time = + self.total_compression_time.checked_div(self.compressions).unwrap_or(Duration::ZERO); + } + + pub fn prepare_next(&mut self) { + self.timestamp = clock(); + self.sequence += 1; + self.serialized_data_size = 0; + self.compressed_data_size = 0; + } +} + +pub enum SerializerMetrics { + Main(Box), + Stream(Box), +} diff --git a/uplink/src/base/serializer/mod.rs b/uplink/src/base/serializer/mod.rs index 1db1c32d4..e536ad24f 100644 --- a/uplink/src/base/serializer/mod.rs +++ b/uplink/src/base/serializer/mod.rs @@ -1,11 +1,12 @@ mod metrics; -use std::collections::{HashMap, VecDeque}; +use std::collections::{BTreeMap, HashMap, VecDeque}; use std::io::{self, Write}; +use std::time::Instant; use std::{sync::Arc, time::Duration}; use bytes::Bytes; -use flume::{Receiver, RecvError, Sender}; +use flume::{bounded, Receiver, RecvError, Sender, TrySendError}; use log::{debug, error, info, trace}; use lz4_flex::frame::FrameEncoder; use rumqttc::*; @@ -13,11 +14,9 @@ use storage::Storage; use thiserror::Error; use tokio::{select, time::interval}; -use crate::base::Compression; +use crate::config::{default_file_size, Compression, StreamConfig}; use crate::{Config, Package}; -pub use metrics::SerializerMetrics; - -use super::default_file_size; +pub use metrics::{Metrics, SerializerMetrics, StreamMetrics}; const METRICS_INTERVAL: Duration = Duration::from_secs(10); @@ -46,8 +45,8 @@ pub enum Error { Serde(#[from] serde_json::Error), #[error("Io error {0}")] Io(#[from] io::Error), - #[error("Disk error {0}")] - Disk(#[from] storage::Error), + #[error("Storage error {0}")] + Storage(#[from] storage::Error), #[error("Mqtt client error {0}")] Client(#[from] MqttError), #[error("Storage is disabled/missing")] @@ -58,14 +57,17 @@ pub enum Error { EmptyStorage, #[error("Permission denied while accessing persistence directory \"{0}\"")] Persistence(String), + #[error("Serializer has shutdown after handling crash")] + Shutdown, } #[derive(Debug, PartialEq)] enum Status { Normal, - SlowEventloop(Publish), + SlowEventloop(Publish, Arc), EventLoopReady, - EventLoopCrash(Publish), + EventLoopCrash(Publish, Arc), + Shutdown, } /// Description of an interface that the [`Serializer`] expects to be provided by the MQTT client to publish the serialized data with. @@ -92,8 +94,8 @@ pub trait MqttClient: Clone { payload: V, ) -> Result<(), MqttError> where - S: Into, - V: Into>; + S: Into + Send, + V: Into> + Send; } #[async_trait::async_trait] @@ -130,14 +132,17 @@ impl MqttClient for AsyncClient { } struct StorageHandler { - map: HashMap, + map: BTreeMap, Storage>, + // Stream being read from + read_stream: Option>, } impl StorageHandler { fn new(config: Arc) -> Result { - let mut map = HashMap::with_capacity(2 * config.streams.len()); + let mut map = BTreeMap::new(); for (stream_name, stream_config) in config.streams.iter() { - let mut storage = Storage::new(stream_config.persistence.max_file_size); + let mut storage = + Storage::new(&stream_config.topic, stream_config.persistence.max_file_size); if stream_config.persistence.max_file_count > 0 { let mut path = config.persistence_path.clone(); path.push(stream_name); @@ -152,26 +157,44 @@ impl StorageHandler { path.display() ); } - map.insert(stream_config.topic.clone(), storage); + map.insert(Arc::new(stream_config.clone()), storage); } - Ok(Self { map }) + Ok(Self { map, read_stream: None }) } - fn select(&mut self, topic: &str) -> &mut Storage { - self.map.entry(topic.to_owned()).or_insert_with(|| Storage::new(default_file_size())) + fn select(&mut self, stream: &Arc) -> &mut Storage { + self.map + .entry(stream.to_owned()) + .or_insert_with(|| Storage::new(&stream.topic, default_file_size())) } - fn next(&mut self, metrics: &mut SerializerMetrics) -> Option<&mut Storage> { - let storages = self.map.values_mut(); + fn next(&mut self, metrics: &mut Metrics) -> Option<(&Arc, &mut Storage)> { + let storages = self.map.iter_mut(); - for storage in storages { - match storage.reload_on_eof() { - // Done reading all the pending files - Ok(true) => continue, - Ok(false) => return Some(storage), + for (stream, storage) in storages { + match (storage.reload_on_eof(), &mut self.read_stream) { + // Done reading all pending files for a persisted stream + (Ok(true), Some(curr_stream)) => { + if curr_stream == stream { + self.read_stream.take(); + debug!("Completed reading from: {}", stream.topic); + } + + continue; + } + // Persisted stream is empty + (Ok(true), _) => continue, + // Reading from a newly loaded non-empty persisted stream + (Ok(false), None) => { + debug!("Reading from: {}", stream.topic); + self.read_stream = Some(stream.to_owned()); + return Some((stream, storage)); + } + // Continuing to read from persisted stream loaded earlier + (Ok(false), _) => return Some((stream, storage)), // Reload again on encountering a corrupted file - Err(e) => { + (Err(e), _) => { metrics.increment_errors(); metrics.increment_lost_segments(); error!("Failed to reload from storage. Error = {e}"); @@ -182,6 +205,19 @@ impl StorageHandler { None } + + fn flush_all(&mut self) { + for (stream_config, storage) in self.map.iter_mut() { + match storage.flush() { + Ok(_) => trace!("Force flushed stream = {} onto disk", stream_config.topic), + Err(storage::Error::NoWrites) => {} + Err(e) => error!( + "Error when force flushing storage = {}; error = {e}", + stream_config.topic + ), + } + } + } } /// The uplink Serializer is the component that deals with serializing, compressing and writing data onto disk or Network. @@ -219,9 +255,12 @@ impl StorageHandler { /// │Serializer::slow(publish)◄───────────────────────────┤SlowEventloop(publish)│ /// └-------------------------┘ └──────────────────────┘ /// Write to storage, Slow network encountered -/// but continue trying to publish +/// but continue trying to publish /// ///``` +/// +/// NOTE: Shutdown mode and crash mode are only different in how they get triggered, +/// but should be considered as interchangeable in the above diagram. /// [`start()`]: Serializer::start /// [`try_publish()`]: AsyncClient::try_publish /// [`publish()`]: AsyncClient::publish @@ -230,9 +269,13 @@ pub struct Serializer { collector_rx: Receiver>, client: C, storage_handler: StorageHandler, - metrics: SerializerMetrics, + metrics: Metrics, metrics_tx: Sender, pending_metrics: VecDeque, + stream_metrics: HashMap, + /// Control handles + ctrl_rx: Receiver, + ctrl_tx: Sender, } impl Serializer { @@ -245,23 +288,59 @@ impl Serializer { metrics_tx: Sender, ) -> Result, Error> { let storage_handler = StorageHandler::new(config.clone())?; + let (ctrl_tx, ctrl_rx) = bounded(1); Ok(Serializer { config, collector_rx, client, storage_handler, - metrics: SerializerMetrics::new("catchup"), + metrics: Metrics::new("catchup"), + stream_metrics: HashMap::new(), metrics_tx, pending_metrics: VecDeque::with_capacity(3), + ctrl_tx, + ctrl_rx, }) } + pub fn ctrl_tx(&self) -> CtrlTx { + CtrlTx { inner: self.ctrl_tx.clone() } + } + + /// Write all data received, from here-on, to disk only, shutdown serializer + /// after handling all data payloads. + fn shutdown(&mut self) -> Result<(), Error> { + debug!("Forced into shutdown mode, writing all incoming data to persistence."); + + loop { + // Collect remaining data packets and write to disk + // NOTE: wait 2s to allow bridge to shutdown and flush leftover data. + let deadline = Instant::now() + Duration::from_secs(2); + let Ok(data) = self.collector_rx.recv_deadline(deadline) else { + self.storage_handler.flush_all(); + return Ok(()); + }; + let stream_config = data.stream_config(); + let publish = construct_publish(data, &mut self.stream_metrics)?; + let storage = self.storage_handler.select(&stream_config); + match write_to_storage(publish, storage) { + Ok(Some(deleted)) => debug!("Lost segment = {deleted}"), + Ok(_) => {} + Err(e) => error!("Shutdown: write error = {:?}", e), + } + } + } + /// Write all data received, from here-on, to disk only. - async fn crash(&mut self, publish: Publish) -> Result { - let storage = self.storage_handler.select(&publish.topic); + async fn crash( + &mut self, + publish: Publish, + stream: Arc, + ) -> Result { + let storage = self.storage_handler.select(&stream); // Write failed publish to disk first, metrics don't matter - match write_to_disk(publish, storage) { + match write_to_storage(publish, storage) { Ok(Some(deleted)) => debug!("Lost segment = {deleted}"), Ok(_) => {} Err(e) => error!("Crash loop: write error = {:?}", e), @@ -270,9 +349,9 @@ impl Serializer { loop { // Collect next data packet and write to disk let data = self.collector_rx.recv_async().await?; - let publish = construct_publish(data)?; - let storage = self.storage_handler.select(&publish.topic); - match write_to_disk(publish, storage) { + let publish = construct_publish(data, &mut self.stream_metrics)?; + let storage = self.storage_handler.select(&stream); + match write_to_storage(publish, storage) { Ok(Some(deleted)) => debug!("Lost segment = {deleted}"), Ok(_) => {} Err(e) => error!("Crash loop: write error = {:?}", e), @@ -282,7 +361,7 @@ impl Serializer { /// Write new data to disk until back pressure due to slow n/w is resolved // TODO: Handle errors. Don't return errors - async fn slow(&mut self, publish: Publish) -> Result { + async fn slow(&mut self, publish: Publish, stream: Arc) -> Result { let mut interval = interval(METRICS_INTERVAL); // Reactlabs setup processes logs generated by uplink info!("Switching to slow eventloop mode!!"); @@ -298,9 +377,10 @@ impl Serializer { select! { data = self.collector_rx.recv_async() => { let data = data?; - let publish = construct_publish(data)?; - let storage = self.storage_handler.select(&publish.topic); - match write_to_disk(publish, storage) { + let stream = data.stream_config(); + let publish = construct_publish(data, &mut self.stream_metrics)?; + let storage = self.storage_handler.select(&stream); + match write_to_storage(publish, storage) { Ok(Some(deleted)) => { debug!("Lost segment = {deleted}"); self.metrics.increment_lost_segments(); @@ -320,14 +400,18 @@ impl Serializer { break Ok(Status::EventLoopReady) } Err(MqttError::Send(Request::Publish(publish))) => { - break Ok(Status::EventLoopCrash(publish)); + break Ok(Status::EventLoopCrash(publish, stream)); }, Err(e) => { unreachable!("Unexpected error: {}", e); } }, _ = interval.tick() => { - check_metrics(&mut self.metrics, &self.storage_handler); + check_metrics(&mut self.metrics, &mut self.stream_metrics, &self.storage_handler); + } + // Transition into crash mode when uplink is shutting down + Ok(SerializerShutdown) = self.ctrl_rx.recv_async() => { + break Ok(Status::Shutdown) } } }; @@ -335,6 +419,7 @@ impl Serializer { save_and_prepare_next_metrics( &mut self.pending_metrics, &mut self.metrics, + &mut self.stream_metrics, &self.storage_handler, ); let v = v?; @@ -355,14 +440,14 @@ impl Serializer { let max_packet_size = self.config.mqtt.max_packet_size; let client = self.client.clone(); - let storage = match self.storage_handler.next(&mut self.metrics) { + let (stream, storage) = match self.storage_handler.next(&mut self.metrics) { Some(s) => s, _ => return Ok(Status::Normal), }; // TODO(RT): This can fail when packet sizes > max_payload_size in config are written to disk. // This leads to force switching to normal mode. Increasing max_payload_size to bypass this - let publish = match read(storage.reader(), max_packet_size) { + let publish = match Packet::read(storage.reader(), max_packet_size) { Ok(Packet::Publish(publish)) => publish, Ok(packet) => unreachable!("Unexpected packet: {:?}", packet), Err(e) => { @@ -371,6 +456,7 @@ impl Serializer { save_and_prepare_next_metrics( &mut self.pending_metrics, &mut self.metrics, + &mut self.stream_metrics, &self.storage_handler, ); return Ok(Status::Normal); @@ -378,6 +464,7 @@ impl Serializer { }; let mut last_publish_payload_size = publish.payload.len(); + let mut last_publish_stream = stream.clone(); let send = send_publish(client, publish.topic, publish.payload); tokio::pin!(send); @@ -385,9 +472,10 @@ impl Serializer { select! { data = self.collector_rx.recv_async() => { let data = data?; - let publish = construct_publish(data)?; - let storage = self.storage_handler.select(&publish.topic); - match write_to_disk(publish, storage) { + let stream = data.stream_config(); + let publish = construct_publish(data, &mut self.stream_metrics)?; + let storage = self.storage_handler.select(&stream); + match write_to_storage(publish, storage) { Ok(Some(deleted)) => { debug!("Lost segment = {deleted}"); self.metrics.increment_lost_segments(); @@ -408,16 +496,16 @@ impl Serializer { // indefinitely write to disk to not loose data let client = match o { Ok(c) => c, - Err(MqttError::Send(Request::Publish(publish))) => break Ok(Status::EventLoopCrash(publish)), + Err(MqttError::Send(Request::Publish(publish))) => break Ok(Status::EventLoopCrash(publish, last_publish_stream.clone())), Err(e) => unreachable!("Unexpected error: {}", e), }; - let storage = match self.storage_handler.next(&mut self.metrics) { + let (stream, storage) = match self.storage_handler.next(&mut self.metrics) { Some(s) => s, _ => return Ok(Status::Normal), }; - let publish = match read(storage.reader(), max_packet_size) { + let publish = match Packet::read(storage.reader(), max_packet_size) { Ok(Packet::Publish(publish)) => publish, Ok(packet) => unreachable!("Unexpected packet: {:?}", packet), Err(e) => { @@ -430,22 +518,28 @@ impl Serializer { let payload = publish.payload; last_publish_payload_size = payload.len(); + last_publish_stream = stream.clone(); send.set(send_publish(client, publish.topic, payload)); } // On a regular interval, forwards metrics information to network _ = interval.tick() => { let _ = check_and_flush_metrics(&mut self.pending_metrics, &mut self.metrics, &self.metrics_tx, &self.storage_handler); } + // Transition into crash mode when uplink is shutting down + Ok(SerializerShutdown) = self.ctrl_rx.recv_async() => { + return Ok(Status::Shutdown) + } } }; save_and_prepare_next_metrics( &mut self.pending_metrics, &mut self.metrics, + &mut self.stream_metrics, &self.storage_handler, ); - let v = v?; - Ok(v) + + v } async fn normal(&mut self) -> Result { @@ -458,16 +552,17 @@ impl Serializer { select! { data = self.collector_rx.recv_async() => { let data = data?; - let publish = construct_publish(data)?; + let stream = data.stream_config(); + let publish = construct_publish(data, &mut self.stream_metrics)?; let payload_size = publish.payload.len(); debug!("publishing on {} with size = {}", publish.topic, payload_size); - match self.client.try_publish(publish.topic, QoS::AtLeastOnce, false, publish.payload) { + match self.client.try_publish(&stream.topic, QoS::AtLeastOnce, false, publish.payload) { Ok(_) => { self.metrics.add_batch(); self.metrics.add_sent_size(payload_size); continue; } - Err(MqttError::TrySend(Request::Publish(publish))) => return Ok(Status::SlowEventloop(publish)), + Err(MqttError::TrySend(Request::Publish(publish))) => return Ok(Status::SlowEventloop(publish, stream)), Err(e) => unreachable!("Unexpected error: {}", e), } @@ -481,6 +576,10 @@ impl Serializer { debug!("Failed to flush serializer metrics (normal). Error = {}", e); } } + // Transition into crash mode when uplink is shutting down + Ok(SerializerShutdown) = self.ctrl_rx.recv_async() => { + return Ok(Status::Shutdown) + } } } } @@ -492,13 +591,20 @@ impl Serializer { loop { let next_status = match status { Status::Normal => self.normal().await?, - Status::SlowEventloop(publish) => self.slow(publish).await?, + Status::SlowEventloop(publish, stream) => self.slow(publish, stream).await?, Status::EventLoopReady => self.catchup().await?, - Status::EventLoopCrash(publish) => self.crash(publish).await?, + Status::EventLoopCrash(publish, stream) => self.crash(publish, stream).await?, + Status::Shutdown => break, }; status = next_status; } + + self.shutdown()?; + + info!("Serializer has handled all pending packets, shutting down"); + + Ok(()) } } @@ -521,26 +627,47 @@ fn lz4_compress(payload: &mut Vec) -> Result<(), Error> { } // Constructs a [Publish] packet given a [Package] element. Updates stream metrics as necessary. -fn construct_publish(data: Box) -> Result { - let stream = data.stream().as_ref().to_owned(); +fn construct_publish( + data: Box, + stream_metrics: &mut HashMap, +) -> Result { + let stream_name = data.stream_name().as_ref().to_owned(); + let stream_config = data.stream_config(); let point_count = data.len(); let batch_latency = data.latency(); - trace!("Data received on stream: {stream}; message count = {point_count}; batching latency = {batch_latency}"); + trace!("Data received on stream: {stream_name}; message count = {point_count}; batching latency = {batch_latency}"); - let topic = data.topic().to_string(); + let topic = stream_config.topic.clone(); + + let metrics = stream_metrics + .entry(stream_name.clone()) + .or_insert_with(|| StreamMetrics::new(&stream_name)); + + let serialization_start = Instant::now(); let mut payload = data.serialize()?; + let serialization_time = serialization_start.elapsed(); + metrics.add_serialization_time(serialization_time); - if let Compression::Lz4 = data.compression() { + let data_size = payload.len(); + let mut compressed_data_size = None; + + if let Compression::Lz4 = stream_config.compression { + let compression_start = Instant::now(); lz4_compress(&mut payload)?; + let compression_time = compression_start.elapsed(); + metrics.add_compression_time(compression_time); + + compressed_data_size = Some(payload.len()); } + metrics.add_serialized_sizes(data_size, compressed_data_size); + Ok(Publish::new(topic, QoS::AtLeastOnce, payload)) } -// Writes the provided publish packet to disk with [Storage], after setting its pkid to 1. -// Updates serializer metrics with appropriate values on success, if asked to do so. -// Returns size in memory, size in disk, number of files in disk, -fn write_to_disk( +// Writes the provided publish packet to [Storage], after setting its pkid to 1. +// If the write buffer is full, it is flushed/written onto disk based on config. +fn write_to_storage( mut publish: Publish, storage: &mut Storage, ) -> Result, storage::Error> { @@ -554,135 +681,204 @@ fn write_to_disk( Ok(deleted) } -fn check_metrics(metrics: &mut SerializerMetrics, storage_handler: &StorageHandler) { +fn check_metrics( + metrics: &mut Metrics, + stream_metrics: &mut HashMap, + storage_handler: &StorageHandler, +) { use pretty_bytes::converter::convert; let mut inmemory_write_size = 0; let mut inmemory_read_size = 0; let mut file_count = 0; + let mut disk_utilized = 0; for storage in storage_handler.map.values() { inmemory_read_size += storage.inmemory_read_size(); inmemory_write_size += storage.inmemory_write_size(); file_count += storage.file_count(); + disk_utilized += storage.disk_utilized(); } metrics.set_write_memory(inmemory_write_size); metrics.set_read_memory(inmemory_read_size); metrics.set_disk_files(file_count); + metrics.set_disk_utilized(disk_utilized); info!( - "{:>17}: batches = {:<3} errors = {} lost = {} disk_files = {:<3} write_memory = {} read_memory = {}", + "{:>17}: batches = {:<3} errors = {} lost = {} disk_files = {:<3} disk_utilized = {} write_memory = {} read_memory = {}", metrics.mode, metrics.batches, metrics.errors, metrics.lost_segments, metrics.disk_files, + convert(metrics.disk_utilized as f64), convert(metrics.write_memory as f64), convert(metrics.read_memory as f64), ); + + for metrics in stream_metrics.values_mut() { + metrics.prepare_snapshot(); + info!( + "{:>17}: serialized_data_size = {} compressed_data_size = {} avg_serialization_time = {}us avg_compression_time = {}us", + metrics.stream, + convert(metrics.serialized_data_size as f64), + convert(metrics.compressed_data_size as f64), + metrics.avg_serialization_time.as_micros(), + metrics.avg_compression_time.as_micros() + ); + } } fn save_and_prepare_next_metrics( pending: &mut VecDeque, - metrics: &mut SerializerMetrics, + metrics: &mut Metrics, + stream_metrics: &mut HashMap, storage_handler: &StorageHandler, ) { let mut inmemory_write_size = 0; let mut inmemory_read_size = 0; let mut file_count = 0; + let mut disk_utilized = 0; for storage in storage_handler.map.values() { inmemory_write_size += storage.inmemory_write_size(); inmemory_read_size += storage.inmemory_read_size(); file_count += storage.file_count(); + disk_utilized += storage.disk_utilized(); } metrics.set_write_memory(inmemory_write_size); metrics.set_read_memory(inmemory_read_size); metrics.set_disk_files(file_count); + metrics.set_disk_utilized(disk_utilized); - let m = metrics.clone(); - pending.push_back(m); + let m = Box::new(metrics.clone()); + pending.push_back(SerializerMetrics::Main(m)); metrics.prepare_next(); + + for metrics in stream_metrics.values_mut() { + metrics.prepare_snapshot(); + let m = Box::new(metrics.clone()); + pending.push_back(SerializerMetrics::Stream(m)); + metrics.prepare_next(); + } } // // Enable actual metrics timers when there is data. This method is called every minute by the bridge fn check_and_flush_metrics( pending: &mut VecDeque, - metrics: &mut SerializerMetrics, + metrics: &mut Metrics, metrics_tx: &Sender, storage_handler: &StorageHandler, -) -> Result<(), flume::TrySendError> { +) -> Result<(), TrySendError> { use pretty_bytes::converter::convert; let mut inmemory_write_size = 0; let mut inmemory_read_size = 0; let mut file_count = 0; + let mut disk_utilized = 0; for storage in storage_handler.map.values() { inmemory_write_size += storage.inmemory_write_size(); inmemory_read_size += storage.inmemory_read_size(); file_count += storage.file_count(); + disk_utilized += storage.disk_utilized(); } metrics.set_write_memory(inmemory_write_size); metrics.set_read_memory(inmemory_read_size); metrics.set_disk_files(file_count); + metrics.set_disk_utilized(disk_utilized); // Send pending metrics. This signifies state change - while let Some(metrics) = pending.get(0) { - // Always send pending metrics. They represent state changes - info!( - "{:>17}: batches = {:<3} errors = {} lost = {} disk_files = {:<3} write_memory = {} read_memory = {}", - metrics.mode, - metrics.batches, - metrics.errors, - metrics.lost_segments, - metrics.disk_files, - convert(metrics.write_memory as f64), - convert(metrics.read_memory as f64), - ); - metrics_tx.try_send(metrics.clone())?; - pending.pop_front(); + while let Some(metrics) = pending.front() { + match metrics { + SerializerMetrics::Main(metrics) => { + // Always send pending metrics. They represent state changes + info!( + "{:>17}: batches = {:<3} errors = {} lost = {} disk_files = {:<3} disk_utilized = {} write_memory = {} read_memory = {}", + metrics.mode, + metrics.batches, + metrics.errors, + metrics.lost_segments, + metrics.disk_files, + convert(metrics.disk_utilized as f64), + convert(metrics.write_memory as f64), + convert(metrics.read_memory as f64), + ); + metrics_tx.try_send(SerializerMetrics::Main(metrics.clone()))?; + pending.pop_front(); + } + SerializerMetrics::Stream(metrics) => { + // Always send pending metrics. They represent state changes + info!( + "{:>17}: serialized_data_size = {} compressed_data_size = {} avg_serialization_time = {}us avg_compression_time = {}us", + metrics.stream, + convert(metrics.serialized_data_size as f64), + convert(metrics.compressed_data_size as f64), + metrics.avg_serialization_time.as_micros(), + metrics.avg_compression_time.as_micros() + ); + metrics_tx.try_send(SerializerMetrics::Stream(metrics.clone()))?; + pending.pop_front(); + } + } } if metrics.batches() > 0 { info!( - "{:>17}: batches = {:<3} errors = {} lost = {} disk_files = {:<3} write_memory = {} read_memory = {}", + "{:>17}: batches = {:<3} errors = {} lost = {} disk_files = {:<3} disk_utilized = {} write_memory = {} read_memory = {}", metrics.mode, metrics.batches, metrics.errors, metrics.lost_segments, metrics.disk_files, + convert(metrics.disk_utilized as f64), convert(metrics.write_memory as f64), convert(metrics.read_memory as f64), ); - metrics_tx.try_send(metrics.clone())?; + metrics_tx.try_send(SerializerMetrics::Main(Box::new(metrics.clone())))?; metrics.prepare_next(); } Ok(()) } +/// Command to remotely trigger `Serializer` shutdown +pub(crate) struct SerializerShutdown; + +/// Handle to send control messages to `Serializer` +#[derive(Debug, Clone)] +pub struct CtrlTx { + pub(crate) inner: Sender, +} + +impl CtrlTx { + /// Triggers shutdown of `Serializer` + pub async fn trigger_shutdown(&self) { + self.inner.send_async(SerializerShutdown).await.unwrap() + } +} + // TODO(RT): Test cases // - Restart with no internet but files on disk #[cfg(test)] mod test { use serde_json::Value; + use tokio::spawn; - use std::collections::HashMap; - use std::time::Duration; + use crate::{ + base::bridge::{stream::Stream, Payload}, + config::MqttConfig, + }; use super::*; - use crate::base::bridge::stream::Stream; - use crate::base::MqttConfig; - use crate::Payload; #[derive(Clone)] pub struct MockClient { - pub net_tx: flume::Sender, + pub net_tx: Sender, } #[async_trait::async_trait] @@ -729,7 +925,7 @@ mod test { panic!("No publishes found in storage"); } - match read(storage.reader(), max_packet_size) { + match Packet::read(storage.reader(), max_packet_size) { Ok(Packet::Publish(publish)) => return publish, v => { panic!("Failed to read publish from storage. read: {:?}", v); @@ -750,10 +946,10 @@ mod test { fn defaults( config: Arc, - ) -> (Serializer, flume::Sender>, Receiver) { - let (data_tx, data_rx) = flume::bounded(1); - let (net_tx, net_rx) = flume::bounded(1); - let (metrics_tx, _metrics_rx) = flume::bounded(1); + ) -> (Serializer, Sender>, Receiver) { + let (data_tx, data_rx) = bounded(1); + let (net_tx, net_rx) = bounded(1); + let (metrics_tx, _metrics_rx) = bounded(1); let client = MockClient { net_tx }; (Serializer::new(config, data_rx, client, metrics_tx).unwrap(), data_tx, net_rx) @@ -772,15 +968,17 @@ mod test { } impl MockCollector { - fn new(data_tx: flume::Sender>) -> MockCollector { - MockCollector { - stream: Stream::new("hello", "hello/world", 1, data_tx, Compression::Disabled), - } + fn new( + stream_name: &str, + stream_config: StreamConfig, + data_tx: Sender>, + ) -> MockCollector { + MockCollector { stream: Stream::new(stream_name, stream_config, data_tx) } } fn send(&mut self, i: u32) -> Result<(), Error> { let payload = Payload { - stream: "hello".to_owned(), + stream: Default::default(), sequence: i, timestamp: 0, payload: serde_json::from_str("{\"msg\": \"Hello, World!\"}")?, @@ -803,7 +1001,11 @@ mod test { net_rx.recv().unwrap(); }); - let mut collector = MockCollector::new(data_tx); + let (stream_name, stream_config) = ( + "hello", + StreamConfig { topic: "hello/world".to_string(), batch_size: 1, ..Default::default() }, + ); + let mut collector = MockCollector::new(stream_name, stream_config, data_tx); std::thread::spawn(move || { for i in 1..3 { collector.send(i).unwrap(); @@ -811,7 +1013,7 @@ mod test { }); match tokio::runtime::Runtime::new().unwrap().block_on(serializer.normal()).unwrap() { - Status::SlowEventloop(Publish { qos: QoS::AtLeastOnce, topic, payload, .. }) => { + Status::SlowEventloop(Publish { qos: QoS::AtLeastOnce, topic, payload, .. }, _) => { assert_eq!(topic, "hello/world"); let recvd: Value = serde_json::from_slice(&payload).unwrap(); let obj = &recvd.as_array().unwrap()[0]; @@ -827,14 +1029,14 @@ mod test { let config = Arc::new(default_config()); let (serializer, _, _) = defaults(config); - let mut storage = Storage::new(1024); + let mut storage = Storage::new("hello/world", 1024); let mut publish = Publish::new( "hello/world", QoS::AtLeastOnce, "[{\"sequence\":2,\"timestamp\":0,\"msg\":\"Hello, World!\"}]".as_bytes(), ); - write_to_disk(publish.clone(), &mut storage).unwrap(); + write_to_storage(publish.clone(), &mut storage).unwrap(); let stored_publish = read_from_storage(&mut storage, serializer.config.mqtt.max_packet_size); @@ -857,7 +1059,11 @@ mod test { net_rx.recv().unwrap(); }); - let mut collector = MockCollector::new(data_tx); + let (stream_name, stream_config) = ( + "hello", + StreamConfig { topic: "hello/world".to_string(), batch_size: 1, ..Default::default() }, + ); + let mut collector = MockCollector::new(stream_name, stream_config, data_tx); // Faster collector, send data every 5s std::thread::spawn(move || { for i in 1..10 { @@ -871,8 +1077,10 @@ mod test { QoS::AtLeastOnce, "[{{\"sequence\":1,\"timestamp\":0,\"msg\":\"Hello, World!\"}}]".as_bytes(), ); - let status = - tokio::runtime::Runtime::new().unwrap().block_on(serializer.slow(publish)).unwrap(); + let status = tokio::runtime::Runtime::new() + .unwrap() + .block_on(serializer.slow(publish, Arc::new(Default::default()))) + .unwrap(); assert_eq!(status, Status::EventLoopReady); } @@ -883,7 +1091,11 @@ mod test { let config = Arc::new(default_config()); let (mut serializer, data_tx, _) = defaults(config); - let mut collector = MockCollector::new(data_tx); + let (stream_name, stream_config) = ( + "hello", + StreamConfig { topic: "hello/world".to_string(), batch_size: 1, ..Default::default() }, + ); + let mut collector = MockCollector::new(stream_name, stream_config, data_tx); // Faster collector, send data every 5s std::thread::spawn(move || { for i in 1..10 { @@ -898,8 +1110,15 @@ mod test { "[{\"sequence\":1,\"timestamp\":0,\"msg\":\"Hello, World!\"}]".as_bytes(), ); - match tokio::runtime::Runtime::new().unwrap().block_on(serializer.slow(publish)).unwrap() { - Status::EventLoopCrash(Publish { qos: QoS::AtLeastOnce, topic, payload, .. }) => { + match tokio::runtime::Runtime::new() + .unwrap() + .block_on(serializer.slow( + publish, + Arc::new(StreamConfig { topic: "hello/world".to_string(), ..Default::default() }), + )) + .unwrap() + { + Status::EventLoopCrash(Publish { qos: QoS::AtLeastOnce, topic, payload, .. }, _) => { assert_eq!(topic, "hello/world"); let recvd = std::str::from_utf8(&payload).unwrap(); assert_eq!(recvd, "[{\"sequence\":1,\"timestamp\":0,\"msg\":\"Hello, World!\"}]"); @@ -930,10 +1149,14 @@ mod test { let mut storage = serializer .storage_handler .map - .entry("hello/world".to_string()) - .or_insert(Storage::new(1024)); + .entry(Arc::new(Default::default())) + .or_insert(Storage::new("hello/world", 1024)); - let mut collector = MockCollector::new(data_tx); + let (stream_name, stream_config) = ( + "hello", + StreamConfig { topic: "hello/world".to_string(), batch_size: 1, ..Default::default() }, + ); + let mut collector = MockCollector::new(stream_name, stream_config, data_tx); // Run a collector practically once std::thread::spawn(move || { for i in 2..6 { @@ -965,7 +1188,7 @@ mod test { QoS::AtLeastOnce, "[{\"sequence\":1,\"timestamp\":0,\"msg\":\"Hello, World!\"}]".as_bytes(), ); - write_to_disk(publish.clone(), &mut storage).unwrap(); + write_to_storage(publish.clone(), &mut storage).unwrap(); let status = tokio::runtime::Runtime::new().unwrap().block_on(serializer.catchup()).unwrap(); @@ -982,10 +1205,17 @@ mod test { let mut storage = serializer .storage_handler .map - .entry("hello/world".to_string()) - .or_insert(Storage::new(1024)); - - let mut collector = MockCollector::new(data_tx); + .entry(Arc::new(StreamConfig { + topic: "hello/world".to_string(), + ..Default::default() + })) + .or_insert(Storage::new("hello/world", 1024)); + + let (stream_name, stream_config) = ( + "hello", + StreamConfig { topic: "hello/world".to_string(), batch_size: 1, ..Default::default() }, + ); + let mut collector = MockCollector::new(stream_name, stream_config, data_tx); // Run a collector std::thread::spawn(move || { for i in 2..6 { @@ -1000,10 +1230,10 @@ mod test { QoS::AtLeastOnce, "[{\"sequence\":1,\"timestamp\":0,\"msg\":\"Hello, World!\"}]".as_bytes(), ); - write_to_disk(publish.clone(), &mut storage).unwrap(); + write_to_storage(publish.clone(), &mut storage).unwrap(); match tokio::runtime::Runtime::new().unwrap().block_on(serializer.catchup()).unwrap() { - Status::EventLoopCrash(Publish { topic, payload, .. }) => { + Status::EventLoopCrash(Publish { topic, payload, .. }, _) => { assert_eq!(topic, "hello/world"); let recvd = std::str::from_utf8(&payload).unwrap(); assert_eq!(recvd, "[{\"sequence\":1,\"timestamp\":0,\"msg\":\"Hello, World!\"}]"); @@ -1011,4 +1241,147 @@ mod test { s => unreachable!("Unexpected status: {:?}", s), } } + + #[tokio::test] + // Ensures that the data of streams are removed on the basis of preference + async fn preferential_send_on_network() { + let mut config = default_config(); + config.stream_metrics.timeout = Duration::from_secs(1000); + config.streams.extend([ + ( + "one".to_owned(), + StreamConfig { topic: "topic/one".to_string(), priority: 1, ..Default::default() }, + ), + ( + "two".to_owned(), + StreamConfig { topic: "topic/two".to_string(), priority: 2, ..Default::default() }, + ), + ( + "top".to_owned(), + StreamConfig { + topic: "topic/top".to_string(), + priority: u8::MAX, + ..Default::default() + }, + ), + ]); + let config = Arc::new(config); + + let (mut serializer, _data_tx, req_rx) = defaults(config.clone()); + + let publish = |topic: String, i: u32| Publish { + dup: false, + qos: QoS::AtMostOnce, + retain: false, + topic, + pkid: 0, + payload: Bytes::from(i.to_string()), + }; + + let mut one = serializer + .storage_handler + .map + .entry(Arc::new(StreamConfig { + topic: "topic/one".to_string(), + priority: 1, + ..Default::default() + })) + .or_insert_with(|| unreachable!()); + write_to_storage(publish("topic/one".to_string(), 1), &mut one).unwrap(); + write_to_storage(publish("topic/one".to_string(), 10), &mut one).unwrap(); + + let top = serializer + .storage_handler + .map + .entry(Arc::new(StreamConfig { + topic: "topic/top".to_string(), + priority: u8::MAX, + ..Default::default() + })) + .or_insert_with(|| unreachable!()); + write_to_storage(publish("topic/top".to_string(), 100), top).unwrap(); + write_to_storage(publish("topic/top".to_string(), 1000), top).unwrap(); + + let two = serializer + .storage_handler + .map + .entry(Arc::new(StreamConfig { + topic: "topic/two".to_string(), + priority: 2, + ..Default::default() + })) + .or_insert_with(|| unreachable!()); + write_to_storage(publish("topic/two".to_string(), 3), two).unwrap(); + + let mut default = serializer + .storage_handler + .map + .entry(Arc::new(StreamConfig { + topic: "topic/default".to_string(), + priority: 0, + ..Default::default() + })) + .or_insert(Storage::new("topic/default", 1024)); + write_to_storage(publish("topic/default".to_string(), 0), &mut default).unwrap(); + write_to_storage(publish("topic/default".to_string(), 2), &mut default).unwrap(); + + // run serializer in the background + spawn(async { serializer.start().await.unwrap() }); + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/top"); + assert_eq!(payload, "100"); + } + _ => unreachable!(), + } + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/top"); + assert_eq!(payload, "1000"); + } + _ => unreachable!(), + } + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/two"); + assert_eq!(payload, "3"); + } + _ => unreachable!(), + } + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/one"); + assert_eq!(payload, "1"); + } + _ => unreachable!(), + } + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/one"); + assert_eq!(payload, "10"); + } + _ => unreachable!(), + } + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/default"); + assert_eq!(payload, "0"); + } + _ => unreachable!(), + } + + match req_rx.recv_async().await.unwrap() { + Request::Publish(Publish { topic, payload, .. }) => { + assert_eq!(topic, "topic/default"); + assert_eq!(payload, "2"); + } + _ => unreachable!(), + } + } } diff --git a/uplink/src/collector/clickhouse.rs b/uplink/src/collector/clickhouse.rs index 48f01729a..bd1146745 100644 --- a/uplink/src/collector/clickhouse.rs +++ b/uplink/src/collector/clickhouse.rs @@ -6,9 +6,12 @@ use tokio::{ time::{interval, Duration}, }; -use crate::base::{ - bridge::{BridgeTx, Payload}, - clock, ClickhouseConfig, QueryLogConfig, +use crate::{ + base::{ + bridge::{BridgeTx, Payload}, + clock, + }, + config::{ClickhouseConfig, QueryLogConfig}, }; #[derive(Debug, clickhouse::Row, Serialize, Deserialize)] @@ -112,7 +115,7 @@ impl QueryLogReader { debug!("Row: {row:?}"); let mut payload: Payload = row.into(); payload.timestamp = clock() as u64; - payload.stream = self.config.stream.to_owned(); + self.config.stream.clone_into(&mut payload.stream); self.sequence += 1; payload.sequence = self.sequence; self.bridge_tx.send_payload(payload).await; diff --git a/uplink/src/collector/device_shadow.rs b/uplink/src/collector/device_shadow.rs index 96d958183..6e01f0ea9 100644 --- a/uplink/src/collector/device_shadow.rs +++ b/uplink/src/collector/device_shadow.rs @@ -3,8 +3,8 @@ use std::time::Duration; use log::{error, trace}; use serde::Serialize; -use crate::base::DeviceShadowConfig; use crate::base::{bridge::BridgeTx, clock}; +use crate::config::DeviceShadowConfig; use crate::Payload; pub const UPLINK_VERSION: &str = env!("VERGEN_BUILD_SEMVER"); diff --git a/uplink/src/collector/downloader.rs b/uplink/src/collector/downloader.rs index 7ce0877db..b121a6ca3 100644 --- a/uplink/src/collector/downloader.rs +++ b/uplink/src/collector/downloader.rs @@ -48,15 +48,19 @@ //! [`action_redirections`]: Config#structfield.action_redirections use bytes::BytesMut; +use flume::{Receiver, Sender}; use futures_util::StreamExt; -use log::{error, info, warn}; -use reqwest::{Certificate, Client, ClientBuilder, Identity, Response}; +use human_bytes::human_bytes; +use log::{debug, error, info, trace, warn}; +use reqwest::{Certificate, Client, ClientBuilder, Error as ReqwestError, Identity}; +use rsa::sha2::{Digest, Sha256}; use serde::{Deserialize, Serialize}; -use tokio::time::timeout; +use tokio::select; +use tokio::time::{sleep, timeout_at, Instant}; -use std::collections::HashMap; -use std::fs::{metadata, remove_dir_all, File}; -use std::sync::Arc; +use std::fs::{metadata, read, remove_dir_all, remove_file, write, File}; +use std::io; +use std::sync::{Arc, Mutex}; use std::time::Duration; #[cfg(unix)] use std::{ @@ -65,16 +69,14 @@ use std::{ }; use std::{io::Write, path::PathBuf}; -use crate::base::bridge::BridgeTx; -use crate::base::DownloaderConfig; -use crate::{Action, ActionResponse, Config}; +use crate::{base::bridge::BridgeTx, config::DownloaderConfig, Action, ActionResponse, Config}; #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Serde error: {0}")] Serde(#[from] serde_json::Error), #[error("Error from reqwest: {0}")] - Reqwest(#[from] reqwest::Error), + Reqwest(#[from] ReqwestError), #[error("File io Error: {0}")] Io(#[from] std::io::Error), #[error("Empty file name")] @@ -83,6 +85,16 @@ pub enum Error { FilePathMissing, #[error("Download failed, content length zero")] EmptyFile, + #[error("Downloaded file has unexpected checksum")] + BadChecksum, + #[error("Disk space is insufficient: {0}")] + InsufficientDisk(String), + #[error("Save file is corrupted")] + BadSave, + #[error("Save file doesn't exist")] + NoSave, + #[error("Download timedout")] + Timeout, } /// This struct contains the necessary components to download and store file as notified by a download file @@ -91,16 +103,23 @@ pub enum Error { /// to the connected bridge application. pub struct FileDownloader { config: DownloaderConfig, + actions_rx: Receiver, action_id: String, bridge_tx: BridgeTx, client: Client, - sequence: u32, - timeouts: HashMap, + shutdown_rx: Receiver, + disabled: Arc>, } impl FileDownloader { /// Creates a handler for download actions within uplink and uses HTTP to download files. - pub fn new(config: Arc, bridge_tx: BridgeTx) -> Result { + pub fn new( + config: Arc, + actions_rx: Receiver, + bridge_tx: BridgeTx, + shutdown_rx: Receiver, + disabled: Arc>, + ) -> Result { // Authenticate with TLS certs from config let client_builder = ClientBuilder::new(); let client = match &config.authentication { @@ -116,20 +135,14 @@ impl FileDownloader { } .build()?; - let timeouts = config - .downloader - .actions - .iter() - .map(|s| (s.name.to_owned(), Duration::from_secs(s.timeout))) - .collect(); - Ok(Self { config: config.downloader.clone(), - timeouts, + actions_rx, client, bridge_tx, - sequence: 0, action_id: String::default(), + shutdown_rx, + disabled, }) } @@ -137,224 +150,424 @@ impl FileDownloader { /// back to bridge for further processing, e.g. OTA update installation. #[tokio::main(flavor = "current_thread")] pub async fn start(mut self) { - let routes = &self.config.actions; - let download_rx = match self.bridge_tx.register_action_routes(routes).await { - Some(r) => r, - _ => return, - }; + self.reload().await; info!("Downloader thread is ready to receive download actions"); - loop { - self.sequence = 0; - let action = match download_rx.recv_async().await { - Ok(a) => a, + while let Ok(action) = self.actions_rx.recv_async().await { + action.action_id.clone_into(&mut self.action_id); + let mut state = match DownloadState::new(action, &self.config) { + Ok(s) => s, Err(e) => { - error!("Downloader thread had to stop: {e}"); - break; - } - }; - self.action_id = action.action_id.clone(); - - let duration = match self.timeouts.get(&action.name) { - Some(t) => *t, - _ => { - error!("Action: {} unconfigured", action.name); + self.forward_error(e).await; continue; } }; - // NOTE: if download has timedout don't do anything, else ensure errors are forwarded after three retries - match timeout(duration, self.retry_thrice(action)).await { - Ok(Err(e)) => self.forward_error(e).await, - Err(_) => error!("Last download has timedout"), - _ => {} + // Update action status for process initiated + let status = ActionResponse::progress(&self.action_id, "Downloading", 0); + self.bridge_tx.send_action_response(status).await; + + if let Err(e) = self.download(&mut state).await { + self.forward_error(e).await; + continue; } + + // Forward updated action as part of response + let DownloadState { current: CurrentDownload { action, .. }, .. } = state; + let status = ActionResponse::done(&self.action_id, "Downloaded", Some(action)); + self.bridge_tx.send_action_response(status).await; } + + error!("Downloader thread stopped"); } - // Forward errors as action response to bridge - async fn forward_error(&mut self, err: Error) { - let status = - ActionResponse::failure(&self.action_id, err.to_string()).set_sequence(self.sequence()); + // Loads a download left uncompleted during the previous run of uplink and continues it + async fn reload(&mut self) { + let mut state = match DownloadState::load(&self.config) { + Ok(s) => s, + Err(Error::NoSave) => return, + Err(e) => { + warn!("Couldn't reload current_download: {e}"); + return; + } + }; + state.current.action.action_id.clone_into(&mut self.action_id); + + if let Err(e) = self.download(&mut state).await { + self.forward_error(e).await; + return; + } + + // Forward updated action as part of response + let DownloadState { current: CurrentDownload { action, .. }, .. } = state; + let status = ActionResponse::done(&self.action_id, "Downloaded", Some(action)); self.bridge_tx.send_action_response(status).await; } - // Retry mechanism tries atleast 3 times before returning an error - async fn retry_thrice(&mut self, action: Action) -> Result<(), Error> { - for _ in 0..3 { - match self.run(action.clone()).await { - Ok(_) => break, + // Accepts `DownloadState`, sets a timeout for the action + async fn download(&mut self, state: &mut DownloadState) -> Result<(), Error> { + let shutdown_rx = self.shutdown_rx.clone(); + let deadline = match &state.current.action.deadline { + Some(d) => *d, + _ => { + error!("Unconfigured deadline: {}", state.current.action.name); + return Ok(()); + } + }; + select! { + Ok(_) = shutdown_rx.recv_async(), if !shutdown_rx.is_disconnected() => { + if let Err(e) = state.save(&self.config) { + error!("Error saving current_download: {e}"); + } + + return Ok(()); + }, + + // NOTE: if download has timedout don't do anything, else ensure errors are forwarded after three retries + o = timeout_at(deadline, self.continuous_retry(state)) => match o { + Ok(r) => r?, + Err(_) => { + // unwrap is safe because download_path is expected to be Some + _ = remove_file(state.current.meta.download_path.as_ref().unwrap()); + error!("Last download has timedout; file deleted"); + + return Err(Error::Timeout); + }, + } + } + + state.current.meta.verify_checksum()?; + // Update Action payload with `download_path`, i.e. downloaded file's location in fs + state.current.action.payload = serde_json::to_string(&state.current.meta)?; + + Ok(()) + } + + // A download must be retried with Range header when HTTP/reqwest errors are faced + async fn continuous_retry(&mut self, state: &mut DownloadState) -> Result<(), Error> { + 'outer: loop { + let mut req = self.client.get(&state.current.meta.url); + if let Some(range) = state.retry_range() { + warn!("Retrying download; Continuing to download file from: {range}"); + req = req.header("Range", range); + } + let mut stream = match req.send().await { + Ok(s) => s.error_for_status()?.bytes_stream(), Err(e) => { - if let Error::Reqwest(e) = e { + error!("Download failed: {e}"); + // Retry after wait + tokio::time::sleep(Duration::from_secs(1)).await; + continue 'outer; + } + }; + + // Download and store to disk by streaming as chunks + loop { + // Checks if downloader is disabled by user or not + if *self.disabled.lock().unwrap() { + // async to ensure download can be cancelled during sleep + sleep(Duration::from_secs(1)).await; + continue; + } + let Some(item) = stream.next().await else { break }; + let chunk = match item { + Ok(c) => c, + // Retry non-status errors + Err(e) if !e.is_status() => { + let status = + ActionResponse::progress(&self.action_id, "Download Failed", 0) + .add_error(e.to_string()); + self.bridge_tx.send_action_response(status).await; error!("Download failed: {e}"); - } else { - return Err(e); + // Retry after wait + tokio::time::sleep(Duration::from_secs(1)).await; + continue 'outer; } + Err(e) => return Err(e.into()), + }; + if let Some(percentage) = state.write_bytes(&chunk)? { + let status = + ActionResponse::progress(&self.action_id, "Downloading", percentage); + self.bridge_tx.send_action_response(status).await; } } - tokio::time::sleep(Duration::from_secs(30)).await; - warn!("Retrying download"); + + info!("Firmware downloaded successfully"); + break; } Ok(()) } - // Accepts a download `Action` and performs necessary data extraction to actually download the file - async fn run(&mut self, mut action: Action) -> Result<(), Error> { - // Update action status for process initiated - let status = ActionResponse::progress(&self.action_id, "Downloading", 0); - let status = status.set_sequence(self.sequence()); + // Forward errors as action response to bridge + async fn forward_error(&mut self, err: Error) { + let status = ActionResponse::failure(&self.action_id, err.to_string()); self.bridge_tx.send_action_response(status).await; + } +} - // Extract url information from action payload - let mut update = match serde_json::from_str::(&action.payload)? { - DownloadFile { file_name, .. } if file_name.is_empty() => { - return Err(Error::EmptyFileName) - } - DownloadFile { content_length: 0, .. } => return Err(Error::EmptyFile), - u => u, - }; +#[cfg(unix)] +/// Custom create_dir_all which sets permissions on each created directory, only works on unix +fn create_dirs_with_perms(path: &Path, perms: Permissions) -> std::io::Result<()> { + let mut current_path = PathBuf::new(); - let url = update.url.clone(); + for component in path.components() { + current_path.push(component); - // Create file to actually download into - let (file, file_path) = self.create_file(&action.name, &update.file_name)?; + if !current_path.exists() { + create_dir(¤t_path)?; + set_permissions(¤t_path, perms.clone())?; + } + } - // Create handler to perform download from URL - // TODO: Error out for 1XX/3XX responses - let resp = self.client.get(&url).send().await?.error_for_status()?; - info!("Downloading from {} into {}", url, file_path); - self.download(resp, file, update.content_length).await?; + Ok(()) +} - // Update Action payload with `download_path`, i.e. downloaded file's location in fs - update.download_path = Some(file_path.clone()); - action.payload = serde_json::to_string(&update)?; +/// Creates file to download into +fn create_file(download_path: &PathBuf, file_name: &str) -> Result<(File, PathBuf), Error> { + let mut file_path = download_path.to_owned(); + file_path.push(file_name); + // NOTE: if file_path is occupied by a directory due to previous working of uplink, remove it + if let Ok(f) = metadata(&file_path) { + if f.is_dir() { + remove_dir_all(&file_path)?; + } + } + let file = File::create(&file_path)?; + #[cfg(unix)] + file.set_permissions(std::os::unix::fs::PermissionsExt::from_mode(0o666))?; - let status = ActionResponse::done(&self.action_id, "Downloaded", Some(action)); - let status = status.set_sequence(self.sequence()); - self.bridge_tx.send_action_response(status).await; + Ok((file, file_path)) +} - Ok(()) +fn check_disk_size(config: &DownloaderConfig, download: &DownloadFile) -> Result<(), Error> { + let disk_free_space = fs2::free_space(&config.path)? as usize; + + let req_size = human_bytes(download.content_length as f64); + let free_size = human_bytes(disk_free_space as f64); + debug!("Download requires {req_size}; Disk free space is {free_size}"); + + if download.content_length > disk_free_space { + return Err(Error::InsufficientDisk(free_size)); } - #[cfg(unix)] - fn create_dirs_with_perms(&self, path: &Path, perms: Permissions) -> std::io::Result<()> { - let mut current_path = PathBuf::new(); + Ok(()) +} - for component in path.components() { - current_path.push(component); +/// Expected JSON format of data contained in the [`payload`] of a download file [`Action`] +/// +/// [`payload`]: Action#structfield.payload +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] +pub struct DownloadFile { + url: String, + #[serde(alias = "content-length")] + content_length: usize, + #[serde(alias = "version")] + file_name: String, + /// Path to location in fs where file will be stored + pub download_path: Option, + /// Checksum that can be used to verify download was successful + pub checksum: Option, +} - if !current_path.exists() { - create_dir(¤t_path)?; - set_permissions(¤t_path, perms.clone())?; - } +impl DownloadFile { + fn verify_checksum(&self) -> Result<(), Error> { + let Some(checksum) = &self.checksum else { return Ok(()) }; + let path = self.download_path.as_ref().expect("Downloader didn't set \"download_path\""); + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + io::copy(&mut file, &mut hasher)?; + let hash = hasher.finalize(); + + if checksum != &hex::encode(hash) { + return Err(Error::BadChecksum); } Ok(()) } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct CurrentDownload { + action: Action, + meta: DownloadFile, + time_left: Option, +} - /// Creates file to download into - fn create_file(&self, name: &str, file_name: &str) -> Result<(File, String), Error> { +// A temporary structure to help us retry downloads +// that failed after partial completion. +#[derive(Debug)] +struct DownloadState { + current: CurrentDownload, + file: File, + bytes_written: usize, + percentage_downloaded: u8, + start: Instant, +} + +impl DownloadState { + fn new(action: Action, config: &DownloaderConfig) -> Result { // Ensure that directory for downloading file into, exists - let mut download_path = PathBuf::from(self.config.path.clone()); - download_path.push(name); - // do manual create_dir_all while setting permissions on each created directory + let mut path = config.path.clone(); + path.push(&action.name); #[cfg(unix)] - self.create_dirs_with_perms( - download_path.as_path(), + create_dirs_with_perms( + path.as_path(), std::os::unix::fs::PermissionsExt::from_mode(0o777), )?; #[cfg(not(unix))] - std::fs::create_dir_all(&download_path)?; - - let mut file_path = download_path.to_owned(); - file_path.push(file_name); - let file_path = file_path.as_path(); - // NOTE: if file_path is occupied by a directory due to previous working of uplink, remove it - if let Ok(f) = metadata(file_path) { - if f.is_dir() { - remove_dir_all(file_path)?; + std::fs::create_dir_all(&path)?; + + // Extract url information from action payload + let mut meta = match serde_json::from_str::(&action.payload)? { + DownloadFile { file_name, .. } if file_name.is_empty() => { + return Err(Error::EmptyFileName) } + DownloadFile { content_length: 0, .. } => return Err(Error::EmptyFile), + u => u, + }; + + check_disk_size(config, &meta)?; + + let url = meta.url.clone(); + + // Create file to actually download into + let (file, file_path) = create_file(&path, &meta.file_name)?; + // Retry downloading upto 3 times in case of connectivity issues + // TODO: Error out for 1XX/3XX responses + info!( + "Downloading from {} into {}; size = {}", + url, + file_path.display(), + human_bytes(meta.content_length as f64) + ); + meta.download_path = Some(file_path); + let current = CurrentDownload { action, meta, time_left: None }; + + Ok(Self { + current, + file, + bytes_written: 0, + percentage_downloaded: 0, + start: Instant::now(), + }) + } + + fn load(config: &DownloaderConfig) -> Result { + let mut path = config.path.clone(); + path.push("current_download"); + + if !path.exists() { + return Err(Error::NoSave); } - let file = File::create(file_path)?; - #[cfg(unix)] - file.set_permissions(std::os::unix::fs::PermissionsExt::from_mode(0o666))?; - let file_path = file_path.to_str().ok_or(Error::FilePathMissing)?.to_owned(); + let read = read(&path)?; + let mut current: CurrentDownload = serde_json::from_slice(&read)?; + // Calculate deadline based on written time left + current.action.deadline = current.time_left.map(|t| Instant::now() + t); + + let file = File::options().append(true).open(current.meta.download_path.as_ref().unwrap())?; + let bytes_written = file.metadata()?.len() as usize; - Ok((file, file_path)) + remove_file(path)?; + + Ok(DownloadState { + current, + file, + bytes_written, + percentage_downloaded: 0, + start: Instant::now(), + }) } - /// Downloads from server and stores into file - async fn download( - &mut self, - resp: Response, - mut file: File, - content_length: usize, - ) -> Result<(), Error> { - let mut downloaded = 0; - let mut next = 1; - let mut stream = resp.bytes_stream(); - - // Download and store to disk by streaming as chunks - while let Some(item) = stream.next().await { - let chunk = item?; - downloaded += chunk.len(); - file.write_all(&chunk)?; - - // Calculate percentage on the basis of content_length - let percentage = 99 * downloaded / content_length; - // NOTE: ensure lesser frequency of action responses, once every percentage points - if percentage >= next { - next += 1; - - //TODO: Simplify progress by reusing action_id and state - //TODO: let response = self.response.progress(percentage);?? - let status = - ActionResponse::progress(&self.action_id, "Downloading", percentage as u8); - let status = status.set_sequence(self.sequence()); - self.bridge_tx.send_action_response(status).await; - } + fn save(&self, config: &DownloaderConfig) -> Result<(), Error> { + if self.bytes_written == self.current.meta.content_length { + return Ok(()); } - info!("Firmware downloaded successfully"); + let mut current = self.current.clone(); + // Calculate time left based on deadline + current.time_left = current.action.deadline.map(|t| t.duration_since(Instant::now())); + let json = serde_json::to_vec(¤t)?; + + let mut path = config.path.clone(); + path.push("current_download"); + write(path, json)?; Ok(()) } - fn sequence(&mut self) -> u32 { - self.sequence += 1; - self.sequence + fn retry_range(&self) -> Option { + if self.bytes_written == 0 { + return None; + } + + Some(format!("bytes={}-{}", self.bytes_written, self.current.meta.content_length)) + } + + fn write_bytes(&mut self, buf: &[u8]) -> Result, Error> { + let bytes_downloaded = buf.len(); + self.file.write_all(buf)?; + self.bytes_written += bytes_downloaded; + let size = human_bytes(self.current.meta.content_length as f64); + + // Calculate percentage on the basis of content_length + let factor = self.bytes_written as f32 / self.current.meta.content_length as f32; + let percentage = (99.99 * factor) as u8; + + // NOTE: ensure lesser frequency of action responses, once every percentage points + if percentage > self.percentage_downloaded { + self.percentage_downloaded = percentage; + debug!( + "Downloading: size = {size}, percentage = {percentage}, elapsed = {}s", + self.start.elapsed().as_secs() + ); + + Ok(Some(percentage)) + } else { + trace!( + "Downloading: size = {size}, percentage = {}, elapsed = {}s", + self.percentage_downloaded, + self.start.elapsed().as_secs() + ); + + Ok(None) + } } } -/// Expected JSON format of data contained in the [`payload`] of a download file [`Action`] -/// -/// [`payload`]: Action#structfield.payload -#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] -pub struct DownloadFile { - url: String, - #[serde(alias = "content-length")] - content_length: usize, - #[serde(alias = "version")] - file_name: String, - /// Path to location in fs where file will be stored - pub download_path: Option, +/// Command to remotely trigger `Downloader` shutdown +pub struct DownloaderShutdown; + +/// Handle to send control messages to `Downloader` +#[derive(Debug, Clone)] +pub struct CtrlTx { + pub(crate) inner: Sender, +} + +impl CtrlTx { + /// Triggers shutdown of `Downloader` + pub async fn trigger_shutdown(&self) { + self.inner.send_async(DownloaderShutdown).await.unwrap() + } } #[cfg(test)] mod test { - use flume::{bounded, TrySendError}; + use std::collections::HashMap; + + use flume::bounded; use serde_json::json; + use tempdir::TempDir; - use std::{collections::HashMap, time::Duration}; + use crate::{ + base::bridge::{DataTx, StatusTx}, + config::{ActionRoute, MqttConfig}, + }; use super::*; - use crate::base::{bridge::Event, ActionRoute, DownloaderConfig, MqttConfig}; - - const DOWNLOAD_DIR: &str = "/tmp/uplink_test"; fn config(downloader: DownloaderConfig) -> Config { Config { @@ -368,23 +581,48 @@ mod test { } } - #[test] - // Test file downloading capabilities of FileDownloader by downloading the uplink logo from GitHub - fn download_file() { - // Ensure path exists - std::fs::create_dir_all(DOWNLOAD_DIR).unwrap(); - // Prepare config + fn create_bridge() -> (BridgeTx, Receiver) { + let (inner, _) = bounded(2); + let data_tx = DataTx { inner }; + let (inner, status_rx) = bounded(2); + let status_tx = StatusTx { inner }; + + (BridgeTx { data_tx, status_tx }, status_rx) + } + + // Prepare config + fn test_config(temp_dir: &Path, test_name: &str) -> Config { + let mut path = PathBuf::from(temp_dir); + path.push(test_name); let downloader_cfg = DownloaderConfig { - actions: vec![ActionRoute { name: "firmware_update".to_owned(), timeout: 10 }], - path: format!("{DOWNLOAD_DIR}/uplink-test"), + actions: vec![ActionRoute { + name: "firmware_update".to_owned(), + timeout: Duration::from_secs(10), + }], + path, }; - let config = config(downloader_cfg.clone()); - let (events_tx, events_rx) = flume::bounded(2); - let (shutdown_handle, _) = bounded(1); - let bridge_tx = BridgeTx { events_tx, shutdown_handle }; + config(downloader_cfg.clone()) + } - // Create channels to forward and push action_status on - let downloader = FileDownloader::new(Arc::new(config), bridge_tx).unwrap(); + #[test] + // Test file downloading capabilities of FileDownloader by downloading the uplink logo from GitHub + fn download_file() { + let temp_dir = TempDir::new("download_file").unwrap(); + let config = test_config(temp_dir.path(), "download_file"); + let mut downloader_path = config.downloader.path.clone(); + let (bridge_tx, status_rx) = create_bridge(); + + // Create channels to forward and push actions on + let (download_tx, download_rx) = bounded(1); + let (_, ctrl_rx) = bounded(1); + let downloader = FileDownloader::new( + Arc::new(config), + download_rx, + bridge_tx, + ctrl_rx, + Arc::new(Mutex::new(false)), + ) + .unwrap(); // Start FileDownloader in separate thread std::thread::spawn(|| downloader.start()); @@ -395,19 +633,17 @@ mod test { content_length: 296658, file_name: "test.txt".to_string(), download_path: None, + checksum: None, }; let mut expected_forward = download_update.clone(); - expected_forward.download_path = Some(downloader_cfg.path + "/firmware_update/test.txt"); + downloader_path.push("firmware_update"); + downloader_path.push("test.txt"); + expected_forward.download_path = Some(downloader_path); let download_action = Action { action_id: "1".to_string(), - kind: "firmware_update".to_string(), name: "firmware_update".to_string(), payload: json!(download_update).to_string(), - }; - - let download_tx = match events_rx.recv().unwrap() { - Event::RegisterActionRoute(_, download_tx) => download_tx, - e => unreachable!("Unexpected event: {e:#?}"), + deadline: Some(Instant::now() + Duration::from_secs(60)), }; std::thread::sleep(Duration::from_millis(10)); @@ -416,19 +652,13 @@ mod test { download_tx.try_send(download_action).unwrap(); // Collect action_status and ensure it is as expected - let status = match events_rx.recv().unwrap() { - Event::ActionResponse(status) => status, - e => unreachable!("Unexpected event: {e:#?}"), - }; + let status = status_rx.recv().unwrap(); assert_eq!(status.state, "Downloading"); let mut progress = 0; // Collect and ensure forwarded action contains expected info loop { - let status = match events_rx.recv().unwrap() { - Event::ActionResponse(status) => status, - e => unreachable!("Unexpected event: {e:#?}"), - }; + let status = status_rx.recv().unwrap(); assert!(progress <= status.progress); progress = status.progress; @@ -438,62 +668,109 @@ mod test { let fwd = serde_json::from_str(&fwd_action.payload).unwrap(); assert_eq!(expected_forward, fwd); break; - } else if status.is_failed() { - break; } } } #[test] - fn multiple_actions_at_once() { - // Ensure path exists - std::fs::create_dir_all(DOWNLOAD_DIR).unwrap(); - // Prepare config - let downloader_cfg = DownloaderConfig { - actions: vec![ActionRoute { name: "firmware_update".to_owned(), timeout: 10 }], - path: format!("{}/download", DOWNLOAD_DIR), - }; - let config = config(downloader_cfg.clone()); - let (events_tx, events_rx) = flume::bounded(3); - let (shutdown_handle, _) = bounded(1); - let bridge_tx = BridgeTx { events_tx, shutdown_handle }; + // Once a file is downloaded FileDownloader must check it's checksum value against what is provided + fn checksum_of_file() { + let temp_dir = TempDir::new("file_checksum").unwrap(); + let config = test_config(temp_dir.path(), "file_checksum"); + let (bridge_tx, status_rx) = create_bridge(); // Create channels to forward and push action_status on - let downloader = FileDownloader::new(Arc::new(config), bridge_tx).unwrap(); + let (download_tx, download_rx) = bounded(1); + let (_, ctrl_rx) = bounded(1); + let downloader = FileDownloader::new( + Arc::new(config), + download_rx, + bridge_tx, + ctrl_rx, + Arc::new(Mutex::new(false)), + ) + .unwrap(); // Start FileDownloader in separate thread std::thread::spawn(|| downloader.start()); - // Create a firmware update action - let download_update = DownloadFile { - content_length: 0, + std::thread::sleep(Duration::from_millis(10)); + + // Correct firmware update action + let correct_update = DownloadFile { url: "https://github.com/bytebeamio/uplink/raw/main/docs/logo.png".to_string(), - file_name: "1.0".to_string(), + content_length: 296658, + file_name: "logo.png".to_string(), download_path: None, + checksum: Some( + "e22d4a7cf60ad13bf885c6d84af2f884f0c044faf0ee40b2e3c81896b226b2fc".to_string(), + ), }; - let mut expected_forward = download_update.clone(); - expected_forward.download_path = Some(downloader_cfg.path + "/firmware_update/test.txt"); - let download_action = Action { + let correct_action = Action { action_id: "1".to_string(), - kind: "firmware_update".to_string(), name: "firmware_update".to_string(), - payload: json!(download_update).to_string(), + payload: json!(correct_update).to_string(), + deadline: Some(Instant::now() + Duration::from_secs(100)), }; - let download_tx = match events_rx.recv().unwrap() { - Event::RegisterActionRoute(_, download_tx) => download_tx, - e => unreachable!("Unexpected event: {e:#?}"), + // Send the correct action to FileDownloader + download_tx.try_send(correct_action).unwrap(); + + // Collect action_status and ensure it is as expected + let status = status_rx.recv().unwrap(); + assert_eq!(status.state, "Downloading"); + let mut progress = 0; + + // Collect and ensure forwarded action contains expected info + loop { + let status = status_rx.recv().unwrap(); + + assert!(progress <= status.progress); + progress = status.progress; + + if status.is_done() { + if status.state != "Downloaded" { + panic!("unexpected status={status:?}") + } + break; + } + } + + // Wrong firmware update action + let wrong_update = DownloadFile { + url: "https://github.com/bytebeamio/uplink/raw/main/docs/logo.png".to_string(), + content_length: 296658, + file_name: "logo.png".to_string(), + download_path: None, + checksum: Some("abcd1234efgh5678".to_string()), + }; + let wrong_action = Action { + action_id: "1".to_string(), + name: "firmware_update".to_string(), + payload: json!(wrong_update).to_string(), + deadline: Some(Instant::now() + Duration::from_secs(100)), }; - std::thread::sleep(Duration::from_millis(10)); + // Send the wrong action to FileDownloader + download_tx.try_send(wrong_action).unwrap(); - // Send action to FileDownloader with Sender - download_tx.try_send(download_action.clone()).unwrap(); + // Collect action_status and ensure it is as expected + let status = status_rx.recv().unwrap(); + assert_eq!(status.state, "Downloading"); + let mut progress = 0; + + // Collect and ensure forwarded action contains expected info + loop { + let status = status_rx.recv().unwrap(); - // Send action to FileDownloader immediately after, this must fail - match download_tx.try_send(download_action).unwrap_err() { - TrySendError::Full(_) => {} - TrySendError::Disconnected(_) => panic!("Unexpected disconnect"), + assert!(progress <= status.progress); + progress = status.progress; + + if status.is_done() { + assert!(status.is_failed()); + assert_eq!(status.errors, vec!["Downloaded file has unexpected checksum"]); + break; + } } } } diff --git a/uplink/src/collector/installer.rs b/uplink/src/collector/installer.rs index 007bf87dc..50dd68b49 100644 --- a/uplink/src/collector/installer.rs +++ b/uplink/src/collector/installer.rs @@ -1,12 +1,12 @@ use std::{fs::File, path::PathBuf}; +use flume::Receiver; use log::{debug, error, warn}; use tar::Archive; use tokio::process::Command; use super::downloader::DownloadFile; -use crate::base::{bridge::BridgeTx, InstallerConfig}; -use crate::{Action, ActionResponse}; +use crate::{base::bridge::BridgeTx, config::InstallerConfig, Action, ActionResponse}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -22,22 +22,18 @@ pub enum Error { pub struct OTAInstaller { config: InstallerConfig, + actions_rx: Receiver, bridge_tx: BridgeTx, } impl OTAInstaller { - pub fn new(config: InstallerConfig, bridge_tx: BridgeTx) -> Self { - Self { config, bridge_tx } + pub fn new(config: InstallerConfig, actions_rx: Receiver, bridge_tx: BridgeTx) -> Self { + Self { config, actions_rx, bridge_tx } } #[tokio::main] pub async fn start(&self) { - let actions_rx = match self.bridge_tx.register_action_routes(&self.config.actions).await { - Some(r) => r, - _ => return, - }; - - while let Ok(action) = actions_rx.recv_async().await { + while let Ok(action) = self.actions_rx.recv_async().await { if let Err(e) = self.extractor(&action) { error!("Error extracting tarball: {e}"); self.forward_action_error(action, e).await; @@ -60,7 +56,7 @@ impl OTAInstaller { let info: DownloadFile = serde_json::from_str(&action.payload)?; let path = info.download_path.ok_or(Error::MissingPath)?; - debug!("Extracting tar from:{path}; to: {}", self.config.path); + debug!("Extracting tar from:{}; to: {}", path.display(), self.config.path); let dst = PathBuf::from(&self.config.path); if dst.exists() { warn!("Cleaning up {}", &self.config.path); diff --git a/uplink/src/collector/journalctl.rs b/uplink/src/collector/journalctl.rs index c934576ee..58a81a054 100644 --- a/uplink/src/collector/journalctl.rs +++ b/uplink/src/collector/journalctl.rs @@ -1,3 +1,4 @@ +use flume::Receiver; use serde::{Deserialize, Serialize}; use std::io::BufRead; @@ -5,7 +6,8 @@ use std::process::{Command, Stdio}; use std::sync::{Arc, Mutex}; use std::{io::BufReader, time::Duration}; -use crate::{base::bridge::BridgeTx, ActionResponse, ActionRoute, Payload}; +use crate::Action; +use crate::{base::bridge::BridgeTx, ActionResponse, Payload}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -100,7 +102,9 @@ impl LogEntry { } pub struct JournalCtl { + config: JournalCtlConfig, kill_switch: Arc>, + actions_rx: Receiver, bridge: BridgeTx, } @@ -112,28 +116,20 @@ impl Drop for JournalCtl { } impl JournalCtl { - pub fn new(bridge: BridgeTx) -> Self { + pub fn new(config: JournalCtlConfig, actions_rx: Receiver, bridge: BridgeTx) -> Self { let kill_switch = Arc::new(Mutex::new(true)); - Self { kill_switch, bridge } + Self { config, kill_switch, actions_rx, bridge } } /// Starts a journalctl instance on a linux system which reports to the logs stream for a given device+project id, /// that logcat instance is killed when this object is dropped. On any other system, it's a noop. #[tokio::main(flavor = "current_thread")] - pub async fn start(mut self, config: JournalCtlConfig) -> Result<(), Error> { - self.spawn_logger(config).await; - - let log_rx = self - .bridge - .register_action_route(ActionRoute { - name: "journalctl_config".to_string(), - timeout: 10, - }) - .await; + pub async fn start(mut self) -> Result<(), Error> { + self.spawn_logger(self.config.clone()).await; loop { - let action = log_rx.recv()?; + let action = self.actions_rx.recv()?; let mut config = serde_json::from_str::(action.payload.as_str())?; config.tags.retain(|tag| !tag.is_empty()); config.units.retain(|unit| !unit.is_empty()); diff --git a/uplink/src/collector/log_reader.rs b/uplink/src/collector/log_reader.rs index d8f4bacbc..9159fa89b 100644 --- a/uplink/src/collector/log_reader.rs +++ b/uplink/src/collector/log_reader.rs @@ -10,7 +10,7 @@ use serde_json::json; use tokio::process::Command; use crate::base::bridge::{BridgeTx, Payload}; -use crate::base::LogReaderConfig; +use crate::config::LogReaderConfig; #[derive(Debug, Serialize, Clone, PartialEq)] struct LogEntry { @@ -104,7 +104,7 @@ impl LogEntry { .map(to_string) .unwrap_or_else(|| std::env::var("LOG_TAG").unwrap_or("".to_owned())); let message = captures.name("message").map(to_string); - let pid = captures.name("pid").map(to_usize).flatten(); + let pid = captures.name("pid").and_then(to_usize); let query_id = captures.name("query_id").map(to_string); return current_line.replace(LogEntry { @@ -232,7 +232,7 @@ mod test { .to_string(), timestamp: 1688407162000, message: Some("Outgoing = Publish(9)".to_string()), - tag: Some("uplink::base::mqtt".to_string()), + tag: "uplink::base::mqtt".to_string(), pid: None, query_id: None } @@ -285,7 +285,7 @@ mod test { .to_string(), timestamp: 1688407162979, message: Some("Outgoing = Publish(9)".to_string()), - tag: Some("uplink::base::mqtt".to_string()), + tag: "uplink::base::mqtt".to_string(), pid: None, query_id: None } @@ -300,7 +300,7 @@ mod test { .to_string(), timestamp: 1688407163012, message: Some("Incoming = PubAck(9)".to_string()), - tag: Some("uplink::base::mqtt".to_string()), + tag: "uplink::base::mqtt".to_string(), pid: None, query_id: None } @@ -331,7 +331,7 @@ mod test { line: "2023-07-11T13:56:44.101585Z INFO beamd::http::endpoint: Method = \"POST\", Uri = \"/tenants/naveentest/devices/8/actions\", Payload = \"{\\\"name\\\":\\\"update_firmware\\\",\\\"id\\\":\\\"830\\\",\\\"payload\\\":\\\"{\\\\\\\"content-length\\\\\\\":35393,\\\\\\\"status\\\\\\\":false,\\\\\\\"url\\\\\\\":\\\\\\\"https://firmware.stage.bytebeam.io/api/v1/firmwares/one/artifact\\\\\\\",\\\\\\\"version\\\\\\\":\\\\\\\"one\\\\\\\"}\\\",\\\"kind\\\":\\\"process\\\"}\"".to_string(), timestamp: 1689083804101, message: Some("Method = \"POST\", Uri = \"/tenants/naveentest/devices/8/actions\", Payload = \"{\\\"name\\\":\\\"update_firmware\\\",\\\"id\\\":\\\"830\\\",\\\"payload\\\":\\\"{\\\\\\\"content-length\\\\\\\":35393,\\\\\\\"status\\\\\\\":false,\\\\\\\"url\\\\\\\":\\\\\\\"https://firmware.stage.bytebeam.io/api/v1/firmwares/one/artifact\\\\\\\",\\\\\\\"version\\\\\\\":\\\\\\\"one\\\\\\\"}\\\",\\\"kind\\\":\\\"process\\\"}\"".to_string()), - tag: Some("beamd::http::endpoint".to_string()), + tag: "beamd::http::endpoint".to_string(), pid: None, query_id: None } @@ -345,7 +345,7 @@ mod test { line: "2023-07-11T13:56:44.113343Z INFO beamd::http::endpoint: Method = \"POST\", Uri = \"/tenants/rpi/devices/6/actions\", Payload = \"{\\\"name\\\":\\\"tunshell\\\",\\\"id\\\":\\\"226\\\",\\\"payload\\\":\\\"{}\\\",\\\"kind\\\":\\\"process\\\"}\"".to_string(), timestamp: 1689083804113, message: Some("Method = \"POST\", Uri = \"/tenants/rpi/devices/6/actions\", Payload = \"{\\\"name\\\":\\\"tunshell\\\",\\\"id\\\":\\\"226\\\",\\\"payload\\\":\\\"{}\\\",\\\"kind\\\":\\\"process\\\"}\"".to_string()), - tag: Some("beamd::http::endpoint".to_string()), + tag: "beamd::http::endpoint".to_string(), pid: None, query_id: None } @@ -356,7 +356,7 @@ mod test { entry, LogEntry { line: "2023-07-11T13:56:44.221249Z ERROR beamd::clickhouse: Flush-error: [Status - 500] Ok(\"Code: 243. DB::Exception: Cannot reserve 11.58 MiB, not enough space. (NOT_ENOUGH_SPACE) (version 22.6.2.12 (official build))\\n\"), back_up_enabled: true\nin beamd::clickhouse::clickhouse_flush with stream: \"demo.uplink_process_stats\"".to_string(), - tag: Some("beamd::clickhouse".to_string()), + tag: "beamd::clickhouse".to_string(), level: Some("ERROR".to_string()), timestamp: 1689083804221, message: Some("Flush-error: [Status - 500] Ok(\"Code: 243. DB::Exception: Cannot reserve 11.58 MiB, not enough space. (NOT_ENOUGH_SPACE) (version 22.6.2.12 (official build))\\n\"), back_up_enabled: true\nin beamd::clickhouse::clickhouse_flush with stream: \"demo.uplink_process_stats\"".to_string()), @@ -387,7 +387,7 @@ mod test { line: "23-07-11 18:03:32 consoled-6cd8795566-76km9 INFO [ring.logger:0] - {:request-method :get, :uri \"/api/v1/devices/count\", :server-name \"cloud.bytebeam.io\", :ring.logger/type :finish, :status 200, :ring.logger/ms 11}\n10.13.2.69 - - [11/Jul/2023:18:03:32 +0000] \"GET /api/v1/devices/count?status=active HTTP/1.1\" 200 1 \"https://cloud.bytebeam.io/projects/kptl/device-management/devices\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"rt=0.016 uct=0.000 cn= o=\n\"Notifying broker for tenant reactlabs device 305 action 683022\"".to_string(), timestamp: 1689098612000, message: Some("[ring.logger:0] - {:request-method :get, :uri \"/api/v1/devices/count\", :server-name \"cloud.bytebeam.io\", :ring.logger/type :finish, :status 200, :ring.logger/ms 11}\n10.13.2.69 - - [11/Jul/2023:18:03:32 +0000] \"GET /api/v1/devices/count?status=active HTTP/1.1\" 200 1 \"https://cloud.bytebeam.io/projects/kptl/device-management/devices\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"rt=0.016 uct=0.000 cn= o=\n\"Notifying broker for tenant reactlabs device 305 action 683022\"".to_string()), - tag: Some("consoled-6cd8795566-76km9".to_string()), + tag: "consoled-6cd8795566-76km9".to_string(), pid: None, query_id: None } diff --git a/uplink/src/collector/logcat.rs b/uplink/src/collector/logcat.rs index 5e3bd9347..4c5ce8084 100644 --- a/uplink/src/collector/logcat.rs +++ b/uplink/src/collector/logcat.rs @@ -1,3 +1,4 @@ +use flume::Receiver; use serde::{Deserialize, Serialize}; use std::io::{BufRead, BufReader}; @@ -6,7 +7,8 @@ use std::process::{Command, Stdio}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use crate::{base::clock, ActionResponse, ActionRoute, BridgeTx, Payload}; +use crate::base::{bridge::BridgeTx, clock}; +use crate::{Action, ActionResponse, Payload}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -97,8 +99,8 @@ pub struct LogEntry { } lazy_static::lazy_static! { - pub static ref LOGCAT_RE: regex::Regex = regex::Regex::new(r#"^(\S+ \S+) (\w)/([^(\s]*).+?:\s*(.*)$"#).unwrap(); - pub static ref LOGCAT_TIME_RE: regex::Regex = regex::Regex::new(r#"^(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\.(\d+)$"#).unwrap(); + pub static ref LOGCAT_RE: regex::Regex = regex::Regex::new(r"^(\S+ \S+) (\w)/([^(\s]*).+?:\s*(.*)$").unwrap(); + pub static ref LOGCAT_TIME_RE: regex::Regex = regex::Regex::new(r"^(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\.(\d+)$").unwrap(); } pub fn parse_logcat_time(s: &str) -> Option { @@ -150,7 +152,9 @@ impl LogEntry { } pub struct Logcat { + config: LogcatConfig, kill_switch: Arc>, + actions_rx: Receiver, bridge: BridgeTx, } @@ -162,25 +166,20 @@ impl Drop for Logcat { } impl Logcat { - pub fn new(bridge: BridgeTx) -> Self { + pub fn new(config: LogcatConfig, actions_rx: Receiver, bridge: BridgeTx) -> Self { let kill_switch = Arc::new(Mutex::new(true)); - Self { kill_switch, bridge } + Self { config, kill_switch, actions_rx, bridge } } /// On an android system, starts a logcat instance that reports to the logs stream for a given device+project id, /// that logcat instance is killed when this object is dropped. On any other system, it's a noop. #[tokio::main(flavor = "current_thread")] - pub async fn start(mut self, config: LogcatConfig) -> Result<(), Error> { - self.spawn_logger(config).await; - - let log_rx = self - .bridge - .register_action_route(ActionRoute { name: "logcat_config".to_string(), timeout: 10 }) - .await; + pub async fn start(mut self) -> Result<(), Error> { + self.spawn_logger(self.config.clone()).await; loop { - let action = log_rx.recv()?; + let action = self.actions_rx.recv()?; let mut config = serde_json::from_str::(action.payload.as_str())?; config.tags.retain(|tag| !tag.is_empty()); diff --git a/uplink/src/collector/mod.rs b/uplink/src/collector/mod.rs index e037ade08..3a5e232c9 100644 --- a/uplink/src/collector/mod.rs +++ b/uplink/src/collector/mod.rs @@ -7,6 +7,7 @@ pub mod journalctl; pub mod log_reader; #[cfg(target_os = "android")] pub mod logcat; +pub mod preconditions; pub mod process; pub mod prometheus; pub mod script_runner; diff --git a/uplink/src/collector/preconditions.rs b/uplink/src/collector/preconditions.rs new file mode 100644 index 000000000..9f9045295 --- /dev/null +++ b/uplink/src/collector/preconditions.rs @@ -0,0 +1,88 @@ +use std::{fs::metadata, os::unix::fs::MetadataExt, sync::Arc}; + +use flume::Receiver; +use human_bytes::human_bytes; +use log::debug; +use serde::Deserialize; + +use crate::{base::bridge::BridgeTx, Action, ActionResponse, Config}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("File io Error: {0}")] + Io(#[from] std::io::Error), + #[error("Disk space is insufficient: {0}")] + InsufficientDisk(String), +} + +#[derive(Deserialize, Clone, PartialEq, Eq, Debug)] +pub struct Preconditions { + #[serde(alias = "content-length")] + content_length: usize, + #[serde(alias = "uncompressed-size")] + uncompressed_length: Option, +} + +pub struct PreconditionChecker { + config: Arc, + actions_rx: Receiver, + bridge_tx: BridgeTx, +} + +impl PreconditionChecker { + pub fn new(config: Arc, actions_rx: Receiver, bridge_tx: BridgeTx) -> Self { + Self { config, actions_rx, bridge_tx } + } + + #[tokio::main] + pub async fn start(self) { + while let Ok(action) = self.actions_rx.recv() { + let preconditions: Preconditions = match serde_json::from_str(&action.payload) { + Ok(c) => c, + Err(e) => { + let response = ActionResponse::failure(&action.action_id, e.to_string()); + self.bridge_tx.send_action_response(response).await; + continue; + } + }; + + if let Err(e) = self.check_disk_size(preconditions) { + let response = ActionResponse::failure(&action.action_id, e.to_string()); + self.bridge_tx.send_action_response(response).await; + continue; + } + + let response = ActionResponse::progress(&action.action_id, "Checked OK", 100); + self.bridge_tx.send_action_response(response).await; + } + } + + // Fails if there isn't enough space to download and/or install update + // NOTE: both download and installation could happen in the same partition + // TODO: this should be significantly simplified once we move to using `expected-free-space` + // comparison instead of making assumptions about what the user might want. + fn check_disk_size(&self, preconditions: Preconditions) -> Result<(), Error> { + let Some(mut required_free_space) = preconditions.uncompressed_length else { + return Ok(()); + }; + let disk_free_space = + fs2::free_space(&self.config.precondition_checks.as_ref().unwrap().path)? as usize; + + // Check if download and installation paths are on the same partition, if yes add download file size to required + if metadata(&self.config.downloader.path)?.dev() + == metadata(&self.config.precondition_checks.as_ref().unwrap().path)?.dev() + { + required_free_space += preconditions.content_length; + } + + let req_size = human_bytes(required_free_space as f64); + let free_size = human_bytes(disk_free_space as f64); + debug!("The installation requires {req_size}; Disk free space is {free_size}"); + + if required_free_space > disk_free_space { + return Err(Error::InsufficientDisk(free_size)); + } + + Ok(()) + } +} diff --git a/uplink/src/collector/process.rs b/uplink/src/collector/process.rs index ba75bcd43..f1aed754e 100644 --- a/uplink/src/collector/process.rs +++ b/uplink/src/collector/process.rs @@ -1,17 +1,16 @@ -use flume::{RecvError, SendError}; +use flume::{Receiver, RecvError, SendError}; use log::{debug, error, info}; use thiserror::Error; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::{Child, Command}; -use tokio::{pin, select, time}; +use tokio::select; +use tokio::time::timeout_at; use crate::base::bridge::BridgeTx; -use crate::base::ActionRoute; -use crate::{ActionResponse, Package}; +use crate::{Action, ActionResponse, Package}; use std::io; use std::process::Stdio; -use std::time::Duration; #[derive(Error, Debug)] pub enum Error { @@ -34,22 +33,19 @@ pub enum Error { /// is in progress. /// It sends result and errors to the broker over collector_tx pub struct ProcessHandler { - // to receive actions and send responses back to bridge + // to receive actions + actions_rx: Receiver, + // to send responses back to bridge bridge_tx: BridgeTx, } impl ProcessHandler { - pub fn new(bridge_tx: BridgeTx) -> Self { - Self { bridge_tx } + pub fn new(actions_rx: Receiver, bridge_tx: BridgeTx) -> Self { + Self { actions_rx, bridge_tx } } /// Run a process of specified command - pub async fn run( - &mut self, - id: String, - command: String, - payload: String, - ) -> Result { + pub async fn run(&mut self, id: &str, command: &str, payload: &str) -> Result { let mut cmd = Command::new(command); cmd.arg(id).arg(payload).kill_on_drop(true).stdout(Stdio::piped()); @@ -63,9 +59,6 @@ impl ProcessHandler { let stdout = child.stdout.take().ok_or(Error::NoStdout)?; let mut stdout = BufReader::new(stdout).lines(); - let timeout = time::sleep(Duration::from_secs(10)); - pin!(timeout); - loop { select! { Ok(Some(line)) = stdout.next_line() => { @@ -77,27 +70,34 @@ impl ProcessHandler { debug!("Action status: {:?}", status); self.bridge_tx.send_action_response(status).await; } - status = child.wait() => info!("Action done!! Status = {:?}", status), - _ = &mut timeout => break + status = child.wait() => { + info!("Action done!! Status = {:?}", status); + return Ok(()); + }, } } - - Ok(()) } - pub async fn start(mut self, processes: Vec) -> Result<(), Error> { - let action_rx = match self.bridge_tx.register_action_routes(processes).await { - Some(r) => r, - _ => return Ok(()), - }; - + #[tokio::main(flavor = "current_thread")] + pub async fn start(mut self) -> Result<(), Error> { loop { - let action = action_rx.recv_async().await?; - let command = String::from("tools/") + &action.name; - - // Spawn the action and capture its stdout - let child = self.run(action.action_id, command, action.payload).await?; - self.spawn_and_capture_stdout(child).await?; + let action = self.actions_rx.recv_async().await?; + let command = format!("tools/{}", action.name); + let deadline = match &action.deadline { + Some(d) => *d, + _ => { + error!("Unconfigured deadline: {}", action.name); + continue; + } + }; + + // Spawn the action and capture its stdout, ignore timeouts + let child = self.run(&action.action_id, &command, &action.payload).await?; + if let Ok(o) = timeout_at(deadline, self.spawn_and_capture_stdout(child)).await { + o?; + } else { + error!("Process timedout: {command}; action_id = {}", action.action_id); + } } } } diff --git a/uplink/src/collector/prometheus.rs b/uplink/src/collector/prometheus.rs index 97773a288..61ca47b5a 100644 --- a/uplink/src/collector/prometheus.rs +++ b/uplink/src/collector/prometheus.rs @@ -4,8 +4,13 @@ use log::error; use reqwest::{Client, Method}; use serde_json::{json, Map, Value}; -use crate::base::bridge::{BridgeTx, Payload}; -use crate::base::{clock, PrometheusConfig}; +use crate::{ + base::{ + bridge::{BridgeTx, Payload}, + clock, + }, + config::PrometheusConfig, +}; #[derive(Debug, thiserror::Error)] enum Error { @@ -88,12 +93,8 @@ fn frame_payload<'a>(mut line: impl Iterator) -> Option let value = value.parse::().ok()?; payload.insert(stream.to_owned(), json!(value)); - let mut payload = Payload { - stream, - sequence: 0, - timestamp: clock() as u64, - payload: payload.into(), - }; + let mut payload = + Payload { stream, sequence: 0, timestamp: clock() as u64, payload: payload.into() }; if let Some(timestamp) = line.next() { let timestamp = timestamp.parse::().ok()?; diff --git a/uplink/src/collector/script_runner.rs b/uplink/src/collector/script_runner.rs index 5412eb791..db8f74ac1 100644 --- a/uplink/src/collector/script_runner.rs +++ b/uplink/src/collector/script_runner.rs @@ -1,19 +1,18 @@ -use flume::{RecvError, SendError}; +use flume::{Receiver, RecvError, SendError}; use log::{debug, error, info, warn}; use thiserror::Error; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::{Child, Command}; use tokio::select; -use tokio::time::timeout; +use tokio::time::timeout_at; use super::downloader::DownloadFile; -use crate::base::{bridge::BridgeTx, ActionRoute}; -use crate::{ActionResponse, Package}; +use crate::base::bridge::BridgeTx; +use crate::{Action, ActionResponse, Package}; -use std::collections::HashMap; use std::io; +use std::path::PathBuf; use std::process::Stdio; -use std::time::Duration; #[derive(Error, Debug)] pub enum Error { @@ -35,19 +34,20 @@ pub enum Error { /// Multiple scripts can't be run in parallel. It can also send progress, result and errors to the platform by using /// the JSON formatted output over STDOUT. pub struct ScriptRunner { - // to receive actions and send responses back to bridge + // to receive actions + actions_rx: Receiver, + // to send responses back to bridge bridge_tx: BridgeTx, - timeouts: HashMap, sequence: u32, } impl ScriptRunner { - pub fn new(bridge_tx: BridgeTx) -> Self { - Self { bridge_tx, timeouts: HashMap::new(), sequence: 0 } + pub fn new(actions_rx: Receiver, bridge_tx: BridgeTx) -> Self { + Self { actions_rx, bridge_tx, sequence: 0 } } /// Spawn a child process to run the script with sh - pub async fn run(&mut self, command: String) -> Result { + pub async fn run(&mut self, command: PathBuf) -> Result { let mut cmd = Command::new("sh"); cmd.arg(command).kill_on_drop(true).stdout(Stdio::piped()); @@ -75,7 +75,7 @@ impl ScriptRunner { continue; }, }; - status.action_id = id.to_owned(); + id.clone_into(&mut status.action_id); debug!("Action status: {:?}", status); self.forward_status(status).await; @@ -92,19 +92,12 @@ impl ScriptRunner { Ok(()) } - pub async fn start(mut self, routes: Vec) -> Result<(), Error> { - self.timeouts = - routes.iter().map(|s| (s.name.to_owned(), Duration::from_secs(s.timeout))).collect(); - - let action_rx = match self.bridge_tx.register_action_routes(routes).await { - Some(r) => r, - _ => return Ok(()), - }; - + #[tokio::main(flavor = "current_thread")] + pub async fn start(mut self) -> Result<(), Error> { info!("Script runner is ready"); loop { - let action = action_rx.recv_async().await?; + let action = self.actions_rx.recv_async().await?; let command = match serde_json::from_str::(&action.payload) { Ok(DownloadFile { download_path: Some(download_path), .. }) => download_path, Ok(_) => { @@ -126,17 +119,17 @@ impl ScriptRunner { continue; } }; - let duration = match self.timeouts.get(&action.name) { + let deadline = match &action.deadline { Some(d) => *d, _ => { - error!("Unconfigured action: {}", action.name); + error!("Unconfigured deadline: {}", action.name); continue; } }; // Spawn the action and capture its stdout let child = self.run(command).await?; if let Ok(o) = - timeout(duration, self.spawn_and_capture_stdout(child, &action.action_id)).await + timeout_at(deadline, self.spawn_and_capture_stdout(child, &action.action_id)).await { o? } @@ -153,84 +146,65 @@ impl ScriptRunner { #[cfg(test)] mod tests { - use std::thread; + use std::thread::spawn; + + use flume::bounded; + + use crate::base::bridge::{DataTx, StatusTx}; use super::*; - use crate::{ - base::bridge::{ActionRouter, Event}, - Action, - }; - use flume::bounded; - use tokio::runtime::Runtime; + fn create_bridge() -> (BridgeTx, Receiver) { + let (inner, _) = bounded(2); + let data_tx = DataTx { inner }; + let (inner, status_rx) = bounded(2); + let status_tx = StatusTx { inner }; + + (BridgeTx { data_tx, status_tx }, status_rx) + } #[test] fn empty_payload() { - let (events_tx, events_rx) = bounded(1); - let (shutdown_handle, _) = bounded(1); - let script_runner = ScriptRunner::new(BridgeTx { events_tx, shutdown_handle }); - let routes = vec![ActionRoute { name: "test".to_string(), timeout: 100 }]; - - thread::spawn(move || { - Runtime::new().unwrap().block_on(async { - script_runner.start(routes).await.unwrap(); - }) - }); + let (bridge_tx, status_rx) = create_bridge(); + + let (actions_tx, actions_rx) = bounded(1); + let script_runner = ScriptRunner::new(actions_rx, bridge_tx); + spawn(move || script_runner.start().unwrap()); - let Event::RegisterActionRoute(_, ActionRouter { actions_tx, .. }) = - events_rx.recv().unwrap() - else { - unreachable!() - }; actions_tx .send(Action { action_id: "1".to_string(), - kind: "1".to_string(), name: "test".to_string(), payload: "".to_string(), + deadline: None, }) .unwrap(); - let Event::ActionResponse(ActionResponse { state, errors, .. }) = events_rx.recv().unwrap() - else { - unreachable!() - }; + let ActionResponse { state, errors, .. } = status_rx.recv().unwrap(); assert_eq!(state, "Failed"); assert_eq!(errors, ["Failed to deserialize action payload: \"EOF while parsing a value at line 1 column 0\"; payload: \"\""]); } #[test] fn missing_path() { - let (events_tx, events_rx) = bounded(1); - let (shutdown_handle, _) = bounded(1); - let script_runner = ScriptRunner::new(BridgeTx { events_tx, shutdown_handle }); - let routes = vec![ActionRoute { name: "test".to_string(), timeout: 100 }]; - - thread::spawn(move || { - Runtime::new().unwrap().block_on(async { - script_runner.start(routes).await.unwrap(); - }) - }); + let (bridge_tx, status_rx) = create_bridge(); + + let (actions_tx, actions_rx) = bounded(1); + let script_runner = ScriptRunner::new(actions_rx, bridge_tx); + + spawn(move || script_runner.start().unwrap()); - let Event::RegisterActionRoute(_, ActionRouter { actions_tx, .. }) = - events_rx.recv().unwrap() - else { - unreachable!() - }; actions_tx .send(Action { action_id: "1".to_string(), - kind: "1".to_string(), name: "test".to_string(), payload: "{\"url\": \"...\", \"content_length\": 0,\"file_name\": \"...\"}" .to_string(), + deadline: None, }) .unwrap(); - let Event::ActionResponse(ActionResponse { state, errors, .. }) = events_rx.recv().unwrap() - else { - unreachable!() - }; + let ActionResponse { state, errors, .. } = status_rx.recv().unwrap(); assert_eq!(state, "Failed"); assert_eq!(errors, ["Action payload doesn't contain path for script execution; payload: \"{\"url\": \"...\", \"content_length\": 0,\"file_name\": \"...\"}\""]); } diff --git a/uplink/src/collector/simulator/data.rs b/uplink/src/collector/simulator/data.rs index 12527b54d..6691cba8f 100644 --- a/uplink/src/collector/simulator/data.rs +++ b/uplink/src/collector/simulator/data.rs @@ -13,6 +13,8 @@ use std::time::Duration; use crate::base::clock; use crate::Payload; +use super::Event; + const RESET_LIMIT: u32 = 1500; #[inline] @@ -60,7 +62,7 @@ pub struct Gps { } impl Gps { - pub async fn simulate(tx: Sender, device: DeviceData) { + pub async fn simulate(tx: Sender, device: DeviceData) { let mut sequence = 0; let mut interval = interval(DataType::Gps.duration()); let path_len = device.path.len() as u32; @@ -74,12 +76,12 @@ impl Gps { trace!("Data Event: {:?}", payload); if let Err(e) = tx - .send_async(Payload { + .send_async(Event::Data(Payload { timestamp: clock() as u64, stream: "gps".to_string(), sequence, payload: json!(payload), - }) + })) .await { error!("{e}"); @@ -194,7 +196,7 @@ pub struct Bms { } impl Bms { - pub async fn simulate(tx: Sender) { + pub async fn simulate(tx: Sender) { let mut sequence = 0; let mut interval = interval(DataType::Bms.duration()); loop { @@ -205,12 +207,12 @@ impl Bms { trace!("Data Event: {:?}", payload); if let Err(e) = tx - .send_async(Payload { + .send_async(Event::Data(Payload { timestamp: clock() as u64, stream: "bms".to_string(), sequence, payload: json!(payload), - }) + })) .await { error!("{e}"); @@ -243,7 +245,7 @@ pub struct Imu { } impl Imu { - pub async fn simulate(tx: Sender) { + pub async fn simulate(tx: Sender) { let mut sequence = 0; let mut interval = interval(DataType::Imu.duration()); loop { @@ -254,12 +256,12 @@ impl Imu { trace!("Data Event: {:?}", payload); if let Err(e) = tx - .send_async(Payload { + .send_async(Event::Data(Payload { timestamp: clock() as u64, stream: "imu".to_string(), sequence, payload: json!(payload), - }) + })) .await { error!("{e}"); @@ -286,7 +288,7 @@ pub struct Motor { } impl Motor { - pub async fn simulate(tx: Sender) { + pub async fn simulate(tx: Sender) { let mut sequence = 0; let mut interval = interval(DataType::Motor.duration()); loop { @@ -297,12 +299,12 @@ impl Motor { trace!("Data Event: {:?}", payload); if let Err(e) = tx - .send_async(Payload { + .send_async(Event::Data(Payload { timestamp: clock() as u64, stream: "motor".to_string(), sequence, payload: json!(payload), - }) + })) .await { error!("{e}"); @@ -335,7 +337,7 @@ pub struct PeripheralState { } impl PeripheralState { - pub async fn simulate(tx: Sender) { + pub async fn simulate(tx: Sender) { let mut sequence = 0; let mut interval = interval(DataType::PeripheralData.duration()); loop { @@ -346,12 +348,12 @@ impl PeripheralState { trace!("Data Event: {:?}", payload); if let Err(e) = tx - .send_async(Payload { + .send_async(Event::Data(Payload { timestamp: clock() as u64, stream: "peripheral_state".to_string(), sequence, payload: json!(payload), - }) + })) .await { error!("{e}"); @@ -381,7 +383,7 @@ pub struct DeviceShadow { } impl DeviceShadow { - pub async fn simulate(tx: Sender) { + pub async fn simulate(tx: Sender) { let mut sequence = 0; let mut interval = interval(DataType::DeviceShadow.duration()); loop { @@ -392,12 +394,12 @@ impl DeviceShadow { trace!("Data Event: {:?}", payload); if let Err(e) = tx - .send_async(Payload { + .send_async(Event::Data(Payload { timestamp: clock() as u64, stream: "device_shadow".to_string(), sequence, payload: json!(payload), - }) + })) .await { error!("{e}"); diff --git a/uplink/src/collector/simulator/mod.rs b/uplink/src/collector/simulator/mod.rs index eca338a22..f6fa99801 100644 --- a/uplink/src/collector/simulator/mod.rs +++ b/uplink/src/collector/simulator/mod.rs @@ -1,12 +1,10 @@ use crate::base::bridge::{BridgeTx, Payload}; -use crate::base::{clock, SimulatorConfig}; -use crate::Action; +use crate::config::SimulatorConfig; +use crate::{Action, ActionResponse}; use data::{Bms, DeviceData, DeviceShadow, Gps, Imu, Motor, PeripheralState}; -use flume::{bounded, Sender}; +use flume::{bounded, Receiver, Sender}; use log::{error, info}; use rand::Rng; -use serde::Serialize; -use serde_json::json; use thiserror::Error; use tokio::time::interval; use tokio::{select, spawn}; @@ -16,6 +14,11 @@ use std::{fs, io, sync::Arc}; mod data; +pub enum Event { + Data(Payload), + ActionResponse(ActionResponse), +} + #[derive(Error, Debug)] pub enum Error { #[error("Io error {0}")] @@ -26,16 +29,8 @@ pub enum Error { Json(#[from] serde_json::error::Error), } -#[derive(Serialize)] -pub struct ActionResponse { - action_id: String, - state: String, - progress: u8, - errors: Vec, -} - impl ActionResponse { - pub async fn simulate(action: Action, tx: Sender) { + pub async fn simulate(action: Action, tx: Sender) { let action_id = action.action_id; info!("Generating action events for action: {action_id}"); let mut sequence = 0; @@ -43,22 +38,11 @@ impl ActionResponse { // Action response, 10% completion per second for i in 1..10 { - let response = ActionResponse { - action_id: action_id.clone(), - progress: i * 10 + rand::thread_rng().gen_range(0..10), - state: String::from("in_progress"), - errors: vec![], - }; + let progress = i * 10 + rand::thread_rng().gen_range(0..10); sequence += 1; - if let Err(e) = tx - .send_async(Payload { - stream: "action_status".to_string(), - sequence, - payload: json!(response), - timestamp: clock() as u64, - }) - .await - { + let response = ActionResponse::progress(&action_id, "in_progress", progress) + .set_sequence(sequence); + if let Err(e) = tx.send_async(Event::ActionResponse(response)).await { error!("{e}"); break; } @@ -66,22 +50,10 @@ impl ActionResponse { interval.tick().await; } - let response = ActionResponse { - action_id, - progress: 100, - state: String::from("Completed"), - errors: vec![], - }; sequence += 1; - if let Err(e) = tx - .send_async(Payload { - stream: "action_status".to_string(), - sequence, - payload: json!(response), - timestamp: clock() as u64, - }) - .await - { + let response = + ActionResponse::progress(&action_id, "Completed", 100).set_sequence(sequence); + if let Err(e) = tx.send_async(Event::ActionResponse(response)).await { error!("{e}"); } info!("Successfully sent all action responses"); @@ -107,41 +79,53 @@ pub fn new_device_data(path: Arc>) -> DeviceData { DeviceData { path, path_offset: path_index } } -pub fn spawn_data_simulators(device: DeviceData, tx: Sender) { +pub fn spawn_data_simulators(device: DeviceData, tx: Sender) { spawn(Gps::simulate(tx.clone(), device)); spawn(Bms::simulate(tx.clone())); spawn(Imu::simulate(tx.clone())); spawn(Motor::simulate(tx.clone())); spawn(PeripheralState::simulate(tx.clone())); - spawn(DeviceShadow::simulate(tx.clone())); + spawn(DeviceShadow::simulate(tx)); } #[tokio::main(flavor = "current_thread")] -pub async fn start(config: SimulatorConfig, bridge_tx: BridgeTx) -> Result<(), Error> { +pub async fn start( + config: SimulatorConfig, + bridge_tx: BridgeTx, + actions_rx: Option>, +) -> Result<(), Error> { let path = read_gps_path(&config.gps_paths); let device = new_device_data(path); - let actions_rx = bridge_tx.register_action_routes(&config.actions).await; - let (tx, rx) = bounded(10); spawn_data_simulators(device, tx.clone()); loop { - select! { - action = actions_rx.as_ref().unwrap().recv_async(), if actions_rx.is_some() => { - let action = action?; - spawn(ActionResponse::simulate(action, tx.clone())); + if let Some(actions_rx) = actions_rx.as_ref() { + select! { + action = actions_rx.recv_async() => { + let action = action?; + spawn(ActionResponse::simulate(action, tx.clone())); + } + event = rx.recv_async() => { + match event { + Ok(Event::ActionResponse(status)) => bridge_tx.send_action_response(status).await, + Ok(Event::Data(payload)) => bridge_tx.send_payload(payload).await, + Err(_) => { + error!("All generators have stopped!"); + return Ok(()) + } + }; + } } - p = rx.recv_async() => { - let payload = match p { - Ok(p) => p, - Err(_) => { - error!("All generators have stopped!"); - return Ok(()) - } - }; - - bridge_tx.send_payload(payload).await; + } else { + match rx.recv_async().await { + Ok(Event::ActionResponse(status)) => bridge_tx.send_action_response(status).await, + Ok(Event::Data(payload)) => bridge_tx.send_payload(payload).await, + Err(_) => { + error!("All generators have stopped!"); + return Ok(()); + } } } } diff --git a/uplink/src/collector/systemstats.rs b/uplink/src/collector/systemstats.rs index 97b8838f6..96fba74c4 100644 --- a/uplink/src/collector/systemstats.rs +++ b/uplink/src/collector/systemstats.rs @@ -45,10 +45,7 @@ pub struct System { impl System { fn init(sys: &sysinfo::System) -> System { System { - kernel_version: match sys.kernel_version() { - Some(kv) => kv, - None => String::default(), - }, + kernel_version: sys.kernel_version().unwrap_or_default(), total_memory: sys.total_memory(), ..Default::default() } @@ -141,8 +138,9 @@ impl Network { /// Update metrics values for network usage over time fn update(&mut self, data: &NetworkData, timestamp: u64, sequence: u32) { let update_period = self.timer.elapsed().as_secs_f64(); - self.incoming_data_rate = data.total_received() as f64 / update_period; - self.outgoing_data_rate = data.total_transmitted() as f64 / update_period; + self.timer = Instant::now(); + self.incoming_data_rate = data.received() as f64 / update_period; + self.outgoing_data_rate = data.transmitted() as f64 / update_period; self.timestamp = timestamp; self.sequence = sequence; } @@ -472,6 +470,7 @@ impl StatCollector { let mut sys = sysinfo::System::new(); sys.refresh_disks_list(); sys.refresh_networks_list(); + sys.refresh_networks(); sys.refresh_memory(); sys.refresh_cpu(); sys.refresh_components(); @@ -570,12 +569,12 @@ impl StatCollector { // Refresh network byte rate stats fn update_network_stats(&mut self) -> Result<(), Error> { - self.sys.refresh_networks(); let timestamp = clock() as u64; for (net_name, net_data) in self.sys.networks() { let payload = self.networks.push(net_name.to_owned(), net_data, timestamp); self.bridge_tx.send_payload_sync(payload); } + self.sys.refresh_networks(); Ok(()) } @@ -612,7 +611,7 @@ impl StatCollector { self.sys.refresh_processes(); let timestamp = clock() as u64; for (&id, p) in self.sys.processes() { - let name = p.cmd().get(0).map(|s| s.to_string()).unwrap_or(p.name().to_string()); + let name = p.cmd().first().map(|s| s.to_string()).unwrap_or(p.name().to_string()); if self.config.system_stats.process_names.contains(&name) { let payload = self.processes.push(id.as_u32(), p, name, timestamp); diff --git a/uplink/src/collector/tcpjson.rs b/uplink/src/collector/tcpjson.rs index a59f09e24..46ebc362c 100644 --- a/uplink/src/collector/tcpjson.rs +++ b/uplink/src/collector/tcpjson.rs @@ -1,4 +1,4 @@ -use flume::{Receiver, RecvError, SendError}; +use flume::{Receiver, RecvError}; use futures_util::SinkExt; use log::{debug, error, info}; use thiserror::Error; @@ -11,7 +11,7 @@ use tokio_util::codec::{Framed, LinesCodec, LinesCodecError}; use std::io; use crate::base::bridge::BridgeTx; -use crate::base::AppConfig; +use crate::config::AppConfig; use crate::{Action, ActionResponse, Payload}; #[derive(Error, Debug)] @@ -20,16 +20,12 @@ pub enum Error { Io(#[from] io::Error), #[error("Receiver error {0}")] Recv(#[from] RecvError), - #[error("Sender error {0}")] - Send(#[from] SendError), #[error("Stream done")] StreamDone, #[error("Lines codec error {0}")] Codec(#[from] LinesCodecError), #[error("Serde error {0}")] Json(#[from] serde_json::error::Error), - #[error("Couldn't fill stream")] - Stream(#[from] crate::base::bridge::Error), } #[derive(Debug, Clone)] @@ -43,9 +39,12 @@ pub struct TcpJson { } impl TcpJson { - pub async fn new(name: String, config: AppConfig, bridge: BridgeTx) -> TcpJson { - let actions_rx = bridge.register_action_routes(&config.actions).await; - + pub fn new( + name: String, + config: AppConfig, + actions_rx: Option>, + bridge: BridgeTx, + ) -> TcpJson { // Note: We can register `TcpJson` itself as an app to direct actions to it TcpJson { name, config, bridge, actions_rx } } diff --git a/uplink/src/collector/tunshell.rs b/uplink/src/collector/tunshell.rs index 4019ceb2f..21608a28a 100644 --- a/uplink/src/collector/tunshell.rs +++ b/uplink/src/collector/tunshell.rs @@ -1,12 +1,10 @@ +use flume::Receiver; use log::error; use serde::{Deserialize, Serialize}; use tokio_compat_02::FutureExt; use tunshell_client::{Client, ClientMode, Config, HostShell}; -use crate::{ - base::{bridge::BridgeTx, ActionRoute}, - Action, ActionResponse, -}; +use crate::{base::bridge::BridgeTx, Action, ActionResponse}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -27,12 +25,13 @@ pub struct Keys { #[derive(Debug, Clone)] pub struct TunshellClient { + actions_rx: Receiver, bridge: BridgeTx, } impl TunshellClient { - pub fn new(bridge: BridgeTx) -> Self { - Self { bridge } + pub fn new(actions_rx: Receiver, bridge: BridgeTx) -> Self { + Self { actions_rx, bridge } } fn config(&self, keys: Keys) -> Config { @@ -50,10 +49,7 @@ impl TunshellClient { #[tokio::main(flavor = "current_thread")] pub async fn start(self) { - let route = ActionRoute { name: "launch_shell".to_owned(), timeout: 10 }; - let actions_rx = self.bridge.register_action_route(route).await; - - while let Ok(action) = actions_rx.recv_async().await { + while let Ok(action) = self.actions_rx.recv_async().await { let session = self.clone(); //TODO(RT): Findout why this is spawned. We want to send other action's with shell? tokio::spawn(async move { diff --git a/uplink/src/config.rs b/uplink/src/config.rs new file mode 100644 index 000000000..2f909f6fc --- /dev/null +++ b/uplink/src/config.rs @@ -0,0 +1,356 @@ +use std::cmp::Ordering; +use std::env::current_dir; +use std::path::PathBuf; +use std::time::Duration; +use std::{collections::HashMap, fmt::Debug}; + +use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DurationSeconds}; + +pub use crate::base::bridge::stream::MAX_BATCH_SIZE; +#[cfg(target_os = "linux")] +use crate::collector::journalctl::JournalCtlConfig; +#[cfg(target_os = "android")] +use crate::collector::logcat::LogcatConfig; + +pub const DEFAULT_TIMEOUT: u64 = 60; + +#[inline] +fn default_timeout() -> Duration { + Duration::from_secs(DEFAULT_TIMEOUT) +} + +#[inline] +fn max_batch_size() -> usize { + MAX_BATCH_SIZE +} + +pub fn default_file_size() -> usize { + 10485760 // 10MB +} + +fn default_persistence_path() -> PathBuf { + let mut path = current_dir().expect("Couldn't figure out current directory"); + path.push(".persistence"); + path +} + +fn default_download_path() -> PathBuf { + let mut path = current_dir().expect("Couldn't figure out current directory"); + path.push(".downloads"); + path +} + +// Automatically assigns port 5050 for default main app, if left unconfigured +fn default_tcpapps() -> HashMap { + let mut apps = HashMap::new(); + apps.insert("main".to_string(), AppConfig { port: 5050, actions: vec![] }); + + apps +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, Default, PartialEq, Eq, PartialOrd)] +pub enum Compression { + #[default] + Disabled, + Lz4, +} + +#[serde_as] +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct StreamConfig { + pub topic: String, + #[serde(default = "max_batch_size")] + pub batch_size: usize, + #[serde(default = "default_timeout")] + #[serde_as(as = "DurationSeconds")] + /// Duration(in seconds) that bridge collector waits from + /// receiving first element, before the stream gets flushed. + pub flush_period: Duration, + #[serde(default)] + pub compression: Compression, + #[serde(default)] + pub persistence: Persistence, + #[serde(default)] + pub priority: u8, +} + +impl Default for StreamConfig { + fn default() -> Self { + Self { + topic: "".to_string(), + batch_size: MAX_BATCH_SIZE, + flush_period: default_timeout(), + compression: Compression::Disabled, + persistence: Persistence::default(), + priority: 0, + } + } +} + +impl Ord for StreamConfig { + fn cmp(&self, other: &Self) -> Ordering { + match (self.priority.cmp(&other.priority), self.topic.cmp(&other.topic)) { + (Ordering::Equal, o) => o, + (o, _) => o.reverse(), + } + } +} + +impl PartialOrd for StreamConfig { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq, PartialOrd)] +pub struct Persistence { + #[serde(default = "default_file_size")] + pub max_file_size: usize, + #[serde(default)] + pub max_file_count: usize, +} + +impl Default for Persistence { + fn default() -> Self { + Persistence { max_file_size: default_file_size(), max_file_count: 0 } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Authentication { + pub ca_certificate: String, + pub device_certificate: String, + pub device_private_key: String, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct Stats { + pub enabled: bool, + pub process_names: Vec, + pub update_period: u64, + pub stream_size: Option, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct SimulatorConfig { + /// path to directory containing files with gps paths to be used in simulation + pub gps_paths: String, + /// actions that are to be routed to simulator + pub actions: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct DownloaderConfig { + #[serde(default = "default_download_path")] + pub path: PathBuf, + pub actions: Vec, +} + +impl Default for DownloaderConfig { + fn default() -> Self { + Self { path: default_download_path(), actions: vec![] } + } +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct InstallerConfig { + pub path: String, + pub actions: Vec, + pub uplink_port: u16, +} + +#[serde_as] +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct StreamMetricsConfig { + pub enabled: bool, + pub bridge_topic: String, + pub serializer_topic: String, + pub blacklist: Vec, + #[serde_as(as = "DurationSeconds")] + pub timeout: Duration, +} + +#[serde_as] +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct SerializerMetricsConfig { + pub enabled: bool, + pub topic: String, + #[serde_as(as = "DurationSeconds")] + pub timeout: Duration, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct MqttMetricsConfig { + pub enabled: bool, + pub topic: String, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct AppConfig { + pub port: u16, + #[serde(default)] + pub actions: Vec, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConsoleConfig { + pub enabled: bool, + pub port: u16, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct MqttConfig { + pub max_packet_size: usize, + pub max_inflight: u16, + pub keep_alive: u64, + pub network_timeout: u64, +} + +#[serde_as] +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ActionRoute { + pub name: String, + #[serde(default = "default_timeout")] + #[serde_as(as = "DurationSeconds")] + pub timeout: Duration, +} + +impl From<&ActionRoute> for ActionRoute { + fn from(value: &ActionRoute) -> Self { + value.clone() + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct DeviceShadowConfig { + pub interval: u64, +} + +impl Default for DeviceShadowConfig { + fn default() -> Self { + Self { interval: DEFAULT_TIMEOUT } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct PreconditionCheckerConfig { + pub path: PathBuf, + pub actions: Vec, +} + +fn default_true() -> bool { + true +} + +#[derive(Clone, Debug, Deserialize)] +pub struct LogReaderConfig { + pub path: String, + pub stream_name: String, + pub log_template: String, + pub timestamp_template: String, + #[serde(default = "default_true")] + pub multi_line: bool, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct PrometheusConfig { + pub endpoint: String, + pub interval: u64, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct QueryLogConfig { + pub stream: String, + pub where_clause: String, + pub interval: u64, +} + +fn default_clickhouse_host() -> String { + "localhost".to_string() +} + +fn default_clickhouse_port() -> u16 { + 8123 +} + +fn default_clickhouse_username() -> String { + std::env::var("CLICKHOUSE_USERNAME").expect("The env variable CLICKHOUSE_USERNAME is not set") +} + +fn default_clickhouse_password() -> String { + std::env::var("CLICKHOUSE_PASSWORD").expect("The env variable CLICKHOUSE_PASSWORD is not set") +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ClickhouseConfig { + #[serde(default = "default_clickhouse_host")] + pub host: String, + #[serde(default = "default_clickhouse_port")] + pub port: u16, + #[serde(default = "default_clickhouse_username")] + pub username: String, + #[serde(default = "default_clickhouse_password")] + pub password: String, + pub query_log: Option, +} + +impl Default for ClickhouseConfig { + fn default() -> Self { + Self { + host: default_clickhouse_host(), + port: default_clickhouse_port(), + username: default_clickhouse_username(), + password: default_clickhouse_password(), + query_log: None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct Config { + pub project_id: String, + pub device_id: String, + pub broker: String, + pub port: u16, + #[serde(default)] + pub console: ConsoleConfig, + pub authentication: Option, + #[serde(default = "default_tcpapps")] + pub tcpapps: HashMap, + pub mqtt: MqttConfig, + #[serde(default)] + pub processes: Vec, + #[serde(default)] + pub script_runner: Vec, + #[serde(skip)] + pub actions_subscription: String, + pub streams: HashMap, + #[serde(default = "default_persistence_path")] + pub persistence_path: PathBuf, + pub action_status: StreamConfig, + pub stream_metrics: StreamMetricsConfig, + pub serializer_metrics: SerializerMetricsConfig, + pub mqtt_metrics: MqttMetricsConfig, + #[serde(default)] + pub downloader: DownloaderConfig, + pub system_stats: Stats, + pub simulator: Option, + #[serde(default)] + pub ota_installer: InstallerConfig, + #[serde(default)] + pub device_shadow: DeviceShadowConfig, + #[serde(default)] + pub action_redirections: HashMap, + #[serde(default)] + pub ignore_actions_if_no_clients: bool, + #[cfg(target_os = "linux")] + pub logging: Option, + #[cfg(target_os = "android")] + pub logging: Option, + pub precondition_checks: Option, + #[serde(default)] + pub log_reader: HashMap, + pub prometheus: Option, + pub clickhouse: Option, +} diff --git a/uplink/src/console.rs b/uplink/src/console.rs index 8b3b48255..ea787c39f 100644 --- a/uplink/src/console.rs +++ b/uplink/src/console.rs @@ -1,23 +1,39 @@ -use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Router}; +use std::sync::{Arc, Mutex}; + +use axum::{ + extract::State, + http::StatusCode, + response::IntoResponse, + routing::{post, put}, + Router, +}; use log::info; -use uplink::base::bridge::BridgeTx; +use uplink::base::CtrlTx; use crate::ReloadHandle; #[derive(Debug, Clone)] struct StateHandle { reload_handle: ReloadHandle, - bridge_handle: BridgeTx, + ctrl_tx: CtrlTx, + downloader_disable: Arc>, } #[tokio::main] -pub async fn start(port: u16, reload_handle: ReloadHandle, bridge_handle: BridgeTx) { +pub async fn start( + port: u16, + reload_handle: ReloadHandle, + ctrl_tx: CtrlTx, + downloader_disable: Arc>, +) { let address = format!("0.0.0.0:{port}"); info!("Starting uplink console server: {address}"); - let state = StateHandle { reload_handle, bridge_handle }; + let state = StateHandle { reload_handle, ctrl_tx, downloader_disable }; let app = Router::new() .route("/logs", post(reload_loglevel)) .route("/shutdown", post(shutdown)) + .route("/disable_downloader", put(disable_downloader)) + .route("/enable_downloader", put(enable_downloader)) .with_state(state); axum::Server::bind(&address.parse().unwrap()).serve(app.into_make_service()).await.unwrap(); @@ -34,7 +50,31 @@ async fn reload_loglevel(State(state): State, filter: String) -> im async fn shutdown(State(state): State) -> impl IntoResponse { info!("Shutting down uplink"); - state.bridge_handle.trigger_shutdown().await; + state.ctrl_tx.trigger_shutdown().await; StatusCode::OK } + +// Stops downloader from downloading even if it was already stopped +async fn disable_downloader(State(state): State) -> impl IntoResponse { + info!("Downloader stopped"); + let mut is_disabled = state.downloader_disable.lock().unwrap(); + if *is_disabled { + StatusCode::ACCEPTED + } else { + *is_disabled = true; + StatusCode::OK + } +} + +// Start downloader back up even if it was already not stopped +async fn enable_downloader(State(state): State) -> impl IntoResponse { + info!("Downloader started"); + let mut is_disabled = state.downloader_disable.lock().unwrap(); + if *is_disabled { + *is_disabled = false; + StatusCode::OK + } else { + StatusCode::ACCEPTED + } +} diff --git a/uplink/src/lib.rs b/uplink/src/lib.rs index 3d0f7f00e..4adf25f2f 100644 --- a/uplink/src/lib.rs +++ b/uplink/src/lib.rs @@ -4,273 +4,86 @@ //! by [`Mqtt`] and [`Serializer`] respectively. [`Action`]s are received and forwarded by [`Mqtt`] to the [`Bridge`] module, where it is handled //! depending on the [`name`], with [`Bridge`] forwarding it to one of many **Action Handlers**, configured with an [`ActionRoute`]. //! -//! Some of the action handlers are [`TcpJson`], [`ProcessHandler`], [`FileDownloader`] and [`TunshellSession`]. [`TcpJson`] forwards Actions received +//! Some of the action handlers are [`TcpJson`], [`ProcessHandler`], [`FileDownloader`] and [`TunshellClient`]. [`TcpJson`] forwards Actions received //! from the platform to the application connected to it through the [`port`] and collects response data from these devices, to forward to the platform. //! Response data can be of multiple types, of interest to us are [`ActionResponse`]s and data [`Payload`]s, which are forwarded to [`Bridge`] and from //! there to the [`Serializer`], where depending on the network, it may be persisted in-memory or on-disk with [`Storage`]. //! //!```text -//! ┌───────────┐ -//! │MQTT broker│ -//! └────┐▲─────┘ -//! ││ -//! Action ││ ActionResponse -//! ││ / Data -//! ┌─▼└─┐ -//! ┌──────────┤Mqtt◄─────────┐ -//! Action │ └────┘ │ ActionResponse -//! │ │ / Data -//! │ │ -//! ┌──▼───┐ ActionResponse ┌────┴─────┐ Publish ┌───────┐ -//! ┌───────────────────────►Bridge├────────────────►Serializer◄─────────────►Storage| -//! │ └┬─┬┬─┬┘ / Data └──────────┘ Packets └───────┘ -//! │ │ ││ │ -//! │ │ ││ | Action (BridgeTx) -//! │ ┌───────────────┘ ││ └────────────────────┐ -//! │ │ ┌─────┘└───────┐ │ -//! │ ------│-----------│--------------│--------------│------ -//! │ ' │ │ Applications │ │ ' -//! │ '┌────▼───┐ ┌───▼───┐ ┌──────▼───────┐ ┌───▼───┐ ' Action ┌───────────┐ -//! │ '│Tunshell│ │Process│ │FileDownloader│ │TcpJson◄───────────────────►Application│ -//! │ '└────┬───┘ └───┬───┘ └──────┬───────┘ └───┬───┘ ' ActionResponse │ / Device │ -//! │ ' │ │ │ │ ' / Data └───────────┘ -//! │ ------│-----------│--------------│--------------│------ -//! │ │ │ │ │ -//! └────────┴───────────┴──────────────┴──────────────┘ -//! ActionResponse / Data +//! ┌───────────┐ +//! │MQTT broker│ +//! └────┐▲─────┘ +//! Action ││ ActionResponse +//! ││ / Data +//! Action ┌─▼└─┐ +//! ┌────────────┤Mqtt◄──────────┐ ActionResponse +//! │ └────┘ │ / Data +//! │ ActionResponse ┌────┴─────┐ Publish ┌───────┐ +//! │ ┌───────┬───────►Serializer◄─────────────►Storage│ +//! │ │ │ Data └──────────┘ Packets └───────┘ +//! ┌------│-------│-------│-----┐ +//! '┌─────▼─────┐ │ ┌────┴────┐' +//! ┌────────────►Action Lane├─┘ │Data Lane◄──────┐ +//! │ '└──┬─┬─┬─┬──┘ └─────────┘' │ +//! │ ' │ │ │ │ Bridge ' │ +//! │ └---│-│-│-│------------------┘ │ +//! │ │ │ │ └─────────────────────┐ │ +//! │ ┌────────┘ │ └──────────┐ │ │ +//! │┌-----│----------│------------│------------│--│--┐ +//! │' │ │Applications│ │ │ ' +//! │'┌────▼───┐ ┌────▼──┐ ┌──────▼───────┐ ┌──▼──┴─┐' Action ┌───────────┐ +//! │'│Tunshell│ │Process│ │FileDownloader│ │TcpJson◄─────────────────►Application│ +//! │'└────┬───┘ └───┬───┘ └──────┬───────┘ └───┬───┘' ActionResponse │ / Device │ +//! │└-----│---------│-------------│-------------│----┘ / Data └───────────┘ +//! └──────┴─────────┴─────────────┴─────────────┘ +//! ActionResponse //!``` //! [`port`]: base::AppConfig#structfield.port //! [`name`]: Action#structfield.name -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::thread; +use std::time::Duration; use anyhow::Error; - -use base::bridge::stream::Stream; -use base::monitor::Monitor; use collector::clickhouse::ClickhouseReader; -use collector::device_shadow::DeviceShadow; -use collector::downloader::FileDownloader; -use collector::installer::OTAInstaller; -use collector::log_reader::LogFileReader; -use collector::process::ProcessHandler; -use collector::script_runner::ScriptRunner; -use collector::systemstats::StatCollector; -use collector::tunshell::TunshellClient; +use collector::prometheus::Prometheus; use flume::{bounded, Receiver, RecvError, Sender}; use log::error; pub mod base; pub mod collector; +pub mod config; -pub mod config { - use crate::base::{bridge::stream::MAX_BUFFER_SIZE, StreamConfig}; - pub use crate::base::{Config, Persistence, Stats}; - use config::{Environment, File, FileFormat}; - use std::fs; - use structopt::StructOpt; - - #[derive(StructOpt, Debug)] - #[structopt(name = "uplink", about = "collect, batch, compress, publish")] - pub struct CommandLine { - /// Binary version - #[structopt(skip = env ! ("VERGEN_BUILD_SEMVER"))] - pub version: String, - /// Build profile - #[structopt(skip = env ! ("VERGEN_CARGO_PROFILE"))] - pub profile: String, - /// Commit SHA - #[structopt(skip = env ! ("VERGEN_GIT_SHA"))] - pub commit_sha: String, - /// Commit SHA - #[structopt(skip = env ! ("VERGEN_GIT_COMMIT_TIMESTAMP"))] - pub commit_date: String, - /// config file - #[structopt(short = "c", help = "Config file")] - pub config: Option, - /// config file - #[structopt(short = "a", help = "Authentication file")] - pub auth: String, - /// log level (v: info, vv: debug, vvv: trace) - #[structopt(short = "v", long = "verbose", parse(from_occurrences))] - pub verbose: u8, - /// list of modules to log - #[structopt(short = "m", long = "modules")] - pub modules: Vec, - } - - const DEFAULT_CONFIG: &str = r#" - [mqtt] - max_packet_size = 256000 - max_inflight = 100 - keep_alive = 30 - network_timeout = 30 - - # Downloader config - [downloader] - actions = [] - path = "/var/tmp/ota-file" - - [stream_metrics] - enabled = false - topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_stream_metrics/jsonarray" - blacklist = [] - timeout = 10 - - [serializer_metrics] - enabled = false - topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_serializer_metrics/jsonarray" - timeout = 10 - - [mqtt_metrics] - enabled = true - topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_mqtt_metrics/jsonarray" - - [action_status] - topic = "/tenants/{tenant_id}/devices/{device_id}/action/status" - flush_period = 2 - - [streams.device_shadow] - topic = "/tenants/{tenant_id}/devices/{device_id}/events/device_shadow/jsonarray" - flush_period = 5 - - [streams.logs] - topic = "/tenants/{tenant_id}/devices/{device_id}/events/logs/jsonarray" - buf_size = 32 - - [system_stats] - enabled = true - process_names = ["uplink"] - update_period = 30 -"#; - - /// Reads config file to generate config struct and replaces places holders - /// like bike id and data version - pub fn initialize(auth_config: &str, uplink_config: &str) -> Result { - let config = config::Config::builder() - .add_source(File::from_str(DEFAULT_CONFIG, FileFormat::Toml)) - .add_source(File::from_str(uplink_config, FileFormat::Toml)) - .add_source(File::from_str(auth_config, FileFormat::Json)) - .add_source(Environment::default()) - .build()?; - - let mut config: Config = config.try_deserialize()?; - - // Create directory at persistence_path if it doesn't already exist - fs::create_dir_all(&config.persistence_path).map_err(|_| { - anyhow::Error::msg(format!( - "Permission denied for creating persistence directory at \"{}\"", - config.persistence_path.display() - )) - })?; - - // replace placeholders with device/tenant ID - let tenant_id = config.project_id.trim(); - let device_id = config.device_id.trim(); - for config in config.streams.values_mut() { - replace_topic_placeholders(&mut config.topic, tenant_id, device_id); - } - - replace_topic_placeholders(&mut config.action_status.topic, tenant_id, device_id); - replace_topic_placeholders(&mut config.stream_metrics.topic, tenant_id, device_id); - replace_topic_placeholders(&mut config.serializer_metrics.topic, tenant_id, device_id); - replace_topic_placeholders(&mut config.mqtt_metrics.topic, tenant_id, device_id); - - // for config in [&mut config.serializer_metrics, &mut config.stream_metrics] { - // if let Some(topic) = &config.topic { - // let topic = topic.replace("{tenant_id}", tenant_id); - // let topic = topic.replace("{device_id}", device_id); - // config.topic = Some(topic); - // } - // } - - if config.system_stats.enabled { - for stream_name in [ - "uplink_disk_stats", - "uplink_network_stats", - "uplink_processor_stats", - "uplink_process_stats", - "uplink_component_stats", - "uplink_system_stats", - ] { - config.stream_metrics.blacklist.push(stream_name.to_owned()); - let stream_config = StreamConfig { - topic: format!( - "/tenants/{tenant_id}/devices/{device_id}/events/{stream_name}/jsonarray" - ), - buf_size: config.system_stats.stream_size.unwrap_or(MAX_BUFFER_SIZE), - ..Default::default() - }; - config.streams.insert(stream_name.to_owned(), stream_config); - } - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - if let Some(buf_size) = config.logging.as_ref().and_then(|c| c.stream_size) { - let stream_config = - config.streams.entry("logs".to_string()).or_insert_with(|| StreamConfig { - topic: format!( - "/tenants/{tenant_id}/devices/{device_id}/events/logs/jsonarray" - ), - buf_size: 32, - ..Default::default() - }); - stream_config.buf_size = buf_size; - } - - config.streams.insert("action_status".to_owned(), config.action_status.to_owned()); - - let action_topic_template = "/tenants/{tenant_id}/devices/{device_id}/actions"; - let mut device_action_topic = action_topic_template.to_string(); - replace_topic_placeholders(&mut device_action_topic, tenant_id, device_id); - config.actions_subscription = device_action_topic; - - Ok(config) - } - - // Replace placeholders in topic strings with configured values for tenant_id and device_id - fn replace_topic_placeholders(topic: &mut String, tenant_id: &str, device_id: &str) { - *topic = topic.replace("{tenant_id}", tenant_id); - *topic = topic.replace("{device_id}", device_id); - } - - #[derive(Debug, thiserror::Error)] - pub enum ReadFileError { - #[error("Auth file not found at {0}")] - Auth(String), - #[error("Config file not found at {0}")] - Config(String), - } - - fn read_file_contents(path: &str) -> Option { - fs::read_to_string(path).ok() - } - - pub fn get_configs( - commandline: &CommandLine, - ) -> Result<(String, Option), ReadFileError> { - let auth = read_file_contents(&commandline.auth) - .ok_or_else(|| ReadFileError::Auth(commandline.auth.to_string()))?; - let config = match &commandline.config { - Some(path) => Some( - read_file_contents(path).ok_or_else(|| ReadFileError::Config(path.to_string()))?, - ), - None => None, - }; - - Ok((auth, config)) - } -} - +use self::config::{ActionRoute, Config}; pub use base::actions::{Action, ActionResponse}; -use base::bridge::{Bridge, BridgeTx, Package, Payload, Point, StreamMetrics}; +use base::bridge::{stream::Stream, Bridge, Package, Payload, Point, StreamMetrics}; +use base::monitor::Monitor; use base::mqtt::Mqtt; use base::serializer::{Serializer, SerializerMetrics}; -pub use base::{ActionRoute, Config}; -use collector::prometheus::Prometheus; +use base::CtrlTx; +use collector::device_shadow::DeviceShadow; +use collector::downloader::{CtrlTx as DownloaderCtrlTx, FileDownloader}; +use collector::installer::OTAInstaller; +#[cfg(target_os = "linux")] +use collector::journalctl::JournalCtl; +#[cfg(target_os = "android")] +use collector::logcat::Logcat; +use collector::preconditions::PreconditionChecker; +use collector::process::ProcessHandler; +use collector::script_runner::ScriptRunner; +use collector::systemstats::StatCollector; +use collector::tunshell::TunshellClient; pub use collector::{simulator, tcpjson::TcpJson}; pub use storage::Storage; +/// Spawn a named thread to run the function f on +pub fn spawn_named_thread(name: &str, f: F) +where + F: FnOnce() + Send + 'static, +{ + thread::Builder::new().name(name.to_string()).spawn(f).unwrap(); +} + pub struct Uplink { config: Arc, action_rx: Receiver, @@ -308,36 +121,27 @@ impl Uplink { }) } - pub fn spawn(&mut self) -> Result { - let config = self.config.clone(); - let mut bridge = Bridge::new( + pub fn configure_bridge(&mut self) -> Bridge { + Bridge::new( self.config.clone(), self.data_tx.clone(), self.stream_metrics_tx(), self.action_rx.clone(), self.shutdown_tx.clone(), - ); + ) + } - let bridge_tx = bridge.tx(); + pub fn spawn( + &mut self, + mut bridge: Bridge, + downloader_disable: Arc>, + ) -> Result { let (mqtt_metrics_tx, mqtt_metrics_rx) = bounded(10); - - // Bridge thread to batch data and redicet actions - thread::spawn(|| { - let rt = tokio::runtime::Builder::new_current_thread() - .thread_name("bridge") - .enable_time() - .build() - .unwrap(); - - rt.block_on(async move { - if let Err(e) = bridge.start().await { - error!("Bridge stopped!! Error = {:?}", e); - } - }) - }); + let (ctrl_actions_lane, ctrl_data_lane) = bridge.ctrl_tx(); let mut mqtt = Mqtt::new(self.config.clone(), self.action_tx.clone(), mqtt_metrics_tx); let mqtt_client = mqtt.client(); + let ctrl_mqtt = mqtt.ctrl_tx(); let serializer = Serializer::new( self.config.clone(), @@ -345,15 +149,28 @@ impl Uplink { mqtt_client.clone(), self.serializer_metrics_tx(), )?; + let ctrl_serializer = serializer.ctrl_tx(); + + let (ctrl_tx, ctrl_rx) = bounded(1); + let ctrl_downloader = DownloaderCtrlTx { inner: ctrl_tx }; + + // Downloader thread if configured + if !self.config.downloader.actions.is_empty() { + let actions_rx = bridge.register_action_routes(&self.config.downloader.actions)?; + let file_downloader = FileDownloader::new( + self.config.clone(), + actions_rx, + bridge.bridge_tx(), + ctrl_rx, + downloader_disable, + )?; + spawn_named_thread("File Downloader", || file_downloader.start()); + } // Serializer thread to handle network conditions state machine // and send data to mqtt thread - thread::spawn(|| { - let rt = tokio::runtime::Builder::new_current_thread() - .thread_name("serializer") - .enable_time() - .build() - .unwrap(); + spawn_named_thread("Serializer", || { + let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); rt.block_on(async { if let Err(e) = serializer.start().await { @@ -363,9 +180,8 @@ impl Uplink { }); // Mqtt thread to receive actions and send data - thread::spawn(|| { + spawn_named_thread("Mqttio", || { let rt = tokio::runtime::Builder::new_current_thread() - .thread_name("mqttio") .enable_time() .enable_io() .build() @@ -376,98 +192,148 @@ impl Uplink { }) }); - let tunshell_client = TunshellClient::new(bridge_tx.clone()); - thread::spawn(move || tunshell_client.start()); - - let file_downloader = FileDownloader::new(config.clone(), bridge_tx.clone())?; - thread::spawn(move || file_downloader.start()); - - let device_shadow = DeviceShadow::new(config.device_shadow.clone(), bridge_tx.clone()); - thread::spawn(move || device_shadow.start()); - - if let Some(config) = &config.ota_installer { - let ota_installer = OTAInstaller::new(config.clone(), bridge_tx.clone()); - thread::spawn(move || ota_installer.start()); - } + let monitor = Monitor::new( + self.config.clone(), + mqtt_client, + self.stream_metrics_rx.clone(), + self.serializer_metrics_rx.clone(), + mqtt_metrics_rx, + ); - #[cfg(target_os = "linux")] - if let Some(config) = &config.logging { - let logger = collector::journalctl::JournalCtl::new(bridge_tx.clone()); - let config = config.clone(); - thread::spawn(move || logger.start(config)); - } + // Metrics monitor thread + spawn_named_thread("Monitor", || { + let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); - #[cfg(target_os = "android")] - if let Some(config) = &config.logging { - let logger = collector::logcat::Logcat::new(bridge_tx.clone()); - let config = config.clone(); - thread::spawn(move || logger.start(config)); - } + rt.block_on(async move { + if let Err(e) = monitor.start().await { + error!("Monitor stopped!! Error = {:?}", e); + } + }) + }); - if config.system_stats.enabled { - let stat_collector = StatCollector::new(config.clone(), bridge_tx.clone()); - thread::spawn(move || stat_collector.start()); - } + let Bridge { data: mut data_lane, actions: mut actions_lane, .. } = bridge; - let process_handler = ProcessHandler::new(bridge_tx.clone()); - let processes = config.processes.clone(); - thread::spawn(move || process_handler.start(processes)); + // Bridge thread to direct actions + spawn_named_thread("Bridge actions_lane", || { + let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); - for (_logger, config) in config.log_reader.iter() { - let stdout_collector = LogFileReader::new(config.clone(), bridge_tx.clone()); - thread::spawn(move || stdout_collector.start()); - } + rt.block_on(async move { + if let Err(e) = actions_lane.start().await { + error!("Actions lane stopped!! Error = {:?}", e); + } + }) + }); - let script_runner = ScriptRunner::new(bridge_tx.clone()); - let routes: Vec = config.script_runner.clone(); - thread::spawn(move || { - let rt = tokio::runtime::Builder::new_current_thread() - .thread_name("script_runner") - .enable_io() - .enable_time() - .build() - .unwrap(); + // Bridge thread to batch and forward data + spawn_named_thread("Bridge data_lane", || { + let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); rt.block_on(async move { - if let Err(e) = script_runner.start(routes).await { - error!("Monitor stopped!! Error = {:?}", e); + if let Err(e) = data_lane.start().await { + error!("Data lane stopped!! Error = {:?}", e); } }) }); - let monitor = Monitor::new( - self.config.clone(), - mqtt_client, - self.stream_metrics_rx.clone(), - self.serializer_metrics_rx.clone(), - mqtt_metrics_rx, - ); + Ok(CtrlTx { + actions_lane: ctrl_actions_lane, + data_lane: ctrl_data_lane, + mqtt: ctrl_mqtt, + serializer: ctrl_serializer, + downloader: ctrl_downloader, + }) + } + + pub fn spawn_builtins(&mut self, bridge: &mut Bridge) -> Result<(), Error> { + let bridge_tx = bridge.bridge_tx(); + + let route = + ActionRoute { name: "launch_shell".to_owned(), timeout: Duration::from_secs(10) }; + let actions_rx = bridge.register_action_route(route)?; + let tunshell_client = TunshellClient::new(actions_rx, bridge_tx.clone()); + spawn_named_thread("Tunshell Client", move || tunshell_client.start()); + + let device_shadow = DeviceShadow::new(self.config.device_shadow.clone(), bridge_tx.clone()); + spawn_named_thread("Device Shadow Generator", move || device_shadow.start()); + + if !self.config.ota_installer.actions.is_empty() { + let actions_rx = bridge.register_action_routes(&self.config.ota_installer.actions)?; + let ota_installer = + OTAInstaller::new(self.config.ota_installer.clone(), actions_rx, bridge_tx.clone()); + spawn_named_thread("OTA Installer", move || ota_installer.start()); + } + + #[cfg(target_os = "linux")] + if let Some(config) = self.config.logging.clone() { + let route = ActionRoute { + name: "journalctl_config".to_string(), + timeout: Duration::from_secs(10), + }; + let actions_rx = bridge.register_action_route(route)?; + let logger = JournalCtl::new(config, actions_rx, bridge_tx.clone()); + spawn_named_thread("Logger", || { + if let Err(e) = logger.start() { + error!("Logger stopped!! Error = {:?}", e); + } + }); + } if let Some(config) = self.config.prometheus.clone() { let prometheus = Prometheus::new(config, bridge_tx.clone()); - thread::spawn(|| prometheus.start()); + spawn_named_thread("prometheus", || prometheus.start()); } if let Some(config) = self.config.clickhouse.clone() { let clickhouse_reader = ClickhouseReader::new(config, bridge_tx.clone()); - thread::spawn(|| clickhouse_reader.start()); + spawn_named_thread("clickhouse reader", || clickhouse_reader.start()); } - // Metrics monitor thread - thread::spawn(|| { - let rt = tokio::runtime::Builder::new_current_thread() - .thread_name("monitor") - .enable_time() - .build() - .unwrap(); + #[cfg(target_os = "android")] + if let Some(config) = self.config.logging.clone() { + let route = ActionRoute { + name: "journalctl_config".to_string(), + timeout: Duration::from_secs(10), + }; + let actions_rx = bridge.register_action_route(route)?; + let logger = Logcat::new(config, actions_rx, bridge_tx.clone()); + spawn_named_thread("Logger", || { + if let Err(e) = logger.start() { + error!("Logger stopped!! Error = {:?}", e); + } + }); + } - rt.block_on(async move { - if let Err(e) = monitor.start().await { - error!("Monitor stopped!! Error = {:?}", e); + if self.config.system_stats.enabled { + let stat_collector = StatCollector::new(self.config.clone(), bridge_tx.clone()); + spawn_named_thread("Stat Collector", || stat_collector.start()); + }; + + if !self.config.processes.is_empty() { + let actions_rx = bridge.register_action_routes(&self.config.processes)?; + let process_handler = ProcessHandler::new(actions_rx, bridge_tx.clone()); + spawn_named_thread("Process Handler", || { + if let Err(e) = process_handler.start() { + error!("Process handler stopped!! Error = {:?}", e); } - }) - }); + }); + } + + if !self.config.script_runner.is_empty() { + let actions_rx = bridge.register_action_routes(&self.config.script_runner)?; + let script_runner = ScriptRunner::new(actions_rx, bridge_tx.clone()); + spawn_named_thread("Script Runner", || { + if let Err(e) = script_runner.start() { + error!("Script runner stopped!! Error = {:?}", e); + } + }); + } + + if let Some(checker_config) = &self.config.precondition_checks { + let actions_rx = bridge.register_action_routes(&checker_config.actions)?; + let checker = PreconditionChecker::new(self.config.clone(), actions_rx, bridge_tx); + spawn_named_thread("Logger", || checker.start()); + } - Ok(bridge_tx) + Ok(()) } pub fn bridge_action_rx(&self) -> Receiver { diff --git a/uplink/src/main.rs b/uplink/src/main.rs index a9386efb7..1809bb58c 100644 --- a/uplink/src/main.rs +++ b/uplink/src/main.rs @@ -1,10 +1,11 @@ mod console; -use std::sync::Arc; -use std::thread; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; use std::time::Duration; use anyhow::Error; +use config::{Environment, File, FileFormat}; use log::info; use structopt::StructOpt; use tokio::time::sleep; @@ -16,91 +17,257 @@ use tracing_subscriber::{EnvFilter, Registry}; pub type ReloadHandle = Handle>, Registry>>; -use uplink::base::AppConfig; -use uplink::config::{get_configs, initialize, CommandLine}; -use uplink::{simulator, Config, TcpJson, Uplink}; +use uplink::config::{AppConfig, Config, StreamConfig, MAX_BATCH_SIZE}; +use uplink::{simulator, spawn_named_thread, TcpJson, Uplink}; -fn initialize_logging(commandline: &CommandLine) -> ReloadHandle { - let level = match commandline.verbose { - 0 => "warn", - 1 => "info", - 2 => "debug", - _ => "trace", - }; +const DEFAULT_CONFIG: &str = r#" + [mqtt] + max_packet_size = 256000 + max_inflight = 100 + keep_alive = 30 + network_timeout = 30 - let levels = - match commandline.modules.clone().into_iter().reduce(|e, acc| format!("{e}={level},{acc}")) - { - Some(f) => format!("{f}={level}"), - _ => format!("uplink={level},disk={level}"), - }; + [stream_metrics] + enabled = false + bridge_topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_stream_metrics/jsonarray" + serializer_topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_serializer_stream_metrics/jsonarray" + blacklist = [] + timeout = 10 + + [serializer_metrics] + enabled = false + topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_serializer_metrics/jsonarray" + timeout = 10 + + [mqtt_metrics] + enabled = true + topic = "/tenants/{tenant_id}/devices/{device_id}/events/uplink_mqtt_metrics/jsonarray" - let builder = tracing_subscriber::fmt() - .pretty() - .with_line_number(false) - .with_file(false) - .with_thread_ids(false) - .with_thread_names(false) - .with_env_filter(levels) - .with_filter_reloading(); + [action_status] + topic = "/tenants/{tenant_id}/devices/{device_id}/action/status" + batch_size = 1 + flush_period = 2 + priority = 255 # highest priority for quick delivery of action status info to platform - let reload_handle = builder.reload_handle(); + [streams.device_shadow] + topic = "/tenants/{tenant_id}/devices/{device_id}/events/device_shadow/jsonarray" + flush_period = 5 - builder.try_init().expect("initialized subscriber succesfully"); + [streams.logs] + topic = "/tenants/{tenant_id}/devices/{device_id}/events/logs/jsonarray" + batch_size = 32 - reload_handle + [system_stats] + enabled = true + process_names = ["uplink"] + update_period = 30 +"#; + +#[derive(StructOpt, Debug)] +#[structopt(name = "uplink", about = "collect, batch, compress, publish")] +pub struct CommandLine { + /// Binary version + #[structopt(skip = env ! ("VERGEN_BUILD_SEMVER"))] + pub version: String, + /// Build profile + #[structopt(skip = env ! ("VERGEN_CARGO_PROFILE"))] + pub profile: String, + /// Commit SHA + #[structopt(skip = env ! ("VERGEN_GIT_SHA"))] + pub commit_sha: String, + /// Commit SHA + #[structopt(skip = env ! ("VERGEN_GIT_COMMIT_TIMESTAMP"))] + pub commit_date: String, + /// config file + #[structopt(short = "c", help = "Config file")] + pub config: Option, + /// config file + #[structopt(short = "a", help = "Authentication file")] + pub auth: PathBuf, + /// log level (v: info, vv: debug, vvv: trace) + #[structopt(short = "v", long = "verbose", parse(from_occurrences))] + pub verbose: u8, + /// list of modules to log + #[structopt(short = "m", long = "modules")] + pub modules: Vec, } -fn banner(commandline: &CommandLine, config: &Arc) { - const B: &str = r#" - ░█░▒█░▄▀▀▄░█░░░▀░░█▀▀▄░█░▄ - ░█░▒█░█▄▄█░█░░░█▀░█░▒█░█▀▄ - ░░▀▀▀░█░░░░▀▀░▀▀▀░▀░░▀░▀░▀ - "#; - - println!("{B}"); - println!(" version: {}", commandline.version); - println!(" profile: {}", commandline.profile); - println!(" commit_sha: {}", commandline.commit_sha); - println!(" commit_date: {}", commandline.commit_date); - println!(" project_id: {}", config.project_id); - println!(" device_id: {}", config.device_id); - println!(" remote: {}:{}", config.broker, config.port); - println!(" persistence_path: {}", config.persistence_path.display()); - if !config.action_redirections.is_empty() { - println!(" action redirections:"); - for (action, redirection) in config.action_redirections.iter() { - println!(" {action} -> {redirection}"); +impl CommandLine { + /// Reads config file to generate config struct and replaces places holders + /// like bike id and data version + fn get_configs(&self) -> Result { + let read_file_contents = |path| std::fs::read_to_string(path).ok(); + let auth = read_file_contents(&self.auth).ok_or_else(|| { + Error::msg(format!("Auth file not found at \"{}\"", self.auth.display())) + })?; + let config = match &self.config { + Some(path) => Some(read_file_contents(path).ok_or_else(|| { + Error::msg(format!("Config file not found at \"{}\"", path.display())) + })?), + None => None, + }; + + let config = config::Config::builder() + .add_source(File::from_str(DEFAULT_CONFIG, FileFormat::Toml)) + .add_source(File::from_str(&config.unwrap_or_default(), FileFormat::Toml)) + .add_source(File::from_str(&auth, FileFormat::Json)) + .add_source(Environment::default()) + .build()?; + + let mut config: Config = config.try_deserialize()?; + + // Create directory at persistence_path if it doesn't already exist + std::fs::create_dir_all(&config.persistence_path).map_err(|_| { + Error::msg(format!( + "Permission denied for creating persistence directory at \"{}\"", + config.persistence_path.display() + )) + })?; + + // replace placeholders with device/tenant ID + let tenant_id = config.project_id.trim(); + let device_id = config.device_id.trim(); + + // Replace placeholders in topic strings with configured values for tenant_id and device_id + // e.g. for tenant_id: "demo"; device_id: "123" + // "/tenants/{tenant_id}/devices/{device_id}/events/stream/jsonarry" ~> "/tenants/demo/devices/123/events/stream/jsonarry" + let replace_topic_placeholders = |topic: &mut String| { + *topic = topic.replace("{tenant_id}", tenant_id).replace("{device_id}", device_id); + }; + + for config in config.streams.values_mut() { + replace_topic_placeholders(&mut config.topic); } - } - if !config.tcpapps.is_empty() { - println!(" tcp applications:"); - for (app, AppConfig { port, actions }) in config.tcpapps.iter() { - println!(" name: {app:?}"); - println!(" port: {port}"); - println!(" actions: {actions:?}"); - println!(" @"); + + replace_topic_placeholders(&mut config.action_status.topic); + replace_topic_placeholders(&mut config.stream_metrics.bridge_topic); + replace_topic_placeholders(&mut config.stream_metrics.serializer_topic); + replace_topic_placeholders(&mut config.serializer_metrics.topic); + replace_topic_placeholders(&mut config.mqtt_metrics.topic); + + if config.system_stats.enabled { + for stream_name in [ + "uplink_disk_stats", + "uplink_network_stats", + "uplink_processor_stats", + "uplink_process_stats", + "uplink_component_stats", + "uplink_system_stats", + ] { + config.stream_metrics.blacklist.push(stream_name.to_owned()); + let stream_config = StreamConfig { + topic: format!( + "/tenants/{tenant_id}/devices/{device_id}/events/{stream_name}/jsonarray" + ), + batch_size: config.system_stats.stream_size.unwrap_or(MAX_BATCH_SIZE), + ..Default::default() + }; + config.streams.insert(stream_name.to_owned(), stream_config); + } } + + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(batch_size) = config.logging.as_ref().and_then(|c| c.stream_size) { + let stream_config = + config.streams.entry("logs".to_string()).or_insert_with(|| StreamConfig { + topic: format!( + "/tenants/{tenant_id}/devices/{device_id}/events/logs/jsonarray" + ), + batch_size: 32, + ..Default::default() + }); + stream_config.batch_size = batch_size; + } + + config.actions_subscription = format!("/tenants/{tenant_id}/devices/{device_id}/actions"); + + Ok(config) } - println!(" secure_transport: {}", config.authentication.is_some()); - println!(" max_packet_size: {}", config.mqtt.max_packet_size); - println!(" max_inflight_messages: {}", config.mqtt.max_inflight); - println!(" keep_alive_timeout: {}", config.mqtt.keep_alive); - - println!( - " downloader:\n\tpath: {}\n\tactions: {:?}", - config.downloader.path, config.downloader.actions - ); - if let Some(installer) = &config.ota_installer { - println!(" installer:\n\tpath: {}\n\tactions: {:?}", installer.path, installer.actions); - } - if config.system_stats.enabled { - println!(" processes: {:?}", config.system_stats.process_names); + + fn initialize_logging(&self) -> ReloadHandle { + let level = match self.verbose { + 0 => "warn", + 1 => "info", + 2 => "debug", + _ => "trace", + }; + + let levels = + match self.modules.clone().into_iter().reduce(|e, acc| format!("{e}={level},{acc}")) { + Some(f) => format!("{f}={level}"), + _ => format!("uplink={level},storage={level}"), + }; + + let builder = tracing_subscriber::fmt() + .pretty() + .with_line_number(false) + .with_file(false) + .with_thread_ids(false) + .with_thread_names(false) + .with_env_filter(levels) + .with_filter_reloading(); + + let reload_handle = builder.reload_handle(); + + builder.try_init().expect("initialized subscriber succesfully"); + + reload_handle } - if config.console.enabled { - println!(" console: http://localhost:{}", config.console.port); + + fn banner(&self, config: &Config) { + const B: &str = r#" + ░█░▒█░▄▀▀▄░█░░░▀░░█▀▀▄░█░▄ + ░█░▒█░█▄▄█░█░░░█▀░█░▒█░█▀▄ + ░░▀▀▀░█░░░░▀▀░▀▀▀░▀░░▀░▀░▀ + "#; + + println!("{B}"); + println!(" version: {}", self.version); + println!(" profile: {}", self.profile); + println!(" commit_sha: {}", self.commit_sha); + println!(" commit_date: {}", self.commit_date); + println!(" project_id: {}", config.project_id); + println!(" device_id: {}", config.device_id); + println!(" remote: {}:{}", config.broker, config.port); + println!(" persistence_path: {}", config.persistence_path.display()); + if !config.action_redirections.is_empty() { + println!(" action redirections:"); + for (action, redirection) in config.action_redirections.iter() { + println!("\t{action} -> {redirection}"); + } + } + if !config.tcpapps.is_empty() { + println!(" tcp applications:"); + for (app, AppConfig { port, actions }) in config.tcpapps.iter() { + println!("\tname: {app:?}\n\tport: {port}\n\tactions: {actions:?}\n\t@"); + } + } + println!(" secure_transport: {}", config.authentication.is_some()); + println!(" max_packet_size: {}", config.mqtt.max_packet_size); + println!(" max_inflight_messages: {}", config.mqtt.max_inflight); + println!(" keep_alive_timeout: {}", config.mqtt.keep_alive); + + if !config.downloader.actions.is_empty() { + println!( + " downloader:\n\tpath: \"{}\"\n\tactions: {:?}", + config.downloader.path.display(), + config.downloader.actions + ); + } + if !config.ota_installer.actions.is_empty() { + println!( + " installer:\n\tpath: {}\n\tactions: {:?}", + config.ota_installer.path, config.ota_installer.actions + ); + } + if config.system_stats.enabled { + println!(" processes: {:?}", config.system_stats.process_names); + } + if config.console.enabled { + println!(" console: http://localhost:{}", config.console.port); + } + println!("\n"); } - println!("\n"); } fn main() -> Result<(), Error> { @@ -110,27 +277,51 @@ fn main() -> Result<(), Error> { } let commandline: CommandLine = StructOpt::from_args(); - let reload_handle = initialize_logging(&commandline); + let reload_handle = commandline.initialize_logging(); + let config = commandline.get_configs()?; + commandline.banner(&config); - let (auth, config) = get_configs(&commandline)?; - let config = Arc::new(initialize(&auth, &config.unwrap_or_default())?); + let config = Arc::new(config); + let mut uplink = Uplink::new(config.clone())?; + let mut bridge = uplink.configure_bridge(); + uplink.spawn_builtins(&mut bridge)?; - banner(&commandline, &config); + let bridge_tx = bridge.bridge_tx(); - let mut uplink = Uplink::new(config.clone())?; - let bridge = uplink.spawn()?; + let mut tcpapps = vec![]; + for (app, cfg) in config.tcpapps.clone() { + let route_rx = if !cfg.actions.is_empty() { + let actions_rx = bridge.register_action_routes(&cfg.actions)?; + Some(actions_rx) + } else { + None + }; + tcpapps.push(TcpJson::new(app, cfg, route_rx, bridge.bridge_tx())); + } + + let simulator_actions = match &config.simulator { + Some(cfg) if !cfg.actions.is_empty() => { + let actions_rx = bridge.register_action_routes(&cfg.actions)?; + Some(actions_rx) + } + _ => None, + }; + + let downloader_disable = Arc::new(Mutex::new(false)); + let ctrl_tx = uplink.spawn(bridge, downloader_disable.clone())?; if let Some(config) = config.simulator.clone() { - let bridge = bridge.clone(); - thread::spawn(move || { - simulator::start(config, bridge).unwrap(); + spawn_named_thread("Simulator", || { + simulator::start(config, bridge_tx, simulator_actions).unwrap(); }); } if config.console.enabled { let port = config.console.port; - let bridge_handle = bridge.clone(); - thread::spawn(move || console::start(port, reload_handle, bridge_handle)); + let ctrl_tx = ctrl_tx.clone(); + spawn_named_thread("Uplink Console", move || { + console::start(port, reload_handle, ctrl_tx, downloader_disable) + }); } let rt = tokio::runtime::Builder::new_current_thread() @@ -141,10 +332,9 @@ fn main() -> Result<(), Error> { .unwrap(); rt.block_on(async { - for (app, cfg) in config.tcpapps.iter() { - let tcpjson = TcpJson::new(app.to_owned(), cfg.clone(), bridge.clone()).await; + for app in tcpapps { tokio::task::spawn(async move { - if let Err(e) = tcpjson.start().await { + if let Err(e) = app.start().await { error!("App failed. Error = {:?}", e); } }); @@ -160,7 +350,7 @@ fn main() -> Result<(), Error> { // Handle a shutdown signal from POSIX while let Some(signal) = signals.next().await { match signal { - SIGTERM | SIGINT | SIGQUIT => bridge.trigger_shutdown().await, + SIGTERM | SIGINT | SIGQUIT => ctrl_tx.trigger_shutdown().await, s => error!("Couldn't handle signal: {s}"), } } @@ -169,7 +359,7 @@ fn main() -> Result<(), Error> { uplink.resolve_on_shutdown().await.unwrap(); info!("Uplink shutting down..."); // NOTE: wait 5s to allow serializer to write to network/disk - sleep(Duration::from_secs(5)).await; + sleep(Duration::from_secs(10)).await; }); Ok(())