From b478ae196e8884d0c3259157ae23a1a201383ae3 Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Thu, 2 Nov 2023 11:57:42 +0100 Subject: [PATCH] Partial transition to anyhow / thiserror for errors Signed-off-by: Heinz N. Gies --- .github/checks/deps.sh | 29 -- .github/workflows/checks.yaml | 24 +- .gitignore | 2 + Cargo.lock | 109 ++-- Cargo.toml | 11 +- src/channel.rs | 21 +- src/config.rs | 8 +- src/connectors.rs | 189 ++++--- src/connectors/google.rs | 2 +- src/connectors/impls/bench.rs | 12 +- src/connectors/impls/cb.rs | 40 +- src/connectors/impls/clickhouse.rs | 21 +- src/connectors/impls/clickhouse/conversion.rs | 53 +- src/connectors/impls/cluster_kv.rs | 21 +- src/connectors/impls/crononome.rs | 10 +- src/connectors/impls/discord.rs | 30 +- src/connectors/impls/dns/client.rs | 80 +-- src/connectors/impls/elastic.rs | 91 ++-- src/connectors/impls/exit.rs | 2 +- src/connectors/impls/file.rs | 32 +- src/connectors/impls/gbq.rs | 10 + src/connectors/impls/gbq/writer/sink.rs | 44 +- src/connectors/impls/gcl.rs | 6 + src/connectors/impls/gcl/writer.rs | 99 ++-- src/connectors/impls/gcl/writer/meta.rs | 26 +- src/connectors/impls/gcl/writer/sink.rs | 27 +- src/connectors/impls/gcs.rs | 25 + src/connectors/impls/gcs/chunked_buffer.rs | 9 +- .../impls/gcs/resumable_upload_client.rs | 52 +- src/connectors/impls/gcs/streamer.rs | 131 ++--- src/connectors/impls/gpubsub.rs | 6 + src/connectors/impls/gpubsub/consumer.rs | 36 +- src/connectors/impls/gpubsub/producer.rs | 15 +- src/connectors/impls/http/auth.rs | 13 +- src/connectors/impls/http/client.rs | 29 +- src/connectors/impls/http/meta.rs | 20 +- src/connectors/impls/http/server.rs | 44 +- src/connectors/impls/kafka.rs | 20 +- src/connectors/impls/kafka/consumer.rs | 53 +- src/connectors/impls/kafka/producer.rs | 18 +- src/connectors/impls/kv.rs | 32 +- src/connectors/impls/metrics.rs | 19 +- src/connectors/impls/metronome.rs | 13 +- src/connectors/impls/null.rs | 8 +- src/connectors/impls/oneshot.rs | 6 +- src/connectors/impls/otel/client.rs | 29 +- src/connectors/impls/otel/common.rs | 25 +- src/connectors/impls/otel/id.rs | 16 +- src/connectors/impls/otel/logs.rs | 43 +- src/connectors/impls/otel/metrics.rs | 42 +- src/connectors/impls/otel/resource.rs | 9 +- src/connectors/impls/otel/server.rs | 37 +- src/connectors/impls/otel/trace.rs | 49 +- src/connectors/impls/s3.rs | 10 + src/connectors/impls/s3/reader.rs | 20 +- src/connectors/impls/s3/streamer.rs | 21 +- src/connectors/impls/stdio.rs | 20 +- src/connectors/impls/tcp/client.rs | 35 +- src/connectors/impls/tcp/server.rs | 44 +- src/connectors/impls/udp/client.rs | 13 +- src/connectors/impls/udp/server.rs | 15 +- src/connectors/impls/unix_socket/client.rs | 13 +- src/connectors/impls/unix_socket/server.rs | 23 +- src/connectors/impls/wal.rs | 22 +- src/connectors/impls/ws/client.rs | 30 +- src/connectors/impls/ws/server.rs | 22 +- src/connectors/prelude.rs | 9 +- src/connectors/sink.rs | 59 +-- src/connectors/sink/channel_sink.rs | 15 +- src/connectors/source.rs | 90 ++-- src/connectors/source/channel_source.rs | 24 +- src/connectors/tests.rs | 33 +- src/connectors/tests/bench.rs | 2 +- .../tests/clickhouse/more_complex_test.rs | 28 +- .../tests/clickhouse/simple_test.rs | 4 +- src/connectors/tests/clickhouse/utils.rs | 2 +- src/connectors/tests/elastic.rs | 4 +- src/connectors/tests/file.rs | 2 +- src/connectors/tests/file_non_existent.rs | 2 +- src/connectors/tests/file_xz.rs | 2 +- src/connectors/tests/http/client.rs | 6 +- src/connectors/tests/http/server.rs | 10 +- src/connectors/tests/kafka/consumer.rs | 47 +- src/connectors/tests/kafka/producer.rs | 4 +- src/connectors/tests/s3.rs | 4 +- src/connectors/tests/tcp.rs | 9 +- src/connectors/tests/ws.rs | 31 +- src/connectors/utils/object_storage.rs | 61 +-- src/connectors/utils/pb.rs | 74 +-- src/connectors/utils/quiescence.rs | 21 +- src/connectors/utils/reconnect.rs | 59 +-- src/connectors/utils/socket.rs | 37 +- src/connectors/utils/tls.rs | 56 +- src/errors.rs | 487 +----------------- src/functions.rs | 4 +- src/lib.rs | 4 +- src/pipeline.rs | 97 ++-- src/raft.rs | 216 ++------ src/raft/api.rs | 46 +- src/raft/api/apps.rs | 38 +- src/raft/api/cluster.rs | 91 ++-- src/raft/archive.rs | 62 ++- src/raft/manager.rs | 206 ++++++-- src/raft/node.rs | 41 +- src/raft/store.rs | 178 +++++-- src/raft/store/statemachine.rs | 31 +- src/raft/store/statemachine/apps.rs | 27 +- src/raft/store/statemachine/kv.rs | 17 + src/raft/store/statemachine/nodes.rs | 14 +- src/raft/test.rs | 16 +- src/raft/test/learner.rs | 16 +- src/system.rs | 61 ++- src/system/flow.rs | 168 +++--- src/system/flow_supervisor.rs | 32 +- src/utils.rs | 11 +- tests/query.rs | 8 +- tests/query_error.rs | 39 +- tests/query_runtime_error.rs | 16 +- tests/script.rs | 2 +- tests/script_error.rs | 4 +- tests/script_runtime_error.rs | 4 +- tests/script_warning.rs | 2 +- tremor-cli/Cargo.toml | 7 - tremor-cli/src/cluster.rs | 43 +- tremor-cli/src/errors.rs | 5 +- tremor-cli/src/test/before.rs | 8 +- tremor-codec/src/errors.rs | 3 + tremor-codec/src/lib.rs | 2 +- tremor-interceptor/src/errors.rs | 3 + tremor-interceptor/src/preprocessor.rs | 4 +- tremor-pipeline/Cargo.toml | 3 - tremor-pipeline/src/errors.rs | 7 +- tremor-pipeline/src/op/generic/batch.rs | 16 +- tremor-pipeline/src/op/grouper/bucket.rs | 22 +- tremor-pipeline/src/op/trickle/select.rs | 8 +- tremor-pipeline/src/op/trickle/select/test.rs | 8 +- tremor-pipeline/src/query.rs | 2 +- tremor-script/Cargo.toml | 6 + tremor-script/src/ast.rs | 2 +- tremor-script/src/ast/deploy/raw.rs | 2 +- tremor-script/src/ast/raw.rs | 2 +- tremor-script/src/errors.rs | 7 +- tremor-script/src/interpreter/imut_expr.rs | 2 +- tremor-script/src/lexer.rs | 2 +- tremor-script/src/lib.rs | 2 +- tremor-script/src/registry.rs | 2 +- tremor-script/src/utils.rs | 2 +- tremor-value/Cargo.toml | 2 - tremor-value/src/known_key.rs | 6 +- tremor-value/src/lib.rs | 1 - tremor-value/src/serde/value.rs | 1 - 151 files changed, 2332 insertions(+), 2567 deletions(-) delete mode 100755 .github/checks/deps.sh diff --git a/.github/checks/deps.sh b/.github/checks/deps.sh deleted file mode 100755 index 3f83257113..0000000000 --- a/.github/checks/deps.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -if [ -d "$1" ] -then - path="$1" -else - path="." -fi - -cnt=0 - -for d in $( remarshal -i $path/Cargo.toml -of json | jq -r '.dependencies | keys []' | if [ -f $path/.depignore ]; then grep -v -f $path/.depignore; else cat; fi ) -do - dep=$(echo $d | sed -e 's/-/_/g') - if ! rg "use $dep(::|;| )" $path -trust > /dev/null - then - if ! rg "extern crate $dep;" $path -trust > /dev/null - then - if ! rg "[^a-z]$dep::" $path -trust > /dev/null - then - cnt=$((cnt + 1)) - echo "Not used: $d"; - fi - - fi - fi -done - -exit $cnt diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index e753979886..fc2eed8e80 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -24,27 +24,11 @@ jobs: unused-deps: runs-on: ubuntu-latest steps: - - name: Install deps - run: sudo snap install remarshal - - name: Install deps - run: sudo snap install --classic ripgrep - uses: actions/checkout@v3 - - name: Check for unused dependencies (tremor-runtime) - run: ./.github/checks/deps.sh . - - name: Check for unused dependencies (tremor-influx) - run: ./.github/checks/deps.sh tremor-influx - - name: Check for unused dependencies (tremor-pipeline) - run: ./.github/checks/deps.sh tremor-pipeline - - name: Check for unused dependencies (tremor-script) - run: ./.github/checks/deps.sh tremor-script - - name: Check for unused dependencies (tremor-cli) - run: ./.github/checks/deps.sh tremor-cli - - name: Check for unused dependencies (tremor-common) - run: ./.github/checks/deps.sh tremor-common - - name: Check for unused dependencies (tremor-value) - run: ./.github/checks/deps.sh tremor-value - - name: Check for unused dependencies (tremor-codec) - run: ./.github/checks/deps.sh tremor-codec + - name: Install deps + run: cargo install cargo-machete + - name: Check for unused dependencies + run: cargo machete --with-metadata format: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index b0fcbac60a..0f9360bcb7 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ tremor-script/current_counterexample.eqc .vscode/dryrun.log tremor-script-nif/libpath docs-test +*.mm_profdata +chrome_profiler.json diff --git a/Cargo.lock b/Cargo.lock index ab88957363..efefb9b535 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -908,9 +908,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.6" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e246206a63c9830e118d12c894f56a82033da1a2361f5544deeee3df85c99d9" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -934,16 +934,15 @@ dependencies = [ "sync_wrapper", "tokio", "tower", - "tower-http", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", "bytes", @@ -2141,16 +2140,15 @@ dependencies = [ [[package]] name = "enum-ordinalize" -version = "3.1.12" +version = "3.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a" +checksum = "1bf1fa3f06bbff1ea5b1a9c7b14aa992a39657db60a2759457328d7e058f49ee" dependencies = [ "num-bigint 0.4.4", "num-traits", "proc-macro2", "quote", - "rustc_version 0.4.0", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] @@ -2464,15 +2462,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - [[package]] name = "getrandom" version = "0.1.16" @@ -2812,12 +2801,6 @@ dependencies = [ "rustls 0.18.1", ] -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - [[package]] name = "http-types" version = "2.12.0" @@ -3071,7 +3054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.3", - "rustix 0.38.20", + "rustix 0.38.21", "windows-sys 0.48.0", ] @@ -3472,9 +3455,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "matrixmultiply" @@ -4838,9 +4821,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.20" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ "bitflags 2.4.1", "errno", @@ -5113,9 +5096,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa 1.0.9", "ryu", @@ -5124,10 +5107,11 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" dependencies = [ + "itoa 1.0.9", "serde", ] @@ -5331,9 +5315,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -5387,9 +5371,9 @@ dependencies = [ [[package]] name = "simd-json" -version = "0.13.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d9ce3bb26ed2e937a1f5dd1fd8747f5e1c8d373a4df4b0218980e316280ef68" +checksum = "e5a3720326b20bf5b95b72dbbd133caae7e0dcf71eae8f6e6656e71a7e5c9aaa" dependencies = [ "ahash", "getrandom 0.2.10", @@ -5841,14 +5825,14 @@ checksum = "af547b166dd1ea4b472165569fc456cfb6818116f854690b0ff205e636523dab" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall 0.3.5", - "rustix 0.38.20", + "redox_syscall 0.4.1", + "rustix 0.38.21", "windows-sys 0.48.0", ] @@ -6310,25 +6294,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags 1.3.2", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.2" @@ -6399,9 +6364,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "sharded-slab", "thread_local", @@ -6415,19 +6380,15 @@ dependencies = [ "anyhow", "clap", "clap_complete", - "criterion", "difference", "env_logger", "error-chain", - "float-cmp", "futures", "globwalk", "halfbrown", "http-types", - "lalrpop", "log", "log4rs", - "matches", "port_scanner", "pretty_assertions", "serde", @@ -6578,7 +6539,6 @@ name = "tremor-pipeline" version = "0.13.0-rc.16" dependencies = [ "beef", - "criterion", "either", "error-chain", "halfbrown", @@ -6594,7 +6554,6 @@ dependencies = [ "simd-json", "simd-json-derive", "sled", - "tempfile", "tokio", "tremor-common", "tremor-config", @@ -6603,7 +6562,6 @@ dependencies = [ "url", "value-trait", "window", - "xz2", ] [[package]] @@ -6618,7 +6576,6 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-s3", - "aws-smithy-http", "aws-types", "axum", "base64 0.21.5", @@ -6634,14 +6591,11 @@ dependencies = [ "either", "elasticsearch", "env_logger", - "error-chain", "event-listener 3.0.1", "file-mode", "futures", - "glob", "googapis", "gouth", - "grok", "halfbrown", "hashbrown 0.14.2", "hdrhistogram", @@ -6658,10 +6612,8 @@ dependencies = [ "log", "matches", "mime", - "num_cpus", "openraft", "pin-project-lite", - "port_scanner", "pretty_assertions", "proptest", "prost", @@ -6683,8 +6635,6 @@ dependencies = [ "serenity", "serial_test", "sha2 0.10.8", - "signal-hook", - "signal-hook-tokio", "simd-json", "simd-json-derive", "sled", @@ -6694,6 +6644,7 @@ dependencies = [ "tempfile", "test-case", "testcontainers", + "thiserror", "tokio", "tokio-rustls 0.24.1", "tokio-stream", @@ -6772,8 +6723,6 @@ version = "0.13.0-rc.16" dependencies = [ "base64 0.21.5", "beef", - "float-cmp", - "getopts", "halfbrown", "proptest", "serde", @@ -7328,7 +7277,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.20", + "rustix 0.38.21", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b2ad823d46..2bfd8315b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ lto = "thin" opt-level = 3 [dependencies] +thiserror = "1" tokio = { version = "1.32", features = ["full"] } tokio-stream = "0.1" anyhow = "1" @@ -64,11 +65,9 @@ jumphash = "0.1" clickhouse-rs = { git = "https://github.com/suharev7/clickhouse-rs", rev = "73d39ba" } dashmap = "5.5" either = { version = "1.9", features = ["serde"] } -error-chain = "0.12" file-mode = "0.1" futures = "0.3.29" event-listener = "3" -glob = "0.3" halfbrown = "0.2" hashbrown = { version = "0.14", features = ["serde"] } hex = "0.4" @@ -138,9 +137,6 @@ rdkafka-sys = { version = "4.6", features = [ # crononome cron = "0.12" -# logstash grok patterns -grok = "2" - # discord serenity = { version = "0.11", default-features = false, features = [ "client", @@ -168,7 +164,6 @@ tremor-otelapis = { version = "=0.2.4" } aws-sdk-s3 = "0.35" aws-types = "0.57" aws-config = "0.57" -aws-smithy-http = "0.57" # gcp googapis = { version = "0.6", default-features = false, features = [ @@ -204,7 +199,6 @@ sha2 = "0.10" rmp-serde = "1" [dev-dependencies] -port_scanner = "0.1" serial_test = { version = "2.0", features = ["logging"] } env_logger = "0.10" matches = "0.1" @@ -213,12 +207,9 @@ proptest = "1.1" regex = "1" # We downgraded to 0.6 because: # in the face of high concurrency serial_test 0.7.0 panics after 60 seconds -signal-hook = "0.3" -signal-hook-tokio = "0.3" tempfile = { version = "3.8" } test-case = "3.1" testcontainers = { version = "0.14", features = ["watchdog"] } -num_cpus = "1" bytes = "1" [features] diff --git a/src/channel.rs b/src/channel.rs index d6ee327407..8442dd33a5 100644 --- a/src/channel.rs +++ b/src/channel.rs @@ -18,6 +18,25 @@ pub(crate) type UnboundedSender = tokio::sync::mpsc::UnboundedSender; pub(crate) type UnboundedReceiver = tokio::sync::mpsc::UnboundedReceiver; pub(crate) type OneShotSender = tokio::sync::oneshot::Sender; // pub(crate) type OneShotReceiver = tokio::sync::oneshot::Receiver; -pub(crate) use tokio::sync::mpsc::error::{SendError, TryRecvError}; +pub(crate) use tokio::sync::mpsc::error::SendError; pub(crate) use tokio::sync::mpsc::{channel as bounded, unbounded_channel as unbounded}; pub(crate) use tokio::sync::oneshot::channel as oneshot; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum ChannelError { + #[error("Could not send")] + Send, + #[error("Could not recv")] + Recv, + #[error("The channel is empty")] + Empty, +} + +#[allow(clippy::needless_pass_by_value)] +pub(crate) fn send_e(_: crate::channel::SendError) -> anyhow::Error { + ChannelError::Send.into() +} + +pub(crate) fn empty_e() -> ChannelError { + ChannelError::Empty +} diff --git a/src/config.rs b/src/config.rs index 56c13bc07d..9a6e35bd26 100644 --- a/src/config.rs +++ b/src/config.rs @@ -113,7 +113,7 @@ impl Connector { defn: &ast::ConnectorDefinition<'static>, ) -> crate::Result { let aggr_reg = tremor_script::registry::aggr(); - let reg = &*FN_REGISTRY.read()?; + let reg = &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?; let mut helper = Helper::new(reg, &aggr_reg); let params = defn.params.clone(); @@ -136,8 +136,8 @@ impl Connector { connector_alias: &alias::Connector, ) -> Result<()> { if v.get(k).is_some() && v.get(k).map(Value::value_type) != Some(t) { - return Err(ErrorKind::InvalidConnectorDefinition( - connector_alias.to_string(), + return Err(ConnectorError::InvalidDefinition( + connector_alias.clone(), format!( "Expected type {t:?} for key {k} but got {:?}", v.get(k).map_or(ValueType::Null, Value::value_type) @@ -316,6 +316,6 @@ mod tests { let id = alias::Connector::new("my_id"); let res = Connector::from_config(&id, "fancy_schmancy".into(), &config); assert!(res.is_err()); - assert_eq!(String::from("Invalid Definition for connector \"my_id\": Expected type I64 for key metrics_interval_s but got String"), res.err().map(|e| e.to_string()).unwrap_or_default()); + assert_eq!(String::from("[my_id] Invalid definition: Expected type I64 for key metrics_interval_s but got String"), res.err().map(|e| e.to_string()).unwrap_or_default()); } } diff --git a/src/connectors.rs b/src/connectors.rs index b2340a95f2..8967dc4750 100644 --- a/src/connectors.rs +++ b/src/connectors.rs @@ -41,8 +41,8 @@ use self::{ }; pub(crate) use crate::config::Connector as ConnectorConfig; use crate::{ - channel::{bounded, Sender}, - errors::{connector_send_err, Error, Kind as ErrorKind, Result}, + channel::{bounded, send_e, Sender}, + errors::Result, instance::State, log_error, pipeline, qsize, raft, system::{flow::AppContext, KillSwitch, Runtime}, @@ -63,6 +63,50 @@ use tremor_value::Value; /// Accept timeout pub(crate) const ACCEPT_TIMEOUT: Duration = Duration::from_millis(100); +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("{0} Invalid port '{1}'")] + InvalidPort(alias::Connector, Port<'static>), + #[error("{0} is a structured connector and can't be configured with a codec")] + UnsupportedCodec(alias::Connector), + #[error("{0} is missing a codec")] + MissingCodec(alias::Connector), + #[error("{0} Connection failed")] + ConnectionFailed(alias::Connector), + #[error("{0} Invalid configuration: {1}")] + InvalidConfiguration(alias::Connector, String), + #[error("[{0}] Invalid definition: {1}")] + InvalidDefinition(alias::Connector, String), + #[error("Conector was already created")] + AlreadyCreated(alias::Connector), + #[error("{0} Missing configuration")] + MissingConfiguration(alias::Connector), + #[error("{0} build_cfg is unimplemented")] + BuildCfg(alias::Connector), +} + +impl Error { + /// the alias of the connector + #[must_use] + pub fn alias(&self) -> &alias::Connector { + match self { + Self::InvalidPort(alias, _) + | Self::UnsupportedCodec(alias) + | Self::MissingCodec(alias) + | Self::ConnectionFailed(alias) + | Self::InvalidConfiguration(alias, _) + | Self::InvalidDefinition(alias, _) + | Self::AlreadyCreated(alias) + | Self::MissingConfiguration(alias) + | Self::BuildCfg(alias) => alias, + } + } +} + +pub(crate) fn error_connector_def(c: &alias::Connector, e: &E) -> Error { + Error::InvalidDefinition(c.clone(), e.to_string()) +} + /// connector address #[derive(Clone, Debug)] pub struct Addr { @@ -85,7 +129,7 @@ impl Addr { /// # Errors /// * If sending failed pub(crate) async fn send(&self, msg: Msg) -> Result<()> { - self.sender.send(msg).await.map_err(connector_send_err) + self.sender.send(msg).await.map_err(send_e) } /// send a message to the sink part of the connector. @@ -95,7 +139,7 @@ impl Addr { /// * if sending failed pub(crate) async fn send_sink(&self, msg: SinkMsg) -> Result<()> { if let Some(sink) = self.sink.as_ref() { - sink.addr.send(msg).await.map_err(connector_send_err)?; + sink.addr.send(msg).await.map_err(send_e)?; } Ok(()) } @@ -107,7 +151,7 @@ impl Addr { /// * if sending failed pub(crate) fn send_source(&self, msg: SourceMsg) -> Result<()> { if let Some(source) = self.source.as_ref() { - source.addr.send(msg).map_err(connector_send_err)?; + source.addr.send(msg).map_err(send_e)?; } Ok(()) } @@ -124,21 +168,21 @@ impl Addr { /// /// # Errors /// * if sending failed - pub(crate) async fn stop(&self, sender: Sender>) -> Result<()> { + pub(crate) async fn stop(&self, sender: Sender) -> Result<()> { self.send(Msg::Stop(sender)).await } /// starts the connector /// /// # Errors /// * if sending failed - pub(crate) async fn start(&self, sender: Sender>) -> Result<()> { + pub(crate) async fn start(&self, sender: Sender) -> Result<()> { self.send(Msg::Start(sender)).await } /// drains the connector /// /// # Errors /// * if sending failed - pub(crate) async fn drain(&self, sender: Sender>) -> Result<()> { + pub(crate) async fn drain(&self, sender: Sender) -> Result<()> { self.send(Msg::Drain(sender)).await } /// pauses the connector @@ -195,7 +239,7 @@ pub(crate) enum Msg { Reconnect, // TODO: fill as needed /// start the connector - Start(Sender>), + Start(Sender), /// pause the connector /// /// source part is not polling for new data @@ -209,39 +253,28 @@ pub(crate) enum Msg { /// - stop reading events from external connections /// - decline events received via the sink part /// - wait for drainage to be finished - Drain(Sender>), + Drain(Sender), /// notify this connector that its source part has been drained SourceDrained, /// notify this connector that its sink part has been drained SinkDrained, /// stop the connector - Stop(Sender>), + Stop(Sender), /// request a status report Report(tokio::sync::oneshot::Sender), } -#[derive(Debug)] -/// result of an async operation of the connector. -/// bears a `url` to identify the connector who finished the operation -pub(crate) struct ConnectorResult { - /// the connector alias - pub(crate) alias: alias::Connector, - /// the actual result - pub(crate) res: Result, +pub(crate) trait AliasableConnectorResult { + fn alias(&self) -> &alias::Connector; } -impl ConnectorResult<()> { - fn ok(ctx: &ConnectorContext) -> Self { - Self { - alias: ctx.alias.clone(), - res: Ok(()), - } - } +pub(crate) type ConnectorResult = std::result::Result; - fn err(ctx: &ConnectorContext, err_msg: &'static str) -> Self { - Self { - alias: ctx.alias.clone(), - res: Err(Error::from(err_msg)), +impl AliasableConnectorResult for ConnectorResult { + fn alias(&self) -> &alias::Connector { + match self { + Ok(alias) => alias, + Err(e) => e.alias(), } } } @@ -261,7 +294,7 @@ pub(crate) trait Context: Display + Clone { fn connector_type(&self) -> &ConnectorType; /// gets the API sender - fn raft(&self) -> &raft::Cluster; + fn raft(&self) -> &raft::ClusterInterface; /// the application context fn app_ctx(&self) -> &AppContext; @@ -286,7 +319,7 @@ pub(crate) trait Context: Display + Clone { msg: &M, ) -> std::result::Result where - E: std::error::Error, + E: Display, M: Display + ?Sized, { if let Err(e) = &expr { @@ -322,7 +355,7 @@ pub(crate) trait Context: Display + Clone { #[derive(Clone)] pub(crate) struct ConnectorContext { /// alias of the connector instance - pub(crate) alias: alias::Connector, + alias: alias::Connector, /// type of the connector connector_type: ConnectorType, /// The Quiescence Beacon @@ -356,7 +389,7 @@ impl Context for ConnectorContext { &self.notifier } - fn raft(&self) -> &raft::Cluster { + fn raft(&self) -> &raft::ClusterInterface { &self.app_ctx.raft } @@ -482,17 +515,24 @@ async fn connector_task( SourceReporter::new(app_ctx.clone(), alias.clone(), config.metrics_interval_s); let codec_requirement = connector.codec_requirements(); + + let ctx = ConnectorContext { + alias: alias.clone(), + connector_type: config.connector_type.clone(), + quiescence_beacon: quiescence_beacon.clone(), + notifier: notifier.clone(), + app_ctx: app_ctx.clone(), + }; + if connector.codec_requirements() == CodecReq::Structured && (config.codec.is_some()) { - return Err(format!( - "[Connector::{alias}] is a structured connector and can't be configured with a codec", - ) - .into()); + return Err(Error::UnsupportedCodec(ctx.alias().clone()).into()); } let source_builder = source::builder( SourceUId::from(uid), &config, codec_requirement, source_metrics_reporter, + &alias, )?; let source_ctx = SourceContext::new( uid.into(), @@ -529,21 +569,12 @@ async fn connector_task( sink: sink_addr, }; - let mut reconnect = ReconnectRuntime::new(&connector_addr, notifier.clone(), &config.reconnect); - let notifier = reconnect.notifier(); - - let ctx = ConnectorContext { - alias: alias.clone(), - connector_type: config.connector_type.clone(), - quiescence_beacon: quiescence_beacon.clone(), - notifier, - app_ctx, - }; + let mut reconnect = ReconnectRuntime::new(&connector_addr, &config.reconnect); let send_addr = connector_addr.clone(); let mut connector_state = State::Initializing; let mut drainage = None; - let mut start_sender: Option>> = None; + let mut start_sender: Option> = None; // TODO: add connector metrics reporter (e.g. for reconnect attempts, cb's received, uptime, etc.) task::spawn(async move { @@ -605,19 +636,11 @@ async fn connector_task( .await .map_err(Into::into) } else { - Err(ErrorKind::InvalidConnect( - connector_addr.alias.to_string(), - port.clone(), - ) - .into()) + Err(Error::InvalidPort(ctx.alias().clone(), port.clone()).into()) } } else { error!("{ctx} Tried to connect to unsupported port: \"{port}\""); - Err(ErrorKind::InvalidConnect( - connector_addr.alias.to_string(), - port.clone(), - ) - .into()) + Err(Error::InvalidPort(ctx.alias().clone(), port.clone()).into()) }; // send back the connect result if let Err(e) = result_tx.send(res).await { @@ -649,19 +672,15 @@ async fn connector_task( let res = source.addr.send(m); log_error!(res, "{ctx} Error sending to source: {e}"); } else { - let e = Err(ErrorKind::InvalidConnect( - connector_addr.alias.to_string(), - port.clone(), - ) - .into()); + let e = + Err(Error::InvalidPort(ctx.alias().clone(), port.clone()).into()); let res = result_tx.send(e).await; log_error!(res, "{ctx} Error sending connect result: {e}"); } } else { error!("{ctx} Tried to connect to unsupported port: \"{port}\""); // send back the connect result - let addr = connector_addr.alias.to_string(); - let e = Err(ErrorKind::InvalidConnect(addr, port.clone()).into()); + let e = Err(Error::InvalidPort(ctx.alias().clone(), port.clone()).into()); let res = result_tx.send(e).await; log_error!(res, "{ctx} Error sending connect result: {e}"); }; @@ -698,7 +717,7 @@ async fn connector_task( connector_addr.send_source(SourceMsg::ConnectionEstablished)?; if let Some(start_sender) = start_sender.take() { ctx.swallow_err( - start_sender.send(ConnectorResult::ok(&ctx)).await, + start_sender.send(Ok(ctx.alias().clone())).await, "Error sending start response.", ); } @@ -719,7 +738,7 @@ async fn connector_task( if let Some(start_sender) = start_sender.take() { ctx.swallow_err( start_sender - .send(ConnectorResult::err(&ctx, "Connect failed.")) + .send(Err(Error::ConnectionFailed(ctx.alias().clone()))) .await, "Error sending start response", ); @@ -753,7 +772,7 @@ async fn connector_task( { // sending an answer if we are connected ctx.swallow_err( - sender.send(ConnectorResult::ok(&ctx)).await, + sender.send(Ok(ctx.alias().clone())).await, "Error sending Start result", ); } @@ -895,7 +914,7 @@ async fn connector_task( expect -= 1; } log_error!( - sender.send(ConnectorResult::ok(&ctx)).await, + sender.send(Ok(ctx.alias().clone())).await, "{ctx} Error sending Stop result: {e}" ); @@ -906,7 +925,7 @@ async fn connector_task( } // while info!("{ctx} Connector Stopped. Reason: {connector_state}"); // TODO: inform registry that this instance is gone now - Result::Ok(()) + Result::<(), anyhow::Error>::Ok(()) }); Ok(send_addr) } @@ -919,14 +938,14 @@ enum DrainState { } struct Drainage { - tx: Sender>, + tx: Sender, alias: alias::Connector, source_drained: DrainState, sink_drained: DrainState, } impl Drainage { - fn new(addr: &Addr, tx: Sender>) -> Self { + fn new(addr: &Addr, tx: Sender) -> Self { Self { tx, alias: addr.alias.clone(), @@ -957,12 +976,7 @@ impl Drainage { } async fn send_all_drained(&self) -> Result<()> { - self.tx - .send(ConnectorResult { - alias: self.alias.clone(), - res: Ok(()), - }) - .await?; + self.tx.send(Ok(self.alias.clone())).await?; Ok(()) } } @@ -1169,7 +1183,7 @@ pub(crate) trait ConnectorBuilder: Sync + Send + std::fmt::Debug { let cc = config .config .as_ref() - .ok_or_else(|| ErrorKind::MissingConfiguration(alias.to_string()))?; + .ok_or_else(|| Error::MissingConfiguration(alias.clone()))?; self.build_cfg(alias, config, cc).await } @@ -1179,11 +1193,11 @@ pub(crate) trait ConnectorBuilder: Sync + Send + std::fmt::Debug { /// * If the config is invalid for the connector async fn build_cfg( &self, - _alias: &alias::Connector, + alias: &alias::Connector, _config: &ConnectorConfig, _connector_config: &Value, ) -> Result> { - Err("build_cfg is unimplemented".into()) + Err(Error::BuildCfg(alias.clone()).into()) } } @@ -1328,7 +1342,7 @@ pub(crate) mod unit_tests { fn connector_type(&self) -> &ConnectorType { &self.t } - fn raft(&self) -> &raft::Cluster { + fn raft(&self) -> &raft::ClusterInterface { &self.app_ctx.raft } fn app_ctx(&self) -> &AppContext { @@ -1338,6 +1352,11 @@ pub(crate) mod unit_tests { #[tokio::test(flavor = "multi_thread")] async fn spawn_task_error() -> Result<()> { + #[derive(Debug, thiserror::Error)] + enum Error { + #[error("snot")] + Snot, + } let (tx, mut rx) = bounded(1); let ctx = FakeContext::new(tx); // thanks coverage @@ -1346,7 +1365,7 @@ pub(crate) mod unit_tests { ctx.connector_type(); ctx.alias(); // end - spawn_task(ctx.clone(), async move { Err("snot".into()) }).await?; + spawn_task(ctx.clone(), async move { Err(Error::Snot.into()) }).await?; assert!(matches!(rx.recv().await, Some(Msg::ConnectionLost))); spawn_task(ctx.clone(), async move { Ok(()) }).await?; @@ -1354,7 +1373,7 @@ pub(crate) mod unit_tests { rx.close(); // this one is just here for coverage for when the call to notify is failing - spawn_task(ctx, async move { Err("snot 2".into()) }).await?; + spawn_task(ctx, async move { Err(Error::Snot.into()) }).await?; Ok(()) } diff --git a/src/connectors/google.rs b/src/connectors/google.rs index 36dd48826f..d066d29b89 100644 --- a/src/connectors/google.rs +++ b/src/connectors/google.rs @@ -283,7 +283,7 @@ PX8efvDMhv16QqDFF0k80d0= let addr = ("127.0.0.1", port) .to_socket_addrs()? .next() - .ok_or("no address")?; + .expect("no address"); let server_handle = tokio::task::spawn(async move { let listener = hyper::Server::bind(&addr).serve(service_fn); listener.await?; diff --git a/src/connectors/impls/bench.rs b/src/connectors/impls/bench.rs index 7a1c53790e..120b25b5c9 100644 --- a/src/connectors/impls/bench.rs +++ b/src/connectors/impls/bench.rs @@ -399,7 +399,7 @@ struct Blaster { #[async_trait::async_trait] impl Source for Blaster { - async fn on_start(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_start(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { self.stop_at = self .stop_after .seconds @@ -408,7 +408,11 @@ impl Source for Blaster { } #[allow(clippy::cast_possible_truncation)] - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { if self.finished { return Ok(SourceReply::Finished); } @@ -511,7 +515,7 @@ impl Sink for Blackhole { ctx: &SinkContext, event_serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { if !self.finished { let now_ns = nanotime(); @@ -551,7 +555,7 @@ impl Sink for Blackhole { _signal: Event, ctx: &SinkContext, _serializer: &mut EventSerializer, - ) -> Result { + ) -> anyhow::Result { let now_ns = nanotime(); if self.stop_at.iter().any(|stop_at| now_ns >= *stop_at) { self.finish(ctx)?; diff --git a/src/connectors/impls/cb.rs b/src/connectors/impls/cb.rs index eb0f769865..1b9aba8271 100644 --- a/src/connectors/impls/cb.rs +++ b/src/connectors/impls/cb.rs @@ -188,8 +188,8 @@ impl Connector for Cb { builder: SourceManagerBuilder, ) -> Result> { if self.config.paths.is_empty() { - return Err(ErrorKind::InvalidConfiguration( - ctx.alias().to_string(), + return Err(ConnectorError::InvalidConfiguration( + ctx.alias().clone(), "\"paths\" config missing".to_string(), ) .into()); @@ -219,7 +219,7 @@ impl Sink for CbSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { for (value, meta) in event.value_meta_iter() { if let Some(cb) = ctx.extract_meta(meta).or_else(|| ctx.extract_meta(value)) { let cb_cmds = if let Some(array) = cb.as_array() { @@ -312,6 +312,12 @@ struct CbSource { kill_switch: KillSwitch, } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("No file to write")] + NoFile, +} + impl CbSource { fn did_receive_all(&self) -> bool { let all_received = if self.config.expect_batched { @@ -357,7 +363,11 @@ impl CbSource { #[async_trait::async_trait()] impl Source for CbSource { - async fn pull_data(&mut self, pull_id: &mut u64, ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result { if self.files.is_empty() { info!("{ctx} finished."); self.finished = true; @@ -379,7 +389,7 @@ impl Source for CbSource { } tokio::task::spawn(async move { kill_switch.stop(ShutdownMode::Graceful).await?; - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }); Ok(SourceReply::Finished) @@ -388,7 +398,7 @@ impl Source for CbSource { let file = self .files .get_mut(idx) // this is safe as we do the module above - .ok_or(ErrorKind::ClientNotAvailable("cb", "No file available"))?; + .ok_or(Error::NoFile)?; let res = if let Some(line) = file.next_line().await? { self.num_sent += 1; self.last_sent @@ -419,16 +429,21 @@ impl Source for CbSource { } } - async fn on_cb_trigger(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_trigger(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { self.received_cbs.trigger += 1; Ok(()) } - async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { self.received_cbs.restore += 1; Ok(()) } - async fn ack(&mut self, stream_id: u64, pull_id: u64, _ctx: &SourceContext) -> Result<()> { + async fn ack( + &mut self, + stream_id: u64, + pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { self.received_cbs .ack .entry(stream_id) @@ -437,7 +452,12 @@ impl Source for CbSource { Ok(()) } - async fn fail(&mut self, stream_id: u64, pull_id: u64, _ctx: &SourceContext) -> Result<()> { + async fn fail( + &mut self, + stream_id: u64, + pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { self.received_cbs .fail .entry(stream_id) diff --git a/src/connectors/impls/clickhouse.rs b/src/connectors/impls/clickhouse.rs index 9ae4818e7a..dc95d7f27b 100644 --- a/src/connectors/impls/clickhouse.rs +++ b/src/connectors/impls/clickhouse.rs @@ -292,6 +292,12 @@ use clickhouse_rs::{ Block, ClientHandle, Pool, }; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("No client available")] + NoClient, +} + #[derive(Default, Debug)] pub(crate) struct Builder {} @@ -413,7 +419,7 @@ pub(crate) struct ClickhouseSink { #[async_trait::async_trait] impl Sink for ClickhouseSink { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let pool = Pool::new(self.db_url.as_str()); let handle = match pool.get_handle().await { Ok(handle) => handle, @@ -423,7 +429,7 @@ impl Sink for ClickhouseSink { ctx.notifier().connection_lost().await?; Ok(false) } - _ => Err(Error::from(e)), + _ => Err(e.into()), } } }; @@ -440,11 +446,8 @@ impl Sink for ClickhouseSink { _ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { - let handle = self - .handle - .as_mut() - .ok_or_else(|| Error::from(ErrorKind::NoClickHouseClientAvailable))?; + ) -> anyhow::Result { + let handle = self.handle.as_mut().ok_or(Error::NoClient)?; let mut block = Block::with_capacity(event.len()); @@ -471,9 +474,7 @@ impl ClickhouseSink { ) -> Result> { let mut rslt = Vec::new(); - let object = input - .as_object() - .ok_or_else(|| Error::from(ErrorKind::ExpectedObjectEvent(input.value_type())))?; + let object = input.try_as_object()?; for (column_name, expected_type) in columns { // If the value is not present, then we can replace it by null. diff --git a/src/connectors/impls/clickhouse/conversion.rs b/src/connectors/impls/clickhouse/conversion.rs index 501fe0a573..6be0e66b3c 100644 --- a/src/connectors/impls/clickhouse/conversion.rs +++ b/src/connectors/impls/clickhouse/conversion.rs @@ -18,6 +18,8 @@ use std::{ sync::Arc, }; +use super::DummySqlType; +use crate::errors::Result; use chrono_tz::Tz; pub(super) use clickhouse_rs::types::Value as CValue; use either::Either; @@ -25,11 +27,22 @@ use tremor_value::Value as TValue; use uuid::Uuid; use value_trait::prelude::*; -use super::DummySqlType; -use crate::errors::{Error, ErrorKind, Result}; - const UTC: Tz = Tz::UTC; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Malformed IP address")] + MalformedIpAddr, + #[error("Malformed UUID")] + MalformedUuid, + #[error("Unexpected event format: column `{column}` is of type `{expected_type}`, but the event has type `{actual_type}`")] + UnexpectedEventFormat { + column: String, + expected_type: DummySqlType, + actual_type: ValueType, + }, +} + pub(super) fn convert_value( column_name: &str, value: &TValue, @@ -99,21 +112,21 @@ pub(super) fn convert_value( // // ... But it does work! |[a, b, c, d]| CValue::Ipv4([d, c, b, a]), - ErrorKind::MalformedIpAddr, + Error::MalformedIpAddr, ), DummySqlType::Ipv6 => convert_string_or_array( context, |ip: Ipv6Addr| ip.octets(), CValue::Ipv6, - ErrorKind::MalformedIpAddr, + Error::MalformedIpAddr, ), DummySqlType::Uuid => convert_string_or_array( context, Uuid::into_bytes, CValue::Uuid, - ErrorKind::MalformedUuid, + Error::MalformedUuid, ), DummySqlType::DateTime => get_and_wrap(context, ValueAsScalar::as_u32, |timestamp| { @@ -160,11 +173,12 @@ where Output: 'event, { f(context.value).ok_or_else(|| { - Error::from(ErrorKind::UnexpectedEventFormat( - context.column_name.to_string(), - context.expected_type.to_string(), - context.value.value_type(), - )) + Error::UnexpectedEventFormat { + column: context.column_name.to_string(), + expected_type: context.expected_type.clone(), + actual_type: context.value.value_type(), + } + .into() }) } @@ -185,7 +199,7 @@ fn convert_string_or_array( context: ConversionContext, extractor: Getter, variant: Variant, - error: ErrorKind, + error: Error, ) -> Result where Output: FromStr, @@ -215,18 +229,19 @@ where if let Some(octets) = context.value.as_array() { coerce_octet_sequence(octets.as_slice()) .map(variant) - .map_err(|()| Error::from(error)) + .map_err(|()| error.into()) } else if let Some(string) = context.value.as_str() { Output::from_str(string) .map(extractor) .map(variant) - .map_err(|_| Error::from(error)) + .map_err(|_| error.into()) } else { - Err(Error::from(ErrorKind::UnexpectedEventFormat( - context.column_name.to_string(), - context.expected_type.to_string(), - context.value.value_type(), - ))) + Err(Error::UnexpectedEventFormat { + column: context.column_name.to_string(), + expected_type: context.expected_type.clone(), + actual_type: context.value.value_type(), + } + .into()) } } diff --git a/src/connectors/impls/cluster_kv.rs b/src/connectors/impls/cluster_kv.rs index 744b3d6ad5..50b24682a2 100644 --- a/src/connectors/impls/cluster_kv.rs +++ b/src/connectors/impls/cluster_kv.rs @@ -15,7 +15,6 @@ // #![cfg_attr(coverage, no_coverage)] use crate::{ channel::{bounded, Receiver, Sender}, - errors::already_created_error, raft, }; use crate::{connectors::prelude::*, system::flow::AppContext}; @@ -28,6 +27,14 @@ use tremor_codec::{ }; use tremor_common::alias::Generic; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Missing `$kv` field for commands")] + MissingMeta, + #[error("Invalid command: {0}")] + InvalidCommand(String), +} + #[derive(Debug)] enum Command { /// Format: @@ -93,7 +100,7 @@ impl<'v> TryFrom<&'v Value<'v>> for Command { type Error = crate::Error; fn try_from(v: &'v Value<'v>) -> Result { - let v = v.get("kv").ok_or("Missing `$kv` field for commands")?; + let v = v.get("kv").ok_or(Error::MissingMeta)?; if let Some(key) = v.get_str("get").map(ToString::to_string) { Ok(Command::Get { key, @@ -116,7 +123,7 @@ impl<'v> TryFrom<&'v Value<'v>> for Command { // end: v.get_bytes("end").map(<[u8]>::to_vec), // }) } else { - Err(format!("Invalid KV command: {v}").into()) + Err(Error::InvalidCommand(v.to_string()).into()) } } } @@ -210,7 +217,9 @@ impl Connector for Kv { ) -> Result> { let source = ChannelSource::from_channel( self.tx.clone(), - self.rx.take().ok_or_else(already_created_error)?, + self.rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, self.source_is_connected.clone(), ); Ok(Some(builder.spawn(source, ctx))) @@ -248,7 +257,7 @@ impl Connector for Kv { struct KvSink { alias: alias::Connector, app_ctx: AppContext, - raft: raft::Cluster, + raft: raft::ClusterInterface, tx: Sender, codec: Json, origin_uri: EventOriginUri, @@ -349,7 +358,7 @@ impl Sink for KvSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { let ingest_ns = tremor_common::time::nanotime(); let send_replies = self.source_is_connected.load(Ordering::Acquire); diff --git a/src/connectors/impls/crononome.rs b/src/connectors/impls/crononome.rs index e468bd35e1..dbbe67f1b8 100644 --- a/src/connectors/impls/crononome.rs +++ b/src/connectors/impls/crononome.rs @@ -161,7 +161,7 @@ impl ConnectorBuilder for Builder { .map(CronEntryInt::try_from) .collect::>>()? } else { - return Err(err_connector_def(id, "missing `entries` array")); + return Err(error_connector_def(id, "missing `entries` array").into()); }; Ok(Box::new(Crononome { entries })) @@ -210,14 +210,18 @@ impl CrononomeSource { } #[async_trait::async_trait()] impl Source for CrononomeSource { - async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { self.cq = ChronomicQueue::default(); // create a new queue to avoid duplication in case of reconnect for entry in &self.entries { self.cq.enqueue(entry); } Ok(true) } - async fn pull_data(&mut self, pull_id: &mut u64, ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result { let mut origin_uri = self.origin_uri.clone(); if let Some(trigger) = self.cq.wait_for_next().await { origin_uri.path.push(trigger.0.clone()); diff --git a/src/connectors/impls/discord.rs b/src/connectors/impls/discord.rs index 46c6126852..fd0c771e63 100644 --- a/src/connectors/impls/discord.rs +++ b/src/connectors/impls/discord.rs @@ -42,6 +42,7 @@ mod utils; use crate::channel::{bounded, Receiver, Sender}; use crate::connectors::prelude::*; +use crate::connectors::Context as ContextTrait; use handler::Handler; use serenity::prelude::*; use tokio::task::JoinHandle; @@ -118,8 +119,13 @@ impl Connector for Discord { builder: SourceManagerBuilder, ) -> Result> { // the source is listening for events formatted as `Value` from the discord client + let rx = self + .message_channel + .1 + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?; let source = DiscordSource { - rx: self.message_channel.1.take().ok_or("No message channel")?, + rx, origin_uri: self.origin_uri.clone(), }; Ok(Some(builder.spawn(source, ctx))) @@ -156,9 +162,8 @@ impl Connector for Discord { rx: RwLock::new(Some(rx)), }); - let mut client = client - .await - .map_err(|e| Error::from(format!("Err discord creating client: {e}")))?; + let mut client = client.await?; + // set up new client task self.client_task = Some(spawn_task( ctx.clone(), @@ -182,16 +187,16 @@ impl Sink for DiscordSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { for v in event.value_iter() { - if self.tx.send(v.clone_static()).await.is_err() { + if let Err(e) = self.tx.send(v.clone_static()).await { error!( "{} Discord Client unreachable. Initiating Reconnect...", &ctx ); ctx.notifier().connection_lost().await?; // return here to avoid notifying the notifier multiple times - return Err("Discord unreachable.".into()); + return Err(e.into()); } } Ok(SinkReply::NONE) @@ -209,8 +214,13 @@ struct DiscordSource { #[async_trait::async_trait()] impl Source for DiscordSource { #[allow(clippy::option_if_let_else)] - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { - self.rx + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + Ok(self + .rx .recv() .await .map(|data| SourceReply::Structured { @@ -219,7 +229,7 @@ impl Source for DiscordSource { stream: DEFAULT_STREAM_ID, port: None, }) - .ok_or_else(|| Error::from("channel closed")) + .ok_or(ChannelError::Recv)?) } fn is_transactional(&self) -> bool { diff --git a/src/connectors/impls/dns/client.rs b/src/connectors/impls/dns/client.rs index 59b697dbcc..20016a6916 100644 --- a/src/connectors/impls/dns/client.rs +++ b/src/connectors/impls/dns/client.rs @@ -22,6 +22,16 @@ use trust_dns_resolver::{ TokioAsyncResolver, }; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Metadata `$dns.lookup` missing or invalid")] + Metadata, + #[error("No DNS resolver available")] + NoResolvers, + #[error("Invalid or unsupported record type: {0}")] + InvalidRecordType(String), +} + #[derive(Debug, Default)] pub(crate) struct Builder {} @@ -67,7 +77,7 @@ impl Connector for Client { self.tx.clone(), self.rx .take() - .ok_or("Tried to create a source from a connector that has already been used")?, + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, self.source_is_connected.clone(), ); Ok(Some(builder.spawn(source, ctx))) @@ -114,10 +124,7 @@ impl DnsSink { correlation: Option<&Value<'event>>, ) -> Result { // check if we have a resolver - let resolver = self - .resolver - .as_ref() - .ok_or_else(|| Error::from("No DNS resolver available"))?; + let resolver = self.resolver.as_ref().ok_or(Error::NoResolvers)?; let data = if let Some(record_type) = record_type { // type lookup @@ -135,9 +142,10 @@ impl DnsSink { Ok(e) } } + #[async_trait::async_trait] impl Sink for DnsSink { - async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { self.resolver = Some(TokioAsyncResolver::tokio_from_system_conf()?); Ok(true) } @@ -148,16 +156,16 @@ impl Sink for DnsSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { let source_is_connected = self.source_is_connected.load(Ordering::Acquire); for (_, m) in event.value_meta_iter() { // verify incoming request and extract DNS query params let dns_meta = m.get("dns"); - let lookup = dns_meta.get("lookup").ok_or("Invalid DNS request")?; + let lookup = dns_meta.get("lookup").ok_or(Error::Metadata)?; let name = lookup .as_str() .or_else(|| lookup.get_str("name")) - .ok_or("Invalid DNS request: Metadata `$dns.lookup` missing")?; + .ok_or(Error::Metadata)?; let record_type = lookup.get_str("type").map(str_to_record_type).transpose()?; // issue DNS query let (port, payload) = match self.query(name, record_type, m.get("correlation")).await { @@ -222,7 +230,7 @@ fn str_to_record_type(s: &str) -> Result { "TLSA" => Ok(RecordType::TLSA), "TXT" => Ok(RecordType::TXT), "ZERO" => Ok(RecordType::ZERO), - other => Err(format!("Invalid or unsupported record type: {other}").into()), + other => Err(Error::InvalidRecordType(other.to_string()).into()), } } @@ -267,33 +275,35 @@ fn lookup_to_value(l: &Lookup) -> Value<'static> { #[cfg(test)] mod test { + use matches::assert_matches; + use super::*; #[test] fn test_str_to_record_type() { - assert_eq!(str_to_record_type("A"), Ok(RecordType::A)); - assert_eq!(str_to_record_type("AAAA"), Ok(RecordType::AAAA)); - assert_eq!(str_to_record_type("ANAME"), Ok(RecordType::ANAME)); - assert_eq!(str_to_record_type("ANY"), Ok(RecordType::ANY)); - assert_eq!(str_to_record_type("AXFR"), Ok(RecordType::AXFR)); - assert_eq!(str_to_record_type("CAA"), Ok(RecordType::CAA)); - assert_eq!(str_to_record_type("CNAME"), Ok(RecordType::CNAME)); - assert_eq!(str_to_record_type("HINFO"), Ok(RecordType::HINFO)); - assert_eq!(str_to_record_type("HTTPS"), Ok(RecordType::HTTPS)); - assert_eq!(str_to_record_type("IXFR"), Ok(RecordType::IXFR)); - assert_eq!(str_to_record_type("MX"), Ok(RecordType::MX)); - assert_eq!(str_to_record_type("NAPTR"), Ok(RecordType::NAPTR)); - assert_eq!(str_to_record_type("NS"), Ok(RecordType::NS)); - assert_eq!(str_to_record_type("NULL"), Ok(RecordType::NULL)); - assert_eq!(str_to_record_type("OPENPGPKEY"), Ok(RecordType::OPENPGPKEY)); - assert_eq!(str_to_record_type("OPT"), Ok(RecordType::OPT)); - assert_eq!(str_to_record_type("PTR"), Ok(RecordType::PTR)); - assert_eq!(str_to_record_type("SOA"), Ok(RecordType::SOA)); - assert_eq!(str_to_record_type("SRV"), Ok(RecordType::SRV)); - assert_eq!(str_to_record_type("SSHFP"), Ok(RecordType::SSHFP)); - assert_eq!(str_to_record_type("SVCB"), Ok(RecordType::SVCB)); - assert_eq!(str_to_record_type("TLSA"), Ok(RecordType::TLSA)); - assert_eq!(str_to_record_type("TXT"), Ok(RecordType::TXT)); - assert_eq!(str_to_record_type("ZERO"), Ok(RecordType::ZERO)); - assert!(str_to_record_type("NOT A DNS ENTRIE").is_err()); + assert_matches!(str_to_record_type("A"), Ok(RecordType::A)); + assert_matches!(str_to_record_type("AAAA"), Ok(RecordType::AAAA)); + assert_matches!(str_to_record_type("ANAME"), Ok(RecordType::ANAME)); + assert_matches!(str_to_record_type("ANY"), Ok(RecordType::ANY)); + assert_matches!(str_to_record_type("AXFR"), Ok(RecordType::AXFR)); + assert_matches!(str_to_record_type("CAA"), Ok(RecordType::CAA)); + assert_matches!(str_to_record_type("CNAME"), Ok(RecordType::CNAME)); + assert_matches!(str_to_record_type("HINFO"), Ok(RecordType::HINFO)); + assert_matches!(str_to_record_type("HTTPS"), Ok(RecordType::HTTPS)); + assert_matches!(str_to_record_type("IXFR"), Ok(RecordType::IXFR)); + assert_matches!(str_to_record_type("MX"), Ok(RecordType::MX)); + assert_matches!(str_to_record_type("NAPTR"), Ok(RecordType::NAPTR)); + assert_matches!(str_to_record_type("NS"), Ok(RecordType::NS)); + assert_matches!(str_to_record_type("NULL"), Ok(RecordType::NULL)); + assert_matches!(str_to_record_type("OPENPGPKEY"), Ok(RecordType::OPENPGPKEY)); + assert_matches!(str_to_record_type("OPT"), Ok(RecordType::OPT)); + assert_matches!(str_to_record_type("PTR"), Ok(RecordType::PTR)); + assert_matches!(str_to_record_type("SOA"), Ok(RecordType::SOA)); + assert_matches!(str_to_record_type("SRV"), Ok(RecordType::SRV)); + assert_matches!(str_to_record_type("SSHFP"), Ok(RecordType::SSHFP)); + assert_matches!(str_to_record_type("SVCB"), Ok(RecordType::SVCB)); + assert_matches!(str_to_record_type("TLSA"), Ok(RecordType::TLSA)); + assert_matches!(str_to_record_type("TXT"), Ok(RecordType::TXT)); + assert_matches!(str_to_record_type("ZERO"), Ok(RecordType::ZERO)); + assert_matches!(str_to_record_type("NOT A DNS ENTRIE"), Err(_)); } } diff --git a/src/connectors/impls/elastic.rs b/src/connectors/impls/elastic.rs index 5d3aa69cc7..06b00ce7ea 100644 --- a/src/connectors/impls/elastic.rs +++ b/src/connectors/impls/elastic.rs @@ -235,7 +235,7 @@ use crate::{ impls::http::utils::Header, prelude::*, sink::concurrency_cap::ConcurrencyCap, utils::tls::TLSClientConfig, }, - errors::{err_connector_def, Error, Result}, + errors::Result, }; use either::Either; use elasticsearch::{ @@ -258,6 +258,21 @@ use tremor_common::time::nanotime; use tremor_value::utils::sorted_serialize; use tremor_value::value::StaticValue; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("No elasticsearch client available.")] + NoClient, + #[error("Invalid Response from ES: No \"items\" or not an array: {0}")] + InvalidResponse(String), + #[error("Invalid `$elastic.action` {0}")] + InvalidAction(String), + + #[error("Missing field `$elastic._id`")] + MissingId, + #[error("Invalid value for `$elastic.refresh`: {0}")] + InvalidRefresh(String), +} + #[derive(Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub(crate) struct Config { @@ -315,7 +330,7 @@ impl ConnectorBuilder for Builder { ) -> Result> { let config = Config::new(raw_config)?; if config.nodes.is_empty() { - Err(err_connector_def(id, "empty nodes provided")) + Err(error_connector_def(id, "empty nodes provided").into()) } else { let tls_config = match config.tls.as_ref() { Some(Either::Left(tls_config)) => Some(tls_config.clone()), @@ -326,7 +341,7 @@ impl ConnectorBuilder for Builder { for node_url in &config.nodes { if node_url.scheme() != "https" { let e = format!("Node URL '{node_url}' needs 'https' scheme with tls."); - return Err(err_connector_def(id, &e)); + return Err(error_connector_def(id, &e).into()); } } } @@ -401,7 +416,7 @@ impl Connector for Elastic { response_rx: self .response_rx .take() - .ok_or("Elasticsearch source can only be created once.")?, + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, }; Ok(Some(builder.spawn(source, ctx))) } @@ -430,11 +445,15 @@ struct ElasticSource { #[async_trait::async_trait] impl Source for ElasticSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { - Ok(self.response_rx.recv().await.ok_or("channel broken")?) + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + Ok(self.response_rx.recv().await.ok_or(ChannelError::Send)?) } - async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { // we will only know if we are connected to some pipelines if we receive a CBAction::Restore contraflow event // we will not send responses to out/err if we are not connected and this is determined by this variable self.source_is_connected.store(true, Ordering::Release); @@ -545,7 +564,7 @@ impl ElasticSink { #[async_trait::async_trait()] impl Sink for ElasticSink { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let mut clients = Vec::with_capacity(self.config.nodes.len()); for node in &self.config.nodes { let conn_pool = SingleNodeConnectionPool::new(node.url().clone()); @@ -619,7 +638,7 @@ impl Sink for ElasticSink { ctx: &SinkContext, _serializer: &mut EventSerializer, start: u64, - ) -> Result { + ) -> anyhow::Result { if event.is_empty() { debug!("{ctx} Received empty event. Won't send it to ES"); return Ok(SinkReply::NONE); @@ -701,17 +720,14 @@ impl Sink for ElasticSink { } drop(guard); - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }); Ok(SinkReply::NONE) } else { // shouldn't happen actually error!("{} No elasticsearch client available.", &ctx); handle_error( - Error::from(ErrorKind::ClientNotAvailable( - "elastic", - "No elasticsearch client available.", - )), + Error::NoClient.into(), &event, &self.origin_uri, &self.response_tx, @@ -811,10 +827,9 @@ async fn handle_response( response_tx.send(source_reply).await?; } } else { - return Err(Error::from(format!( - "Invalid Response from ES: No \"items\" or not an array: {}", - String::from_utf8(sorted_serialize(&response)?)? - ))); + return Err( + Error::InvalidResponse(String::from_utf8(sorted_serialize(&response)?)?).into(), + ); } // ack the event Ok(()) @@ -823,16 +838,13 @@ async fn handle_response( /// handle an error for the whole event /// /// send event to err port -async fn handle_error( - e: E, +async fn handle_error( + e: anyhow::Error, event: &Event, elastic_origin_uri: &EventOriginUri, response_tx: &Sender, include_payload: bool, -) -> Result<()> -where - E: std::error::Error, -{ +) -> Result<()> { let e_str = e.to_string(); let mut meta = literal!({ "elastic": { @@ -881,9 +893,6 @@ struct ESMeta<'a, 'value> { } impl<'a, 'value> ESMeta<'a, 'value> { - // ALLOW: this is a string - const MISSING_ID: &'static str = "Missing field `$elastic[\"_id\"]`"; - fn new(meta: &'a Value<'value>) -> Self { Self { meta: if let Some(elastic_meta) = meta.get("elastic") { @@ -908,7 +917,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { self.insert_update_op(data, ops) } } - other => Err(Error::from(format!("Invalid `$elastic.action` {other}"))), + other => Err(Error::InvalidAction(other.to_string()).into()), } } @@ -938,7 +947,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { if let Some(pipeline) = self.get_pipeline() { op = op.pipeline(pipeline); } - ops.push(op).map_err(Error::from)?; + ops.push(op)?; Ok(()) } @@ -946,7 +955,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { let mut op: BulkDeleteOperation<()> = self .get_id() .map(BulkOperation::delete) - .ok_or_else(|| Error::from(Self::MISSING_ID))?; + .ok_or(Error::MissingId)?; if let Some(index) = self.get_index() { op = op.index(index); } @@ -966,7 +975,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { op = op.routing(routing); } - ops.push(op).map_err(Error::from)?; + ops.push(op)?; Ok(()) } @@ -977,7 +986,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { let mut op = self .get_id() .map(|id| BulkOperation::create(id, data)) - .ok_or_else(|| Error::from(Self::MISSING_ID))?; + .ok_or(Error::MissingId)?; if let Some(index) = self.get_index() { op = op.index(index); } @@ -987,7 +996,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { if let Some(routing) = self.get_routing() { op = op.routing(routing); } - ops.push(op).map_err(Error::from)?; + ops.push(op)?; Ok(()) } @@ -996,10 +1005,10 @@ impl<'a, 'value> ESMeta<'a, 'value> { let mut op = self .get_id() .map(|id| BulkOperation::update(id, data)) - .ok_or_else(|| Error::from(Self::MISSING_ID))?; + .ok_or(Error::MissingId)?; op = self.apply_update_params(op); - ops.push(op).map_err(Error::from)?; + ops.push(op)?; Ok(()) } @@ -1012,10 +1021,10 @@ impl<'a, 'value> ESMeta<'a, 'value> { let src = literal!({ "doc": data.clone() }); BulkOperation::update(id, src) }) - .ok_or_else(|| Error::from(Self::MISSING_ID))?; + .ok_or(Error::MissingId)?; op = self.apply_update_params(op); - ops.push(op).map_err(Error::from)?; + ops.push(op)?; Ok(()) } @@ -1139,9 +1148,7 @@ impl<'a, 'value> ESMeta<'a, 'value> { } else if refresh.as_str() == Some("wait_for") { Ok(Some(Refresh::WaitFor)) } else if let Some(other) = refresh { - Err(Error::from(format!( - "Invalid value for `$elastic.refresh`: {other}", - ))) + Err(Error::InvalidRefresh(other.to_string()).into()) } else { Ok(None) } @@ -1189,7 +1196,7 @@ mod tests { let connector_config = ConnectorConfig::from_config(&alias, builder.connector_type(), &config)?; assert_eq!( - String::from("Invalid Definition for connector \"my_elastic\": empty nodes provided"), + String::from("[my_elastic] Invalid definition: empty nodes provided"), builder .build(&alias, &connector_config,) .await @@ -1317,7 +1324,7 @@ mod tests { assert_eq!(false, es_meta.get_raw_payload()); assert_eq!(Some(2), es_meta.get_if_primary_term()); assert_eq!(Some(3), es_meta.get_if_seq_no()); - assert_eq!(Ok(Some(Refresh::WaitFor)), es_meta.get_refresh()); + assert_eq!(Some(Refresh::WaitFor), es_meta.get_refresh().ok().flatten()); assert_eq!(Some("routing"), es_meta.get_routing()); assert_eq!(Some("10s"), es_meta.get_timeout()); assert_eq!(Some("ttt"), es_meta.get_type()); diff --git a/src/connectors/impls/exit.rs b/src/connectors/impls/exit.rs index b28e273946..668f449bbc 100644 --- a/src/connectors/impls/exit.rs +++ b/src/connectors/impls/exit.rs @@ -187,7 +187,7 @@ impl Sink for ExitSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { if self.done { debug!("{ctx} Already exited."); } else if let Some((value, _meta)) = event.value_meta_iter().next() { diff --git a/src/connectors/impls/file.rs b/src/connectors/impls/file.rs index b1920483c6..e42d8329c1 100644 --- a/src/connectors/impls/file.rs +++ b/src/connectors/impls/file.rs @@ -290,9 +290,15 @@ impl FileSource { } } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("No file available")] + NoFile, +} + #[async_trait::async_trait] impl Source for FileSource { - async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { self.meta = ctx.meta(literal!({ "path": self.config.path.display().to_string() })); @@ -311,14 +317,15 @@ impl Source for FileSource { }; Ok(true) } - async fn pull_data(&mut self, _pull_id: &mut u64, ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + _pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result { let reply = if self.eof { SourceReply::Finished } else { - let reader = self - .reader - .as_mut() - .ok_or_else(|| Error::from("No file available."))?; + let reader = self.reader.as_mut().ok_or(Error::NoFile)?; let bytes_read = reader.read(&mut self.buf).await?; if bytes_read == 0 { self.eof = true; @@ -343,7 +350,7 @@ impl Source for FileSource { Ok(reply) } - async fn on_stop(&mut self, ctx: &SourceContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SourceContext) -> anyhow::Result<()> { if let Some(mut file) = self.underlying_file.take() { if let Err(e) = file.flush().await { error!("{} Error flushing file: {}", &ctx, e); @@ -381,7 +388,7 @@ impl FileSink { #[async_trait::async_trait] impl Sink for FileSink { - async fn connect(&mut self, _ctx: &SinkContext, attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SinkContext, attempt: &Attempt) -> anyhow::Result { let mode = if attempt.is_first() || attempt.success() == 0 { &self.config.mode } else { @@ -408,11 +415,8 @@ impl Sink for FileSink { ctx: &SinkContext, serializer: &mut EventSerializer, _start: u64, - ) -> Result { - let file = self - .file - .as_mut() - .ok_or_else(|| Error::from("No file available."))?; + ) -> anyhow::Result { + let file = self.file.as_mut().ok_or(Error::NoFile)?; let ingest_ns = event.ingest_ns; for (value, meta) in event.value_meta_iter() { let data = serializer.serialize(value, meta, ingest_ns).await?; @@ -442,7 +446,7 @@ impl Sink for FileSink { false } - async fn on_stop(&mut self, ctx: &SinkContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SinkContext) -> anyhow::Result<()> { if let Some(file) = self.file.take() { if let Err(e) = file.sync_all().await { error!("{} Error flushing file: {}", &ctx, e); diff --git a/src/connectors/impls/gbq.rs b/src/connectors/impls/gbq.rs index f272c61f6d..e074c377c4 100644 --- a/src/connectors/impls/gbq.rs +++ b/src/connectors/impls/gbq.rs @@ -110,3 +110,13 @@ //! [storage API]: https://cloud.google.com/bigquery/docs/reference/storage/rpc/google.cloud.bigquery.storage.v1 pub(crate) mod writer; + +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("No client available")] + NoClient, + #[error("The client is not connected")] + NotConnected, + #[error("The table '{0}' has no schema provided")] + SchemaNotProvided(String), +} diff --git a/src/connectors/impls/gbq/writer/sink.rs b/src/connectors/impls/gbq/writer/sink.rs index 33f7c6dd80..1d89a7431c 100644 --- a/src/connectors/impls/gbq/writer/sink.rs +++ b/src/connectors/impls/gbq/writer/sink.rs @@ -15,7 +15,7 @@ use crate::connectors::google::ChannelFactory; use crate::connectors::{ google::{AuthInterceptor, TokenProvider}, - impls::gbq::writer::Config, + impls::gbq::{writer::Config, Error}, prelude::*, }; use futures::{stream, StreamExt}; @@ -339,7 +339,7 @@ where ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { let request_size_limit = self.config.request_size_limit; let request_data = self @@ -350,10 +350,7 @@ where warn!("{ctx} The batch is too large to be sent in a single request, splitting it into {} requests. Consider lowering the batch size.", request_data.len()); } - let client = self.client.as_mut().ok_or(ErrorKind::ClientNotAvailable( - "BigQuery", - "The client is not connected", - ))?; + let client = self.client.as_mut().ok_or(Error::NotConnected)?; for request in request_data { let req_timeout = Duration::from_nanos(self.config.request_timeout); let append_response = @@ -422,7 +419,7 @@ where Ok(SinkReply::ACK) } - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { info!("{ctx} Connecting to BigQuery"); let channel = self @@ -570,10 +567,7 @@ where table_id: String, ctx: &SinkContext, ) -> Result<&ConnectedWriteStream> { - let client = self.client.as_mut().ok_or(ErrorKind::ClientNotAvailable( - "BigQuery", - "The client is not connected", - ))?; + let client = self.client.as_mut().ok_or(Error::NoClient)?; match self.write_streams.entry(table_id.clone()) { Entry::Occupied(entry) => { @@ -602,7 +596,7 @@ where &stream .table_schema .as_ref() - .ok_or_else(|| ErrorKind::GbqSchemaNotProvided(table_id))? + .ok_or(Error::SchemaNotProvided(table_id))? .clone() .fields, ctx, @@ -1210,11 +1204,7 @@ mod test { fields.try_insert("a", 12); let result = mapping.map(&fields); - if let Err(Error(ErrorKind::TypeError(ValueType::Custom("bytes"), x), _)) = result { - assert_eq!(x, ValueType::I64); - } else { - panic!("Bytes conversion did not fail on type mismatch"); - } + assert!(result.is_err()); } #[test] @@ -1235,11 +1225,7 @@ mod test { ); let result = mapping.map(&Value::from(123_i64)); - if let Err(Error(ErrorKind::TypeError(ValueType::Object, x), _)) = result { - assert_eq!(x, ValueType::I64); - } else { - panic!("Mapping did not fail on non-object event"); - } + assert!(result.is_err()); } #[tokio::test(flavor = "multi_thread")] @@ -1335,8 +1321,7 @@ mod test { }], }), } - .encode(&mut buffer_write_stream) - .map_err(|_| "encode failed")?; + .encode(&mut buffer_write_stream)?; AppendRowsResponse { updated_schema: Some(TableSchema { @@ -1357,8 +1342,7 @@ mod test { details: vec![], })), } - .encode(&mut buffer_append_rows_response) - .map_err(|_| "encode failed")?; + .encode(&mut buffer_append_rows_response)?; let responses = Arc::new(RwLock::new(VecDeque::from([ buffer_write_stream, @@ -1419,8 +1403,7 @@ mod test { }], }), } - .encode(&mut buffer_write_stream) - .map_err(|_| "encode failed")?; + .encode(&mut buffer_write_stream)?; AppendRowsResponse { updated_schema: None, @@ -1428,8 +1411,7 @@ mod test { offset: None, })), } - .encode(&mut buffer_append_rows_response) - .map_err(|_| "encode failed")?; + .encode(&mut buffer_append_rows_response)?; let responses = Arc::new(RwLock::new(VecDeque::from([ buffer_write_stream, @@ -1485,7 +1467,7 @@ mod test { .await?; assert_eq!(result.ack, SinkAck::Ack); - assert_eq!(0, responses.read()?.len()); + assert_eq!(0, responses.read().expect("read lock").len()); Ok(()) } diff --git a/src/connectors/impls/gcl.rs b/src/connectors/impls/gcl.rs index fea212602d..79a9098a38 100644 --- a/src/connectors/impls/gcl.rs +++ b/src/connectors/impls/gcl.rs @@ -259,6 +259,12 @@ use tremor_value::Value; pub(crate) mod writer; +#[derive(Debug, Clone, thiserror::Error)] +enum Error { + #[error("log_severity is not an integer")] + SeverityNotInteger, +} + // The span ID within the trace associated with the log entry. // For Trace spans, this is the same format that the Trace API v2 // uses: a 16-character hexadecimal encoding of an 8-byte array, diff --git a/src/connectors/impls/gcl/writer.rs b/src/connectors/impls/gcl/writer.rs index 936ecf7814..b2839ffa10 100644 --- a/src/connectors/impls/gcl/writer.rs +++ b/src/connectors/impls/gcl/writer.rs @@ -18,11 +18,9 @@ mod sink; use crate::connectors::google::{GouthTokenProvider, TokenSrc}; use crate::connectors::impls::gcl::writer::sink::{GclSink, TonicChannelFactory}; use crate::connectors::prelude::*; -use crate::errors::Error; use googapis::google::api::MonitoredResource; use googapis::google::logging::r#type::LogSeverity; use serde::Deserialize; -use simd_json::OwnedValue; use std::collections::HashMap; use tonic::transport::Channel; use tremor_common::alias; @@ -162,9 +160,9 @@ impl Config { // Override for a specific per event severity if let Some(has_meta) = meta { if let Some(log_severity) = has_meta.get("log_severity") { - return log_severity + return Ok(log_severity .as_i32() - .ok_or_else(|| "log_severity is not an integer".into()); + .ok_or(super::Error::SeverityNotInteger)?); }; } @@ -192,41 +190,29 @@ fn value_to_monitored_resource( match from { None => Ok(None), Some(from) => { - let vt = from.value_type(); - match from { - // Consider refactoring for unwrap_or_default when #1819 is resolved - OwnedValue::Object(from) => { - let kind = from.get("type"); - let kind = kind.as_str(); - let maybe_labels = from.get("labels"); - let labels: HashMap = match maybe_labels { - None => HashMap::new(), - Some(labels) => labels - .as_object() - .ok_or_else(|| { - Error::from(ErrorKind::GclTypeMismatch("Value::Object", vt)) - })? - .iter() - .map(|(key, value)| { - let key = key.to_string(); - let value = value.to_string(); - (key, value) - }) - .collect(), - }; - Ok(Some(MonitoredResource { - r#type: match kind { - None => String::new(), - Some(kind) => kind.to_string(), - }, - labels, - })) - } - _otherwise => Err(Error::from(ErrorKind::GclTypeMismatch( - "Value::Object", - from.value_type(), - ))), - } + let from = from.try_as_object()?; + let kind = from.get("type"); + let kind = kind.as_str(); + let maybe_labels = from.get("labels"); + let labels: HashMap = match maybe_labels { + None => HashMap::new(), + Some(labels) => labels + .try_as_object()? + .iter() + .map(|(key, value)| { + let key = key.to_string(); + let value = value.to_string(); + (key, value) + }) + .collect(), + }; + Ok(Some(MonitoredResource { + r#type: match kind { + None => String::new(), + Some(kind) => kind.to_string(), + }, + labels, + })) } } } @@ -280,10 +266,10 @@ impl ConnectorBuilder for Builder { #[cfg(test)] mod tests { use super::*; + use simd_json::OwnedValue; #[test] fn value_to_monitored_resource_conversion() -> Result<()> { - let mut ok_count = 0; let from: OwnedValue = literal!({ "type": "gce_instance".to_string(), "labels": { @@ -292,30 +278,24 @@ mod tests { } }) .into(); - let value = value_to_monitored_resource(Some(&from))?; - if let Some(value) = value { - assert_eq!("gce_instance", &value.r#type); - assert_eq!("us-central1-a".to_string(), value.labels["zone"]); - assert_eq!( - "00000000000000000000".to_string(), - value.labels["instance_id"] - ); - ok_count += 1; - } else { - return Err("Skipped test asserts due to serialization error".into()); - } + let value = value_to_monitored_resource(Some(&from))? + .expect("Skipped test asserts due to serialization error"); + + assert_eq!("gce_instance", &value.r#type); + assert_eq!("us-central1-a".to_string(), value.labels["zone"]); + assert_eq!( + "00000000000000000000".to_string(), + value.labels["instance_id"] + ); let from: OwnedValue = literal!({ "type": "gce_instance".to_string(), }) .into(); - let value = value_to_monitored_resource(Some(&from))?; - if let Some(value) = value { - assert_eq!(0, value.labels.len()); - ok_count += 1; - } else { - return Err("Skipped test asserts due to serialization error".into()); - } + let value = value_to_monitored_resource(Some(&from))? + .expect("Skipped test asserts due to serialization error"); + + assert_eq!(0, value.labels.len()); let from: OwnedValue = literal!({ "type": "gce_instance".to_string(), @@ -330,7 +310,6 @@ mod tests { let bad_value = value_to_monitored_resource(Some(&from)); assert!(bad_value.is_err()); - assert_eq!(2, ok_count); Ok(()) } } diff --git a/src/connectors/impls/gcl/writer/meta.rs b/src/connectors/impls/gcl/writer/meta.rs index d2340758d5..16035c5a38 100644 --- a/src/connectors/impls/gcl/writer/meta.rs +++ b/src/connectors/impls/gcl/writer/meta.rs @@ -124,17 +124,9 @@ pub(crate) fn span_id(meta: Option<&Value>) -> String { get_or_default(meta, "span_id") } -pub(crate) fn trace_sampled(meta: Option<&Value>) -> Result { +pub(crate) fn trace_sampled(meta: Option<&Value>) -> bool { // Override for a specific per event severity - if let Some(has_meta) = meta { - if let Some(trace_sampled) = has_meta.get("trace_sampled") { - return trace_sampled - .as_bool() - .ok_or_else(|| "trace_sampled is not a boolean".into()); - }; - } - - Ok(false) + meta.get_bool("trace_sampled").unwrap_or(false) } pub(crate) fn source_location(meta: Option<&Value>) -> Option { @@ -200,7 +192,7 @@ mod test { assert_eq!(None, operation(Some(&meta))); assert_eq!(String::new(), trace(Some(&meta))); assert_eq!(String::new(), span_id(Some(&meta))); - assert_eq!(false, trace_sampled(Some(&meta))?); + assert_eq!(false, trace_sampled(Some(&meta))); assert_eq!(None, source_location(Some(&meta))); Ok(()) @@ -421,20 +413,12 @@ mod test { } #[test] - fn trace_sampled_overrides() -> Result<()> { + fn trace_sampled_overrides() { let meta = literal!({ "trace_sampled": true }); - let meta_trace_ok = trace_sampled(Some(&meta))?; + let meta_trace_ok = trace_sampled(Some(&meta)); assert!(meta_trace_ok); - - let meta = literal!({ - "trace_sampled": [ "snot" ] - }); - let trace_err = trace_sampled(Some(&meta)); - assert!(trace_err.is_err()); - - Ok(()) } #[test] diff --git a/src/connectors/impls/gcl/writer/sink.rs b/src/connectors/impls/gcl/writer/sink.rs index 0cf62f066c..1a3abd9385 100644 --- a/src/connectors/impls/gcl/writer/sink.rs +++ b/src/connectors/impls/gcl/writer/sink.rs @@ -78,7 +78,7 @@ fn value_to_log_entry( operation: meta::operation(meta), trace: meta::trace(meta), span_id: meta::span_id(meta), - trace_sampled: meta::trace_sampled(meta)?, + trace_sampled: meta::trace_sampled(meta), source_location: meta::source_location(meta), payload: Some(Payload::JsonPayload(pb::value_to_prost_struct(data)?)), }) @@ -110,6 +110,11 @@ impl< } } } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("The client is not connected")] + NotConnected, +} #[async_trait::async_trait] impl< @@ -133,11 +138,8 @@ where ctx: &SinkContext, _serializer: &mut EventSerializer, start: u64, - ) -> Result { - let client = self.client.as_mut().ok_or(ErrorKind::ClientNotAvailable( - "Google Cloud Logging", - "The client is not connected", - ))?; + ) -> anyhow::Result { + let client = self.client.as_mut().ok_or(Error::NotConnected)?; let mut entries = Vec::with_capacity(event.len()); for (data, meta) in event.value_meta_iter() { @@ -174,7 +176,6 @@ where }), ) .await?; - if let Err(error) = log_entries_response { error!("Failed to write a log entries: {}", error); @@ -194,7 +195,6 @@ where "Failed to notify about Google Cloud Logging connection loss", ); } - reply_tx.send(AsyncSinkReply::Fail(ContraflowData::from(event)))?; } else { reply_tx.send(AsyncSinkReply::Ack( @@ -210,7 +210,7 @@ where Ok(SinkReply::NONE) } - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { info!("{} Connecting to Google Cloud Logging", ctx); let channel = self .channel_factory @@ -376,7 +376,6 @@ mod test { 0, ) .await?; - assert!(matches!( rx.recv().await.expect("no msg"), AsyncSinkReply::CB(_, Trigger) @@ -396,12 +395,8 @@ mod test { let meta = meta.get("gcl_writer").or(None); let result = value_to_log_entry(now, &config, data, meta); - if let Err(Error(ErrorKind::GclTypeMismatch("Value::Object", x), _)) = result { - assert_eq!(x, ValueType::String); - Ok(()) - } else { - Err("Mapping did not fail on non-object event".into()) - } + assert!(result.is_err()); + Ok(()) } #[tokio::test(flavor = "multi_thread")] diff --git a/src/connectors/impls/gcs.rs b/src/connectors/impls/gcs.rs index 0fac005565..4e4502534f 100644 --- a/src/connectors/impls/gcs.rs +++ b/src/connectors/impls/gcs.rs @@ -126,6 +126,31 @@ //! //! [`s3_streamer`]: ./s3.md#s3-streamer +use http::StatusCode; + mod chunked_buffer; mod resumable_upload_client; pub(crate) mod streamer; + +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("No client available")] + NoClient, + #[error("Buffer was marked as done at index {0} which is not in memory anymore")] + InvalidBuffer(usize), + #[error("Not enough data in the buffer")] + NotEnoughData, + #[error("Request still failing after {0} retries")] + RequestFailed(u32), + #[error("Request for bucket {0} failed with {1}")] + Bucket(String, StatusCode), + #[error("Upload failed with {0}")] + Upload(StatusCode), + + #[error("Delete failed with {0}")] + Delete(StatusCode), + #[error("Missing location header")] + MissingLocationHeader, + #[error("Missing or invalite range header")] + MissingOrInvalidRangeHeader, +} diff --git a/src/connectors/impls/gcs/chunked_buffer.rs b/src/connectors/impls/gcs/chunked_buffer.rs index 987398fc80..30bcd78570 100644 --- a/src/connectors/impls/gcs/chunked_buffer.rs +++ b/src/connectors/impls/gcs/chunked_buffer.rs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::Error; use crate::connectors::utils::object_storage::{BufferPart, ObjectStorageBuffer}; -use crate::errors::{err_gcs, Result}; +use crate::errors::Result; /// This structure is similar to a Vec, but with some special methods. /// `write` will add data (any size of data is accepted) @@ -55,16 +56,14 @@ impl ObjectStorageBuffer for ChunkedBuffer { fn mark_done_until(&mut self, position: usize) -> Result<()> { if position < self.buffer_start { - return Err(err_gcs(format!( - "Buffer was marked as done at index {position} which is not in memory anymore" - ))); + return Err(Error::InvalidBuffer(position).into()); } let bytes_to_remove = position - self.buffer_start; self.data = Vec::from( self.data .get(bytes_to_remove..) - .ok_or_else(|| err_gcs("Not enough data in the buffer"))?, + .ok_or(Error::NotEnoughData)?, ); self.buffer_start += bytes_to_remove; diff --git a/src/connectors/impls/gcs/resumable_upload_client.rs b/src/connectors/impls/gcs/resumable_upload_client.rs index 3e5cf0f9d8..223818d244 100644 --- a/src/connectors/impls/gcs/resumable_upload_client.rs +++ b/src/connectors/impls/gcs/resumable_upload_client.rs @@ -12,13 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - connectors::{ - google::TokenSrc, - prelude::{Result, Url}, - utils::object_storage::{BufferPart, ObjectId}, - }, - errors::err_gcs, +use crate::connectors::{ + google::TokenSrc, + prelude::{Result, Url}, + utils::object_storage::{BufferPart, ObjectId}, }; #[cfg(not(test))] use gouth::Token; @@ -29,6 +26,8 @@ use std::time::Duration; use tokio::time::sleep; use tremor_common::url::HttpsDefaults; +use super::Error; + pub(crate) type GcsHttpClient = hyper::Client>; @@ -129,9 +128,7 @@ async fn retriable_request< } } - Err(err_gcs(format!( - "Request still failing after {max_retries} retries" - ))) + Err(Error::RequestFailed(max_retries).into()) } pub(crate) struct DefaultClient { @@ -169,10 +166,7 @@ impl let body_string = String::from_utf8(data)?; // assuming error error!("Error checking that bucket exists: {body_string}",); - return Err(err_gcs(format!( - "Check if bucket {bucket} exists failed with {} status", - response.status() - ))); + return Err(Error::Bucket(bucket.to_string(), status).into()); } } @@ -199,17 +193,14 @@ impl let body_string = String::from_utf8(data)?; error!("Error from Google Cloud Storage: {body_string}",); - return Err(err_gcs(format!( - "Start upload failed with {} status", - response.status() - ))); + return Err(Error::Upload(response.status()).into()); } Ok(url::Url::parse( response .headers() .get("Location") - .ok_or_else(|| err_gcs("Missing Location header".to_string()))? + .ok_or(Error::MissingLocationHeader)? .to_str()?, )?) } @@ -243,16 +234,13 @@ impl } let body_string = String::from_utf8(data)?; error!("Error from Google Cloud Storage: {body_string}",); - return Err(err_gcs(format!( - "Start upload failed with status {}", - response.status() - ))); + return Err(Error::Upload(response.status()).into()); } let raw_range = response .headers() .get(header::RANGE) - .ok_or_else(|| err_gcs("No range header"))?; + .ok_or(Error::MissingOrInvalidRangeHeader)?; let raw_range = raw_range.to_str()?; // Range format: bytes=0-262143 @@ -260,10 +248,10 @@ impl .get( raw_range .find('-') - .ok_or_else(|| err_gcs("Did not find a - in the Range header"))? + .ok_or(Error::MissingOrInvalidRangeHeader)? + 1.., ) - .ok_or_else(|| err_gcs("Unable to get the end of the Range"))?; + .ok_or(Error::MissingOrInvalidRangeHeader)?; Ok(range_end.parse()?) } @@ -288,10 +276,7 @@ impl let body_string = String::from_utf8(data)?; error!("Error from Google Cloud Storage while cancelling an upload: {body_string}",); - Err(err_gcs(format!( - "Delete upload failed with status {}", - response.status() - ))) + Err(Error::Delete(response.status()).into()) } } @@ -325,10 +310,7 @@ impl error!("Error from Google Cloud Storage: {body_string}"); - return Err(err_gcs(format!( - "Finish upload failed with status {}", - response.status() - ))); + return Err(Error::Upload(response.status()).into()); } Ok(()) @@ -732,7 +714,7 @@ mod tests { }, || { if !request_handled_clone.swap(true, Ordering::Acquire) { - return Err("boo".into()); + anyhow::bail!("boo"); } Ok(Request::builder() diff --git a/src/connectors/impls/gcs/streamer.rs b/src/connectors/impls/gcs/streamer.rs index ee9954c93a..84e30139fe 100644 --- a/src/connectors/impls/gcs/streamer.rs +++ b/src/connectors/impls/gcs/streamer.rs @@ -12,29 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - connectors::{ - google::TokenSrc, - impls::gcs::{ - chunked_buffer::ChunkedBuffer, - resumable_upload_client::{ - create_client, DefaultClient, ExponentialBackoffRetryStrategy, GcsHttpClient, - ResumableUploadClient, - }, - }, - prelude::*, - utils::object_storage::{ - BufferPart, ConsistentSink, Mode, ObjectId, ObjectStorageCommon, ObjectStorageSinkImpl, - ObjectStorageUpload, YoloSink, +use crate::connectors::{ + google::TokenSrc, + impls::gcs::{ + chunked_buffer::ChunkedBuffer, + resumable_upload_client::{ + create_client, DefaultClient, ExponentialBackoffRetryStrategy, GcsHttpClient, + ResumableUploadClient, }, }, - errors::err_gcs, + prelude::*, + utils::object_storage::{ + BufferPart, ConsistentSink, Mode, ObjectId, ObjectStorageCommon, ObjectStorageSinkImpl, + ObjectStorageUpload, YoloSink, + }, }; use std::time::Duration; use tremor_common::time::nanotime; use tremor_pipeline::{EventId, OpMeta}; use tremor_value::Value; +use super::Error; + const CONNECTOR_TYPE: &str = "gcs_streamer"; #[derive(Deserialize, Debug, Clone)] @@ -285,10 +284,7 @@ impl ObjectStorageSinkImpl Result { - let client = self - .upload_client - .as_mut() - .ok_or_else(|| err_gcs("No client available"))?; + let client = self.upload_client.as_mut().ok_or(Error::NoClient)?; client.bucket_exists(&self.config.url, bucket).await } async fn start_upload( @@ -297,10 +293,7 @@ impl ObjectStorageSinkImpl Result { - let client = self - .upload_client - .as_mut() - .ok_or_else(|| err_gcs("No client available"))?; + let client = self.upload_client.as_mut().ok_or(Error::NoClient)?; let session_uri = client .start_upload(&self.config.url, object_id.clone()) .await?; @@ -314,10 +307,7 @@ impl ObjectStorageSinkImpl Result { - let client = self - .upload_client - .as_mut() - .ok_or_else(|| err_gcs("No client available"))?; + let client = self.upload_client.as_mut().ok_or(Error::NoClient)?; debug!( "{ctx} Uploading bytes {}-{} for {}", data.start(), @@ -336,10 +326,7 @@ impl ObjectStorageSinkImpl ObjectStorageSinkImpl Result { if self.inject_failure { - return Err(err_gcs("Error on start_upload")); + return Err(Error::Upload(StatusCode::INTERNAL_SERVER_ERROR).into()); } let count = self.counter.fetch_add(1, Ordering::AcqRel); let mut session_uri = url.url().clone(); @@ -464,7 +449,7 @@ pub(crate) mod tests { } async fn upload_data(&mut self, url: &url::Url, part: BufferPart) -> Result { if self.inject_failure { - return Err(err_gcs("Error on upload_data")); + return Err(Error::Upload(StatusCode::INTERNAL_SERVER_ERROR).into()); } if let Some((_file_id, buffers)) = self.running.get_mut(url) { let end = part.start + part.len(); @@ -472,19 +457,19 @@ pub(crate) mod tests { // TODO: create mode where it always keeps some bytes Ok(end) } else { - Err("upload not found".into()) + Err(crate::connectors::utils::object_storage::Error::NoUpload.into()) } } async fn finish_upload(&mut self, url: &url::Url, part: BufferPart) -> Result<()> { if self.inject_failure { - return Err(err_gcs("Error on finish_upload")); + return Err(Error::Upload(StatusCode::INTERNAL_SERVER_ERROR).into()); } if let Some((file_id, mut buffers)) = self.running.remove(url) { buffers.push(part); self.finished.insert(url.clone(), (file_id, buffers)); Ok(()) } else { - Err("upload not found".into()) + return Err(Error::Upload(StatusCode::INTERNAL_SERVER_ERROR).into()); } } async fn delete_upload(&mut self, url: &url::Url) -> Result<()> { @@ -493,16 +478,14 @@ pub(crate) mod tests { self.deleted.insert(url.clone(), upload); Ok(()) } else { - Err("upload not found".into()) + return Err(Error::Upload(StatusCode::INTERNAL_SERVER_ERROR).into()); } } - async fn bucket_exists( - &mut self, - _url: &Url, - _bucket: &str, - ) -> Result { + async fn bucket_exists(&mut self, _url: &Url, bucket: &str) -> Result { if self.inject_failure { - return Err(err_gcs("Error on bucket_exists")); + return Err( + Error::Bucket(bucket.to_string(), StatusCode::INTERNAL_SERVER_ERROR).into(), + ); } Ok(true) } @@ -594,15 +577,15 @@ pub(crate) mod tests { assert_eq!(1, upload_client(&mut sink).count()); let mut uploads = upload_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, mut buffers) = uploads.pop().ok_or("no data")?; + let (file_id, mut buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("yolo", "happy.txt"), file_id); assert_eq!(2, buffers.len()); - let buffer = buffers.pop().ok_or("no data")?; + let buffer = buffers.pop().expect("no data"); assert_eq!(10, buffer.start); assert_eq!(10, buffer.len()); assert_eq!("badg\",\"er\"".as_bytes(), &buffer.data); - let buffer = buffers.pop().ok_or("no data")?; + let buffer = buffers.pop().expect("no data"); assert_eq!(0, buffer.start); assert_eq!(10, buffer.len()); assert_eq!("{\"snot\":[\"".as_bytes(), &buffer.data); @@ -665,30 +648,30 @@ pub(crate) mod tests { assert_eq!(2, upload_client(&mut sink).count()); let mut uploads = upload_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, mut buffers) = uploads.pop().ok_or("no data")?; + let (file_id, mut buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("yolo", "sad.txt"), file_id); assert_eq!(2, buffers.len()); - let buffer = buffers.pop().ok_or("no data")?; + let buffer = buffers.pop().expect("no data"); assert_eq!(10, buffer.start); assert_eq!(10, buffer.len()); assert_eq!("e,false,nu".as_bytes(), &buffer.data); - let buffer = buffers.pop().ok_or("no data")?; + let buffer = buffers.pop().expect("no data"); assert_eq!(0, buffer.start); assert_eq!(10, buffer.len()); assert_eq!("[1,2,3,tru".as_bytes(), &buffer.data); let mut finished = upload_client(&mut sink).finished_uploads(); assert_eq!(1, finished.len()); - let (file_id, mut buffers) = finished.pop().ok_or("no data")?; + let (file_id, mut buffers) = finished.pop().expect("no data"); assert_eq!(ObjectId::new("yolo", "happy.txt"), file_id); assert_eq!(4, buffers.len()); - let buffer = buffers.pop().ok_or("no data")?; + let buffer = buffers.pop().expect("no data"); assert_eq!(30, buffer.start); assert_eq!("adger\"".as_bytes(), &buffer.data); - let buffer = buffers.pop().ok_or("no data")?; + let buffer = buffers.pop().expect("no data"); assert_eq!(20, buffer.start); assert_eq!("]}\"snot\"\"b".as_bytes(), &buffer.data); @@ -699,10 +682,10 @@ pub(crate) mod tests { // we finish outstanding upload upon stop let mut finished = upload_client(&mut sink).finished_uploads(); assert_eq!(2, finished.len()); - let (file_id, mut buffers) = finished.pop().ok_or("no data")?; + let (file_id, mut buffers) = finished.pop().expect("no data"); assert_eq!(ObjectId::new("yolo", "sad.txt"), file_id); assert_eq!(3, buffers.len()); - let last = buffers.pop().ok_or("no data")?; + let last = buffers.pop().expect("no data"); assert_eq!(20, last.start); assert_eq!("ll]".as_bytes(), &last.data); @@ -792,7 +775,7 @@ pub(crate) mod tests { assert_eq!(1, upload_client(&mut sink).count()); let mut uploads = upload_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, buffers) = uploads.pop().ok_or("no data")?; + let (file_id, buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("failure", "test.txt"), file_id); assert!(buffers.is_empty()); assert!(upload_client(&mut sink).finished_uploads().is_empty()); @@ -825,7 +808,7 @@ pub(crate) mod tests { assert_eq!(1, upload_client(&mut sink).count()); let mut uploads = upload_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, buffers) = uploads.pop().ok_or("no data")?; + let (file_id, buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("failure", "test.txt"), file_id); assert!(buffers.is_empty()); assert!(upload_client(&mut sink).finished_uploads().is_empty()); @@ -854,7 +837,7 @@ pub(crate) mod tests { assert_eq!(1, upload_client(&mut sink).count()); let mut uploads = upload_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, buffers) = uploads.pop().ok_or("no data")?; + let (file_id, buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("failure", "test.txt"), file_id); assert!(buffers.is_empty()); assert!(upload_client(&mut sink).finished_uploads().is_empty()); @@ -1045,7 +1028,7 @@ pub(crate) mod tests { // verify it started the upload upon first request let mut uploads = test_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, buffers) = uploads.pop().ok_or("no data")?; + let (file_id, buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("woah", "test.txt"), file_id); assert!(buffers.is_empty()); assert_eq!(1, test_client(&mut sink).count()); @@ -1074,18 +1057,18 @@ pub(crate) mod tests { // verify it did upload some parts let mut uploads = test_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, mut buffers) = uploads.pop().ok_or("no data")?; + let (file_id, mut buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("woah", "test.txt"), file_id); assert_eq!(3, buffers.len()); - let part = buffers.pop().ok_or("no data")?; + let part = buffers.pop().expect("no data"); assert_eq!(20, part.start); assert_eq!(10, part.len()); assert_eq!("qrstuvwxyz".as_bytes(), &part.data); - let part = buffers.pop().ok_or("no data")?; + let part = buffers.pop().expect("no data"); assert_eq!(10, part.start); assert_eq!(10, part.len()); assert_eq!("ghijklmnop".as_bytes(), &part.data); - let part = buffers.pop().ok_or("no data")?; + let part = buffers.pop().expect("no data"); assert_eq!(0, part.start); assert_eq!(10, part.len()); assert_eq!("{}[\"abcdef".as_bytes(), &part.data); @@ -1116,7 +1099,7 @@ pub(crate) mod tests { let mut uploads = test_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, buffers) = uploads.pop().ok_or("no data")?; + let (file_id, buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("woah2", "test.txt"), file_id); assert!(buffers.is_empty()); @@ -1125,11 +1108,11 @@ pub(crate) mod tests { // 1 finished upload let mut finished = test_client(&mut sink).finished_uploads(); assert_eq!(1, finished.len()); - let (file_id, mut buffers) = finished.pop().ok_or("no data")?; + let (file_id, mut buffers) = finished.pop().expect("no data"); assert_eq!(ObjectId::new("woah", "test.txt"), file_id); assert_eq!(4, buffers.len()); - let last = buffers.pop().ok_or("no data")?; + let last = buffers.pop().expect("no data"); assert_eq!(30, last.start); assert_eq!(2, last.len()); assert_eq!("\"]".as_bytes(), &last.data); @@ -1167,7 +1150,7 @@ pub(crate) mod tests { let mut uploads = test_client(&mut sink).running_uploads(); assert_eq!(1, uploads.len()); - let (file_id, buffers) = uploads.pop().ok_or("no data")?; + let (file_id, buffers) = uploads.pop().expect("no data"); assert_eq!(ObjectId::new("woah2", "test5.txt"), file_id); assert!(buffers.is_empty()); @@ -1176,11 +1159,11 @@ pub(crate) mod tests { // 2 finished uploads let mut finished = test_client(&mut sink).finished_uploads(); assert_eq!(2, finished.len()); - let (file_id, mut buffers) = finished.pop().ok_or("no data")?; + let (file_id, mut buffers) = finished.pop().expect("no data"); assert_eq!(ObjectId::new("woah2", "test.txt"), file_id); assert_eq!(1, buffers.len()); - let last = buffers.pop().ok_or("no data")?; + let last = buffers.pop().expect("no data"); assert_eq!(0, last.start); assert_eq!(2, last.len()); assert_eq!("42".as_bytes(), &last.data); @@ -1378,7 +1361,7 @@ pub(crate) mod tests { .await?; let (tx, mut rx) = bounded(1); addr.stop(tx).await?; - assert!(rx.recv().await.expect("rx empty").res.is_ok()); + assert!(rx.recv().await.expect("rx empty").is_ok()); Ok(()) } @@ -1415,7 +1398,7 @@ pub(crate) mod tests { .await?; let (tx, mut rx) = bounded(1); addr.stop(tx).await?; - assert!(rx.recv().await.expect("rx empty").res.is_ok()); + assert!(rx.recv().await.expect("rx empty").is_ok()); Ok(()) } } diff --git a/src/connectors/impls/gpubsub.rs b/src/connectors/impls/gpubsub.rs index 75de4fcb6b..a89054cc8f 100644 --- a/src/connectors/impls/gpubsub.rs +++ b/src/connectors/impls/gpubsub.rs @@ -138,6 +138,12 @@ fn default_request_timeout() -> u64 { 10_000_000_000u64 // 10 seconds } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("The client is not connected")] + NotConnected, +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/connectors/impls/gpubsub/consumer.rs b/src/connectors/impls/gpubsub/consumer.rs index 6723f67034..7d60286614 100644 --- a/src/connectors/impls/gpubsub/consumer.rs +++ b/src/connectors/impls/gpubsub/consumer.rs @@ -37,6 +37,8 @@ use tonic::transport::{Certificate, Channel, ClientTlsConfig}; use tonic::{Code, Status}; use tremor_common::blue_green_hashmap::BlueGreenHashMap; +use super::Error; + // controlling retries upon gpubsub returning `Unavailable` from StreamingPull // this in on purpose not exposed via config as this should remain an internal thing const RETRY_WAIT_INTERVAL: Duration = Duration::from_secs(1); @@ -223,7 +225,7 @@ async fn consumer_task( attempt += 1; if attempt >= MAX_RETRIES { info!("{ctx} Got `Unavailable` for {MAX_RETRIES} times. Bailing out!"); - return Err(Error::from(status)); + return Err(status.into()); } info!( "{ctx} ERROR: {status}. Waiting for {}s for a retry...", @@ -236,7 +238,7 @@ async fn consumer_task( continue 'retry; } Err(s) => { - return Err(Error::from(s)); + return Err(s.into()); } }; @@ -286,7 +288,7 @@ fn pubsub_metadata( #[async_trait::async_trait] impl Source for GSubSource { - async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { let mut channel = Channel::from_shared(self.config.url.to_string())? .connect_timeout(Duration::from_nanos(self.config.connect_timeout)); if self.url.scheme() == "https" { @@ -353,12 +355,13 @@ impl Source for GSubSource { Ok(true) } - async fn pull_data(&mut self, pull_id: &mut u64, _ctx: &SourceContext) -> Result { - let receiver = self.receiver.as_mut().ok_or(ErrorKind::ClientNotAvailable( - "PubSub", - "The receiver is not connected", - ))?; - let (ack_id, pubsub_message) = receiver.recv().await.ok_or("The channel is closed")??; + async fn pull_data( + &mut self, + pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + let receiver = self.receiver.as_mut().ok_or(Error::NotConnected)?; + let (ack_id, pubsub_message) = receiver.recv().await.ok_or(ChannelError::Recv)??; *pull_id = ack_id; Ok(SourceReply::Data { origin_uri: EventOriginUri::default(), @@ -388,14 +391,13 @@ impl Source for GSubSource { true } - async fn ack(&mut self, _stream_id: u64, pull_id: u64, _ctx: &SourceContext) -> Result<()> { - let sender = self - .ack_sender - .as_mut() - .ok_or(ErrorKind::ClientNotAvailable( - "PubSub", - "The client is not connected", - ))?; + async fn ack( + &mut self, + _stream_id: u64, + pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { + let sender = self.ack_sender.as_mut().ok_or(Error::NotConnected)?; sender.send(pull_id).await?; Ok(()) diff --git a/src/connectors/impls/gpubsub/producer.rs b/src/connectors/impls/gpubsub/producer.rs index 66856584c3..0f4eb776df 100644 --- a/src/connectors/impls/gpubsub/producer.rs +++ b/src/connectors/impls/gpubsub/producer.rs @@ -28,6 +28,8 @@ use tremor_common::url::HttpsDefaults; use tremor_pipeline::Event; use tremor_value::Value; +use super::Error; + #[derive(Deserialize, Clone)] #[serde(deny_unknown_fields)] pub(crate) struct Config { @@ -93,8 +95,8 @@ impl Connector for GpubConnector { .url .host_str() .ok_or_else(|| { - ErrorKind::InvalidConfiguration( - "gpubsub-publisher".to_string(), + ConnectorError::InvalidConfiguration( + ctx.alias().clone(), "Missing hostname".to_string(), ) })? @@ -122,7 +124,7 @@ struct GpubSink { #[async_trait::async_trait()] impl Sink for GpubSink { - async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let mut channel = Channel::from_shared(self.config.url.to_string())? .connect_timeout(Duration::from_nanos(self.config.connect_timeout)); if self.config.url.scheme() == "https" { @@ -152,11 +154,8 @@ impl Sink for GpubSink { ctx: &SinkContext, serializer: &mut EventSerializer, _start: u64, - ) -> Result { - let client = self.client.as_mut().ok_or(ErrorKind::ClientNotAvailable( - "PubSub", - "The publisher is not connected", - ))?; + ) -> anyhow::Result { + let client = self.client.as_mut().ok_or(Error::NotConnected)?; let mut messages = Vec::with_capacity(event.len()); diff --git a/src/connectors/impls/http/auth.rs b/src/connectors/impls/http/auth.rs index cd7fd1dee8..1e9e96aa66 100644 --- a/src/connectors/impls/http/auth.rs +++ b/src/connectors/impls/http/auth.rs @@ -79,15 +79,18 @@ mod tests { password: "snot".to_string(), }; assert_eq!( - Ok(Some("Basic YmFkZ2VyOnNub3Q=".to_string())), - auth.as_header_value() + Some("Basic YmFkZ2VyOnNub3Q=".to_string()), + auth.as_header_value().ok().flatten() ); } #[test] fn header_value_bearer() { let auth = Auth::Bearer("token".to_string()); - assert_eq!(Ok(Some("Bearer token".to_string())), auth.as_header_value()); + assert_eq!( + Some("Bearer token".to_string()), + auth.as_header_value().ok().flatten() + ); } #[test] @@ -97,8 +100,8 @@ mod tests { api_key: "snot".to_string(), }; assert_eq!( - Ok(Some("ApiKey YmFkZ2VyOnNub3Q=".to_string())), - auth.as_header_value() + Some("ApiKey YmFkZ2VyOnNub3Q=".to_string()), + auth.as_header_value().ok().flatten() ); } } diff --git a/src/connectors/impls/http/client.rs b/src/connectors/impls/http/client.rs index 189ecfbd7f..a8194591fc 100644 --- a/src/connectors/impls/http/client.rs +++ b/src/connectors/impls/http/client.rs @@ -15,14 +15,14 @@ use super::auth::Auth; use super::meta::{extract_request_meta, extract_response_meta, HttpRequestBuilder}; use super::utils::{Header, RequestId}; +use crate::connectors::prelude::*; use crate::connectors::sink::concurrency_cap::ConcurrencyCap; use crate::connectors::utils::mime::MimeCodecMap; use crate::connectors::utils::tls::TLSClientConfig; use crate::{ + channel::empty_e, channel::{bounded, Receiver, Sender}, - errors::empty_error, }; -use crate::{connectors::prelude::*, errors::err_connector_def}; use either::Either; use halfbrown::HashMap; use http_body::Body; @@ -126,10 +126,10 @@ impl ConnectorBuilder for Builder { Some(Either::Right(false)) | None => None, }; if config.url.scheme() == "https" && tls_client_config.is_none() { - return Err(err_connector_def( + return Err(error_connector_def( id, "missing tls config with 'https' url. Set 'tls' to 'true' or provide a full tls config.", - )); + ).into()); } let (response_tx, response_rx) = bounded(qsize()); let mime_codec_map = Arc::new(if let Some(codec_map) = config.mime_mapping.clone() { @@ -173,7 +173,10 @@ impl Connector for Client { ) -> Result> { let source = HttpRequestSource { source_is_connected: self.source_is_connected.clone(), - rx: self.response_rx.take().ok_or("source already created")?, + rx: self + .response_rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, }; Ok(Some(builder.spawn(source, ctx))) } @@ -207,8 +210,12 @@ struct HttpRequestSource { #[async_trait::async_trait()] impl Source for HttpRequestSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { - self.rx.recv().await.ok_or_else(empty_error) + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + Ok(self.rx.recv().await.ok_or_else(empty_e)?) } fn is_transactional(&self) -> bool { @@ -220,7 +227,7 @@ impl Source for HttpRequestSource { false } - async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { // we will only know if we are connected to some pipelines if we receive a CBAction::Restore contraflow event // we will not send responses to out/err if we are not connected and this is determined by this variable self.source_is_connected.store(true, Ordering::Release); @@ -280,7 +287,7 @@ impl HttpRequestSink { #[async_trait::async_trait()] impl Sink for HttpRequestSink { - async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let https = if let Some(tls_config) = self.tls_client_config.clone() { HttpsConnectorBuilder::new() .with_tls_config(tls_config) @@ -311,7 +318,7 @@ impl Sink for HttpRequestSink { ctx: &SinkContext, serializer: &mut EventSerializer, start: u64, - ) -> Result { + ) -> anyhow::Result { // constrain to max concurrency - propagate CB close on hitting limit let guard = self.concurrency_cap.inc_for(&event)?; @@ -438,7 +445,7 @@ impl Sink for HttpRequestSink { } } drop(guard); - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }); // if we have a chunked request we still gotta do some work (sending the chunks) diff --git a/src/connectors/impls/http/meta.rs b/src/connectors/impls/http/meta.rs index 3645dce0d5..687509d6e5 100644 --- a/src/connectors/impls/http/meta.rs +++ b/src/connectors/impls/http/meta.rs @@ -26,6 +26,16 @@ use hyper::{header::HeaderValue, Body, Method, Request, Response}; use mime::Mime; use tremor_value::Value; +#[derive(Clone, Debug, thiserror::Error)] +enum Error { + #[error("Invalid HTTP Method")] + InvalidMethod, + #[error("Invalid HTTP URL")] + InvalidUrl, + #[error("Request already consumed")] + AlreadyConsumed, +} + /// Utility for building an HTTP request from a possibly batched event /// and some configuration values pub(crate) struct HttpRequestBuilder { @@ -95,12 +105,12 @@ impl HttpRequestBuilder { ) -> Result { let request_meta = meta.get("request"); let method = if let Some(method_v) = request_meta.get("method") { - Method::from_bytes(method_v.as_bytes().ok_or("Invalid HTTP Method")?)? + Method::from_bytes(method_v.as_bytes().ok_or(Error::InvalidMethod)?)? } else { config.method.0.clone() }; let uri: Uri = if let Some(url_v) = request_meta.get("url") { - url_v.as_str().ok_or("Invalid HTTP URL")?.parse()? + url_v.as_str().ok_or(Error::InvalidUrl)?.parse()? } else { config.url.to_string().parse()? }; @@ -193,7 +203,7 @@ impl HttpRequestBuilder { } pub(super) fn take_request(&mut self) -> Result> { - Ok(self.request.take().ok_or("Request already consumed")?) + Ok(self.request.take().ok_or(Error::AlreadyConsumed)?) } /// Finalize and send the response. /// In the chunked case we have already sent it before. @@ -209,9 +219,7 @@ impl HttpRequestBuilder { } } -pub(crate) fn content_type( - headers: Option<&HeaderMap>, -) -> std::result::Result, Box> { +pub(crate) fn content_type(headers: Option<&HeaderMap>) -> Result> { if let Some(headers) = headers { let header_content_type = headers .get(hyper::header::CONTENT_TYPE) diff --git a/src/connectors/impls/http/server.rs b/src/connectors/impls/http/server.rs index ad86ce5b38..65a6a2f87a 100644 --- a/src/connectors/impls/http/server.rs +++ b/src/connectors/impls/http/server.rs @@ -16,17 +16,14 @@ use super::{ meta::{consolidate_mime, content_type, extract_request_meta, HeaderValueValue}, utils::RequestId, }; +use crate::connectors::{ + prelude::*, + utils::{mime::MimeCodecMap, tls::TLSServerConfig}, +}; use crate::{ + channel::empty_e, channel::{bounded, Receiver, Sender}, connectors::utils::tls, - errors::empty_error, -}; -use crate::{ - connectors::{ - prelude::*, - utils::{mime::MimeCodecMap, tls::TLSServerConfig}, - }, - errors::err_connector_def, }; use dashmap::DashMap; use halfbrown::{Entry, HashMap}; @@ -86,7 +83,7 @@ impl ConnectorBuilder for Builder { let tls_server_config = config.tls.clone(); if tls_server_config.is_some() && config.url.scheme() != "https" { - return Err(err_connector_def(id, Self::HTTPS_REQUIRED)); + return Err(error_connector_def(id, Self::HTTPS_REQUIRED).into()); } let origin_uri = EventOriginUri { scheme: "http-server".to_string(), @@ -177,9 +174,15 @@ struct HttpServerSource { codec_map: MimeCodecMap, } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Address is missing")] + AddressMissing, +} + #[async_trait::async_trait()] impl Source for HttpServerSource { - async fn on_stop(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_stop(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { if let Some(accept_task) = self.server_task.take() { // stop acceptin' new connections accept_task.abort(); @@ -187,7 +190,7 @@ impl Source for HttpServerSource { Ok(()) } - async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { let host = self.url.host_or_local().to_string(); let port = self.url.port().unwrap_or_else(|| { if self.url.scheme() == "https" { @@ -215,7 +218,10 @@ impl Source for HttpServerSource { .map(Arc::new); let server_context = HttpServerState::new(tx, ctx.clone(), "http"); - let addr = (host, port).to_socket_addrs()?.next().ok_or("no address")?; + let addr = (host, port) + .to_socket_addrs()? + .next() + .ok_or(Error::AddressMissing)?; // Server task - this is the main receive loop for http server instances self.server_task = Some(spawn_task(ctx.clone(), async move { @@ -280,13 +286,17 @@ impl Source for HttpServerSource { Ok(true) } - async fn pull_data(&mut self, pull_id: &mut u64, ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result { let RawRequestData { data, request_meta, content_type, response_channel, - } = self.request_rx.recv().await.ok_or_else(empty_error)?; + } = self.request_rx.recv().await.ok_or_else(empty_e)?; // assign request id, set pull_id let request_id = RequestId::new(self.request_counter); @@ -371,7 +381,7 @@ impl Sink for HttpServerSink { ctx: &SinkContext, serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { let ingest_ns = event.ingest_ns; let min_pull_id = event .id @@ -526,7 +536,7 @@ impl Sink for HttpServerSink { _signal: Event, _ctx: &SinkContext, _serializer: &mut EventSerializer, - ) -> Result { + ) -> anyhow::Result { // clean out closed channels self.inflight.retain(|_key, sender| !sender.is_closed()); Ok(SinkReply::NONE) @@ -608,7 +618,7 @@ impl SinkResponse { } }); tx.send(response.body(body)?) - .map_err(|_| "Error sending response")?; + .map_err(|_| ChannelError::Send)?; Ok(Self { request_id, chunk_tx, diff --git a/src/connectors/impls/kafka.rs b/src/connectors/impls/kafka.rs index bb605a2542..d038ba90a4 100644 --- a/src/connectors/impls/kafka.rs +++ b/src/connectors/impls/kafka.rs @@ -305,6 +305,18 @@ use tokio::task::JoinHandle; use tremor_script::EventPayload; use tremor_value::Value; +#[derive(Debug, Clone, thiserror::Error)] +enum Error { + #[error("Unknown stats client_type \"{0}\"")] + UnknownStatsClient(String), + #[error("not an int value")] + InvalidInt, + #[error("Provided rdkafka_option that will be overwritten: {0}")] + OptionOverwritten(String), + #[error("Cannot enable `retry_failed_events` and `enable.auto.commit` and `enable.auto.offset.store` at the same time.")] + CommitConfigConflict, +} + const KAFKA_CONNECT_TIMEOUT: Duration = Duration::from_secs(1); /// verify broker host:port pairs in kafka connector configs @@ -317,17 +329,17 @@ fn verify_brokers(alias: &alias::Connector, brokers: &[String]) -> Result<(Strin } [host, port] => { let port: u16 = port.parse().map_err(|_| { - err_connector_def(alias, &format!("Invalid broker: {host}:{port}")) + error_connector_def(alias, &format!("Invalid broker: {host}:{port}")) })?; first_broker.get_or_insert_with(|| ((*host).to_string(), Some(port))); } b => { let e = format!("Invalid broker: {}", b.join(":")); - return Err(err_connector_def(alias, &e)); + return Err(error_connector_def(alias, &e).into()); } } } - first_broker.ok_or_else(|| err_connector_def(alias, "Missing brokers.")) + first_broker.ok_or_else(|| error_connector_def(alias, "Missing brokers.").into()) } /// Returns `true` if the error denotes a failed connect attempt @@ -457,7 +469,7 @@ where make_metrics_payload(Self::KAFKA_CONSUMER_STATS, fields, tags, timestamp) } other => { - return Err(format!("Unknown stats client_type \"{other}\"").into()); + return Err(Error::UnknownStatsClient(other.to_string()).into()); } }; self.metrics_tx.send(metrics_payload)?; diff --git a/src/connectors/impls/kafka/consumer.rs b/src/connectors/impls/kafka/consumer.rs index 36adc743be..895339b4e2 100644 --- a/src/connectors/impls/kafka/consumer.rs +++ b/src/connectors/impls/kafka/consumer.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::channel::empty_e; use crate::connectors::{ impls::kafka::{TremorRDKafkaContext, KAFKA_CONNECT_TIMEOUT, NO_ERROR}, prelude::*, }; -use crate::errors::empty_error; use futures::StreamExt; use halfbrown::HashMap; use indexmap::IndexMap; @@ -38,6 +38,8 @@ use tokio::{ use tremor_common::time::nanotime; use tremor_value::value::StaticValue; +use super::Error; + const KAFKA_CONSUMER_META_KEY: &str = "kafka_consumer"; #[derive(Deserialize, Debug, Clone, Default)] @@ -94,7 +96,7 @@ impl Mode { } else if let Some(str_int) = v.as_str() { str_int.parse::()? } else { - return Err("not an int value".into()); + return Err(Error::InvalidInt.into()); }; Ok(res) } @@ -190,7 +192,7 @@ impl Mode { .unwrap_or_default() && *retry_failed_events { - return Err("Cannot enable `retry_failed_events` and `enable.auto.commit` and `enable.auto.offset.store` at the same time.".into()); + return Err(Error::CommitConfigConflict.into()); } for (k, v) in rdkafka_options { client_config.set(k, v.to_string()); @@ -278,12 +280,7 @@ impl ConnectorBuilder for Builder { }; let client_id = format!("tremor-{}-{alias}", hostname()); - let mut client_config = config.mode.to_config().map_err(|e| { - Error::from(ErrorKind::InvalidConfiguration( - alias.to_string(), - e.to_string(), - )) - })?; + let mut client_config = config.mode.to_config()?; // we do overwrite the rdkafka options to ensure a sane config set_client_config(&mut client_config, "group.id", &config.group_id)?; @@ -329,7 +326,7 @@ fn set_client_config>( value: V, ) -> Result<()> { if client_config.get(key).is_some() { - return Err(format!("Provided rdkafka_option that will be overwritten: {key}").into()); + return Err(Error::OptionOverwritten(key.to_string()).into()); } client_config.set(key, value); Ok(()) @@ -562,7 +559,7 @@ impl KafkaConsumerSource { #[async_trait::async_trait()] impl Source for KafkaConsumerSource { - async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { if let Some(consumer_task) = self.consumer_task.take() { self.cached_assignment.take(); // clear out references to the consumer // drop the consumer @@ -629,7 +626,7 @@ impl Source for KafkaConsumerSource { } Ok(None) => { // receive error - bail out - Err(empty_error()) + Err(empty_e().into()) } Ok(Some(KafkaError::Global(RDKafkaErrorCode::NoError))) => { debug!("{ctx} Consumer connected."); @@ -639,15 +636,24 @@ impl Source for KafkaConsumerSource { } } - async fn pull_data(&mut self, pull_id: &mut u64, _ctx: &SourceContext) -> Result { - let (reply, custom_pull_id) = self.source_rx.recv().await.ok_or_else(empty_error)?; + async fn pull_data( + &mut self, + pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + let (reply, custom_pull_id) = self.source_rx.recv().await.ok_or_else(empty_e)?; if let Some(custom_pull_id) = custom_pull_id { *pull_id = custom_pull_id; } Ok(reply) } - async fn ack(&mut self, stream_id: u64, pull_id: u64, ctx: &SourceContext) -> Result<()> { + async fn ack( + &mut self, + stream_id: u64, + pull_id: u64, + ctx: &SourceContext, + ) -> anyhow::Result<()> { debug!("{ctx} ACK {stream_id} {pull_id}"); if let Some(offsets) = self.offsets.as_mut() { if let Some(consumer) = self.consumer.as_ref() { @@ -697,7 +703,12 @@ impl Source for KafkaConsumerSource { Ok(()) } - async fn fail(&mut self, stream_id: u64, pull_id: u64, ctx: &SourceContext) -> Result<()> { + async fn fail( + &mut self, + stream_id: u64, + pull_id: u64, + ctx: &SourceContext, + ) -> anyhow::Result<()> { debug!("{ctx} FAIL {stream_id} {pull_id}"); // how can we make sure we do not conflict with the store_offset handling in `ack`? if let KafkaConsumerSource { @@ -741,14 +752,14 @@ impl Source for KafkaConsumerSource { Ok(()) } - async fn on_cb_trigger(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_trigger(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { let assignment = self.get_assignment()?; if let Some(consumer) = self.consumer.as_ref() { consumer.pause(&assignment)?; } Ok(()) } - async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { let assignment = self.get_assignment()?; if let Some(consumer) = self.consumer.as_mut() { consumer.resume(&assignment)?; @@ -756,7 +767,7 @@ impl Source for KafkaConsumerSource { Ok(()) } - async fn on_pause(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_pause(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { let assignment = self.get_assignment()?; if let Some(consumer) = self.consumer.as_mut() { consumer.pause(&assignment)?; @@ -764,7 +775,7 @@ impl Source for KafkaConsumerSource { Ok(()) } - async fn on_resume(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_resume(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { let assignment = self.get_assignment()?; if let Some(consumer) = self.consumer.as_mut() { consumer.resume(&assignment)?; @@ -772,7 +783,7 @@ impl Source for KafkaConsumerSource { Ok(()) } - async fn on_stop(&mut self, ctx: &SourceContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SourceContext) -> anyhow::Result<()> { // free references, see: https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#high-level-kafkaconsumer self.cached_assignment.take(); diff --git a/src/connectors/impls/kafka/producer.rs b/src/connectors/impls/kafka/producer.rs index ee995c6e64..111a29deab 100644 --- a/src/connectors/impls/kafka/producer.rs +++ b/src/connectors/impls/kafka/producer.rs @@ -17,8 +17,8 @@ use std::time::Duration; +use crate::channel::empty_e; use crate::connectors::prelude::*; -use crate::errors::empty_error; use crate::{ connectors::impls::kafka::{is_fatal_error, TremorRDKafkaContext, KAFKA_CONNECT_TIMEOUT}, utils::task_id, @@ -158,6 +158,12 @@ struct KafkaProducerSink { metrics_rx: Option>, } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("The producer {0} is not available")] + ProducerNotAvailable(String), +} + impl KafkaProducerSink { fn new(config: Config, producer_config: ClientConfig, reply_tx: ReplySender) -> Self { Self { @@ -179,11 +185,11 @@ impl Sink for KafkaProducerSink { ctx: &SinkContext, serializer: &mut EventSerializer, start: u64, - ) -> Result { + ) -> anyhow::Result { let producer = self .producer .as_ref() - .ok_or_else(|| ErrorKind::ProducerNotAvailable(ctx.alias().to_string()))?; + .ok_or_else(|| Error::ProducerNotAvailable(ctx.alias().to_string()))?; let transactional = event.transactional; let mut delivery_futures: Vec = if transactional { Vec::with_capacity(event.len()) @@ -257,7 +263,7 @@ impl Sink for KafkaProducerSink { Ok(SinkReply::NONE) } - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { // enforce cleaning out the previous producer // We might lose some in flight messages if let Some(old_producer) = self.producer.take() { @@ -287,7 +293,7 @@ impl Sink for KafkaProducerSink { } Ok(None) => { // receive error - we cannot tell what happened, better error here to trigger a retry - Err(empty_error()) + Err(empty_e().into()) } Ok(Some(kafka_error)) => { // we received an error from rdkafka - fail it big time @@ -296,7 +302,7 @@ impl Sink for KafkaProducerSink { } } - async fn on_stop(&mut self, ctx: &SinkContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SinkContext) -> anyhow::Result<()> { if let Some(producer) = self.producer.take() { let wait_secs = Duration::from_secs(1); if producer.in_flight_count() > 0 { diff --git a/src/connectors/impls/kv.rs b/src/connectors/impls/kv.rs index 4ba07b71f2..c6518d41af 100644 --- a/src/connectors/impls/kv.rs +++ b/src/connectors/impls/kv.rs @@ -115,22 +115,29 @@ //! in the intervening time since we last read from the store for this key. //! ::: -use crate::{ - channel::{bounded, Receiver, Sender}, - errors::already_created_error, -}; +use crate::channel::{bounded, Receiver, Sender}; use tremor_codec::{ json::{Json, Sorted}, Codec, }; -use crate::{connectors::prelude::*, errors::err_connector_def}; +use crate::connectors::prelude::*; use serde::Deserialize; use sled::{CompareAndSwapError, Db, IVec}; use std::path::PathBuf; use std::sync::Arc; use std::{boxed::Box, convert::TryFrom, sync::atomic::AtomicBool}; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("CAS error: expected {0} but found {1}.")] + Cas(Value<'static>, Value<'static>), + #[error("Missing `$kv` field for commands")] + MissingMeta, + #[error("Invalid command: {0}")] + InvalidCommand(String), +} + #[derive(Debug)] enum Command<'v> { /// Format: @@ -196,7 +203,7 @@ impl<'v> TryFrom<&'v Value<'v>> for Command<'v> { type Error = crate::Error; fn try_from(v: &'v Value<'v>) -> Result { - let v = v.get("kv").ok_or("Missing `$kv` field for commands")?; + let v = v.get("kv").ok_or(Error::MissingMeta)?; if let Some(key) = v.get_bytes("get").map(<[u8]>::to_vec) { Ok(Command::Get { key }) } else if let Some(key) = v.get_bytes("put").map(<[u8]>::to_vec) { @@ -216,7 +223,7 @@ impl<'v> TryFrom<&'v Value<'v>> for Command<'v> { end: v.get_bytes("end").map(<[u8]>::to_vec), }) } else { - Err(format!("Invalid KV command: {v}").into()) + Err(Error::InvalidCommand(v.to_string()).into()) } } } @@ -291,7 +298,7 @@ impl ConnectorBuilder for Builder { ) -> Result> { let config: Config = Config::new(config)?; if !PathBuf::from(&config.path).is_dir() { - return Err(err_connector_def(id, Builder::INVALID_DIR)); + return Err(error_connector_def(id, Builder::INVALID_DIR).into()); } let (tx, rx) = bounded(qsize()); @@ -323,7 +330,9 @@ impl Connector for Kv { ) -> Result> { let source = ChannelSource::from_channel( self.tx.clone(), - self.rx.take().ok_or_else(already_created_error)?, + self.rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, self.source_is_connected.clone(), ); Ok(Some(builder.spawn(source, ctx))) @@ -432,8 +441,7 @@ impl KvSink { if let Err(CompareAndSwapError { current, proposed }) = self.db.compare_and_swap(&key, old, Some(vec))? { - Err(format!( - "CAS error: expected {} but found {}.", + Err(Error::Cas( self.decode(proposed, ingest_ns).await?, self.decode(current, ingest_ns).await?, ) @@ -473,7 +481,7 @@ impl Sink for KvSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { let ingest_ns = tremor_common::time::nanotime(); let send_replies = self.source_is_connected.load(Ordering::Acquire); diff --git a/src/connectors/impls/metrics.rs b/src/connectors/impls/metrics.rs index 5d4bb49197..bd2d7a5f70 100644 --- a/src/connectors/impls/metrics.rs +++ b/src/connectors/impls/metrics.rs @@ -160,6 +160,12 @@ const TAGS: Cow<'static, str> = Cow::const_str("tags"); const FIELDS: Cow<'static, str> = Cow::const_str("fields"); const TIMESTAMP: Cow<'static, str> = Cow::const_str("timestamp"); +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Invalid metrics data")] + InvalidMetricsData, +} + /// This is a system connector to collect and forward metrics. /// System metrics are fed to this connector and can be received by binding this connector's `out` port to a pipeline to handle metrics events. /// It can also be used to send custom metrics and have them handled the same way as system metrics. @@ -206,7 +212,7 @@ impl Connector for MetricsConnector { ctx: SourceContext, builder: SourceManagerBuilder, ) -> Result> { - let source = MetricsSource::new(ctx.app_ctx.metrics.rx()); + let source = MetricsSource::new(ctx.app_ctx().metrics.rx()); info!("{ctx} Metrics connector id: {}", source.rx.id()); Ok(Some(builder.spawn(source, ctx))) } @@ -245,7 +251,11 @@ impl MetricsSource { #[async_trait::async_trait()] impl Source for MetricsSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { loop { match self.rx.recv().await { Ok(msg) => { @@ -313,7 +323,8 @@ pub(crate) fn verify_metrics_value(value: &Value<'_>) -> Result<()> { None } }) - .ok_or_else(|| ErrorKind::InvalidMetricsData.into()) + .ok_or(Error::InvalidMetricsData)?; + Ok(()) } /// passing events through to the source channel @@ -331,7 +342,7 @@ impl Sink for MetricsSink { _ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { // verify event format for (value, _meta) in event.value_meta_iter() { verify_metrics_value(value)?; diff --git a/src/connectors/impls/metronome.rs b/src/connectors/impls/metronome.rs index e2868e98f3..f6e9d4c738 100644 --- a/src/connectors/impls/metronome.rs +++ b/src/connectors/impls/metronome.rs @@ -185,11 +185,15 @@ impl MetronomeSource { #[async_trait::async_trait()] impl Source for MetronomeSource { - async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { self.next = nanotime() + self.interval_ns; Ok(true) } - async fn pull_data(&mut self, pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { let now = nanotime(); // we need to wait here before we continue to fulfill the interval conditions if now < self.next { @@ -238,10 +242,7 @@ mod tests { reconnect: Reconnect::None, metrics_interval_s: Some(5), }; - assert!(matches!( - builder.build(&alias, &connector_config,).await.err(), - Some(Error(ErrorKind::MissingConfiguration(_), _)) - )); + assert!(builder.build(&alias, &connector_config,).await.is_err(),); Ok(()) } } diff --git a/src/connectors/impls/null.rs b/src/connectors/impls/null.rs index 6953c04302..c594c9b44b 100644 --- a/src/connectors/impls/null.rs +++ b/src/connectors/impls/null.rs @@ -134,7 +134,11 @@ impl Connector for Null { struct NullSource {} #[async_trait::async_trait] impl Source for NullSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { Ok(SourceReply::Finished) } @@ -157,7 +161,7 @@ impl Sink for NullSink { _ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { Ok(SinkReply::NONE) } diff --git a/src/connectors/impls/oneshot.rs b/src/connectors/impls/oneshot.rs index c54eb4a282..95c61867f0 100644 --- a/src/connectors/impls/oneshot.rs +++ b/src/connectors/impls/oneshot.rs @@ -70,7 +70,11 @@ struct OneshotSource { #[async_trait::async_trait] impl Source for OneshotSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { if let Some(value) = self.value.take() { Ok(SourceReply::Structured { origin_uri: EventOriginUri { diff --git a/src/connectors/impls/otel/client.rs b/src/connectors/impls/otel/client.rs index b4b0ab3187..615f2ea246 100644 --- a/src/connectors/impls/otel/client.rs +++ b/src/connectors/impls/otel/client.rs @@ -86,9 +86,9 @@ impl ConnectorBuilder for Builder { #[derive(Clone)] pub(crate) struct RemoteOpenTelemetryEndpoint { - logs_client: LogsServiceClient, - metrics_client: MetricsServiceClient, - trace_client: TraceServiceClient, + logs: LogsServiceClient, + metrics: MetricsServiceClient, + traces: TraceServiceClient, } #[async_trait::async_trait] @@ -120,17 +120,14 @@ struct OtelSink { #[async_trait::async_trait()] impl Sink for OtelSink { - async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let endpoint = self.config.url.to_string(); - let channel = TonicEndpoint::from_shared(endpoint) - .map_err(|e| format!("Unable to connect to remote otel endpoint: {e}"))? - .connect() - .await?; + let channel = TonicEndpoint::from_shared(endpoint)?.connect().await?; self.remote = Some(RemoteOpenTelemetryEndpoint { - logs_client: LogsServiceClient::new(channel.clone()), - metrics_client: MetricsServiceClient::new(channel.clone()), - trace_client: TraceServiceClient::new(channel), + logs: LogsServiceClient::new(channel.clone()), + metrics: MetricsServiceClient::new(channel.clone()), + traces: TraceServiceClient::new(channel), }); Ok(true) @@ -142,7 +139,7 @@ impl Sink for OtelSink { ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { if let Some(remote) = &mut self.remote { for value in event.value_iter() { let err = if self.config.metrics && value.contains_key("metrics") { @@ -152,7 +149,7 @@ impl Sink for OtelSink { "Error converting payload to otel metrics", )?, }; - remote.metrics_client.export(request).await.err() + remote.metrics.export(request).await.err() } else if self.config.logs && value.contains_key("logs") { let request = ExportLogsServiceRequest { resource_logs: ctx.bail_err( @@ -160,7 +157,7 @@ impl Sink for OtelSink { "Error converting payload to otel logs", )?, }; - remote.logs_client.export(request).await.err() + remote.logs.export(request).await.err() } else if self.config.trace && value.contains_key("trace") { let request = ExportTraceServiceRequest { resource_spans: ctx.bail_err( @@ -168,7 +165,7 @@ impl Sink for OtelSink { "Error converting payload to otel span", )?, }; - remote.trace_client.export(request).await.err() + remote.traces.export(request).await.err() } else { warn!("{ctx} Invalid or disabled otel payload: {value}"); None @@ -196,7 +193,7 @@ impl Sink for OtelSink { _signal: Event, _ctx: &SinkContext, _serializer: &mut EventSerializer, - ) -> Result { + ) -> anyhow::Result { Ok(SinkReply::default()) } diff --git a/src/connectors/impls/otel/common.rs b/src/connectors/impls/otel/common.rs index 28166d2d90..1c29b1cbb2 100644 --- a/src/connectors/impls/otel/common.rs +++ b/src/connectors/impls/otel/common.rs @@ -16,10 +16,25 @@ use crate::connectors::prelude::*; use crate::connectors::utils::pb; +use anyhow::Context; use tremor_otelapis::opentelemetry::proto::common::v1::{ any_value, AnyValue, ArrayValue, InstrumentationLibrary, KeyValue, KeyValueList, StringKeyValue, }; +#[derive(Debug, Clone, thiserror::Error)] +pub(crate) enum Error { + #[error("missing field `{0}`")] + MissingField(&'static str), + #[error("Invalid `{0}` id ( wrong array element ) - values must be between 0 and 255 - cannot convert to pb")] + InvalidArrayContent(String), + #[error("Cannot convert json value to otel pb `{0}` id")] + FaildToConvert(String), + #[error("Invalid json to mapping for `{0}`")] + InvalidMapping(&'static str), + #[error("Invalid `{0}` id ( wrong array length ) - cannot convert to pb")] + InvalidLength(String), +} + pub(crate) struct OtelDefaults; impl Defaults for OtelDefaults { // We do add the port here since it's different from http's default @@ -118,8 +133,8 @@ pub(crate) fn string_key_value_to_json(pb: Vec) -> Value<'static } pub(crate) fn string_key_value_to_pb(data: Option<&Value<'_>>) -> Result> { - data.as_object() - .ok_or("Unable to map json to Vec pb")? + data.try_as_object() + .context("Unable to map json to Vec pb")? .iter() .map(|(key, value)| { let key = key.to_string(); @@ -137,14 +152,14 @@ pub(crate) fn key_value_list_to_json(pb: Vec) -> Value<'static> { pub(crate) fn maybe_key_value_list_to_pb(data: Option<&Value<'_>>) -> Result> { let obj = data - .as_object() - .ok_or("Expected a json object, found otherwise - cannot map to pb")?; + .try_as_object() + .context("Expected a json object, found otherwise - cannot map to pb")?; Ok(obj_key_value_list_to_pb(obj)) } pub(crate) fn get_attributes_or_labes(data: &Value) -> Result> { match (data.get_object("attributes"), data.get_object("labels")) { - (None, None) => Err("missing field `attributes`".into()), + (None, None) => Err(Error::MissingField("attributes").into()), (Some(a), None) | (None, Some(a)) => Ok(obj_key_value_list_to_pb(a)), (Some(a), Some(l)) => { let mut a = obj_key_value_list_to_pb(a); diff --git a/src/connectors/impls/otel/id.rs b/src/connectors/impls/otel/id.rs index d2ccb530b3..d511475f48 100644 --- a/src/connectors/impls/otel/id.rs +++ b/src/connectors/impls/otel/id.rs @@ -14,6 +14,7 @@ #![allow(dead_code)] +use super::common::Error; use crate::connectors::prelude::*; use ::rand::Rng; use std::fmt::Write; @@ -92,16 +93,17 @@ fn hex_id_to_pb( } else if let Some(json) = data.and_then(Value::as_bytes) { json.to_vec() } else if let Some(arr) = data.as_array() { - arr.iter().map(Value::as_u8).collect::>>().ok_or_else(|| format!( - "Invalid {kind} id ( wrong array element ) - values must be between 0 and 255 - cannot convert to pb" - ))? + arr.iter() + .map(Value::as_u8) + .collect::>>() + .ok_or_else(|| Error::InvalidArrayContent(kind.to_string()))? } else { - return Err(format!("Cannot convert json value to otel pb {kind} id").into()); + return Err(Error::FaildToConvert(kind.to_string()).into()); }; if (allow_empty && data.is_empty()) || data.len() == len_bytes { Ok(data) } else { - Err(format!("Invalid {kind} id ( wrong array length ) - cannot convert to pb",).into()) + Err(Error::InvalidLength(kind.to_string()).into()) } } @@ -156,7 +158,7 @@ pub mod test { let bytes = hex::decode(expected).unwrap_or_default(); let json = Value::Bytes(bytes.clone().into()); - let pb = hex_span_id_to_pb(Some(&json))?; + let pb = hex_span_id_to_pb(Some(&json)).expect("Failed to convert"); prop_assert_eq!(bytes.clone(), pb); } } @@ -169,7 +171,7 @@ pub mod test { let bytes = hex::decode(expected).unwrap_or_default(); let json = Value::Bytes(bytes.clone().into()); - let pb = hex_trace_id_to_pb(Some(&json))?; + let pb = hex_trace_id_to_pb(Some(&json)).expect("Failed to convert"); prop_assert_eq!(bytes.clone(), pb); } } diff --git a/src/connectors/impls/otel/logs.rs b/src/connectors/impls/otel/logs.rs index a3550cf2da..c94f581725 100644 --- a/src/connectors/impls/otel/logs.rs +++ b/src/connectors/impls/otel/logs.rs @@ -22,20 +22,28 @@ use super::{ use crate::connectors::utils::pb; use crate::errors::Result; +use anyhow::Context; use tremor_otelapis::opentelemetry::proto::{ collector::logs::v1::ExportLogsServiceRequest, logs::v1::{InstrumentationLibraryLogs, LogRecord, ResourceLogs}, }; use tremor_value::{literal, prelude::*, Value}; +#[derive(Debug, Clone, thiserror::Error)] +pub(crate) enum Error { + #[error("The `traceflags` is invalid, expected: 0b10000000, actual: {0}")] + InvalidTraceFlags(u32), + #[error("The `severity_number` is NOT in the valid range 0 <= {0} <= 24")] + InvalidSeverityNumber(i32), + #[error("Missing `logs` array")] + MissingLogs, +} + fn affirm_traceflags_valid(traceflags: u32) -> Result { if (traceflags == 128) || (traceflags == 0) { Ok(traceflags) } else { - Err( - format!("The `traceflags` is invalid, expected: 0b10000000, actual: {traceflags}",) - .into(), - ) + Err(Error::InvalidTraceFlags(traceflags).into()) } } @@ -44,10 +52,7 @@ fn affirm_severity_number_valid(severity_number: i32) -> Result { // NOTE `0` implies unspecified severity Ok(severity_number) } else { - Err( - format!("The `severity_number` is NOT in the valid range 0 <= {severity_number} <= 24") - .into(), - ) + Err(Error::InvalidSeverityNumber(severity_number).into()) } } @@ -94,8 +99,10 @@ pub(crate) fn log_record_to_pb(log: &Value<'_>) -> Result { // severity value is optional - default to 0 if not specified severity_number: pb::maybe_int_to_pbi32(log.get("severity_number")) .ok() - .and_then(|sn| affirm_severity_number_valid(sn).ok()) + .map(affirm_severity_number_valid) + .transpose()? .unwrap_or_default(), + // defined as optional - fallback to an empty string severity_text: pb::maybe_string_to_pb(log.get("severity_text")).unwrap_or_default(), // name is defined as optional - fallback to empty string @@ -116,8 +123,8 @@ pub(crate) fn log_record_to_pb(log: &Value<'_>) -> Result { pub(crate) fn maybe_instrumentation_library_logs_to_pb( data: Option<&Value<'_>>, ) -> Result> { - data.as_array() - .ok_or("Invalid json mapping for InstrumentationLibraryLogs")? + data.try_as_array() + .context("Invalid json mapping for InstrumentationLibraryLogs")? .iter() .filter_map(Value::as_object) .map(|ill| { @@ -166,7 +173,7 @@ pub(crate) fn resource_logs_to_json(request: ExportLogsServiceRequest) -> Result pub(crate) fn resource_logs_to_pb(json: &Value<'_>) -> Result> { json.get_array("logs") - .ok_or("Missing `logs` array")? + .ok_or(Error::MissingLogs)? .iter() .filter_map(Value::as_object) .map(|data| { @@ -406,12 +413,12 @@ mod tests { ] }); assert_eq!( - Ok(vec![ResourceLogs { + Some(vec![ResourceLogs { instrumentation_library_logs: vec![], resource: None, schema_url: String::new() }]), - resource_logs_to_pb(&log) + resource_logs_to_pb(&log).ok() ); } @@ -424,12 +431,12 @@ mod tests { } ]); assert_eq!( - Ok(vec![InstrumentationLibraryLogs { + Some(vec![InstrumentationLibraryLogs { instrumentation_library: None, logs: vec![], schema_url: String::new() }]), - maybe_instrumentation_library_logs_to_pb(Some(&ill)) + maybe_instrumentation_library_logs_to_pb(Some(&ill)).ok() ); } @@ -437,7 +444,7 @@ mod tests { fn minimal_log_record() { let lr = literal!({}); assert_eq!( - Ok(LogRecord { + Some(LogRecord { time_unix_nano: 0, severity_number: 0, severity_text: String::new(), @@ -449,7 +456,7 @@ mod tests { trace_id: vec![], span_id: vec![] }), - log_record_to_pb(&lr) + log_record_to_pb(&lr).ok() ); } } diff --git a/src/connectors/impls/otel/metrics.rs b/src/connectors/impls/otel/metrics.rs index b5d67d0a13..b225bc1abb 100644 --- a/src/connectors/impls/otel/metrics.rs +++ b/src/connectors/impls/otel/metrics.rs @@ -15,13 +15,15 @@ #![allow(dead_code)] use super::{ - common, id, + common::{self, Error}, + id, resource::{self, resource_to_pb}, }; use crate::{ connectors::utils::pb::{self, maybe_from_value}, errors::Result, }; +use anyhow::Context; use tremor_otelapis::opentelemetry::proto::{ collector::metrics::v1::ExportMetricsServiceRequest, metrics::v1::{ @@ -52,7 +54,7 @@ pub(crate) fn int_exemplars_to_json(data: Vec) -> Value<'static> { pub(crate) fn int_exemplars_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() - .ok_or("Unable to map json value to Exemplars pb")? + .ok_or(Error::InvalidMapping("IntExemplar"))? .iter() .map(|data| { Ok(IntExemplar { @@ -104,14 +106,14 @@ pub(crate) fn double_exemplars_to_json(data: Vec) -> Value<'static> { #[allow(deprecated)] // handling depricated fields is required by the PB files pub(crate) fn double_exemplars_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() - .ok_or("Unable to map json value to Exemplars pb")? + .ok_or(Error::InvalidMapping("Exemplar"))? .iter() .map(|data| { let filtered_attributes = match ( data.get_object("filtered_attributes"), data.get_object("filtered_labels"), ) { - (None, None) => return Err("missing field `filtered_attributes`".into()), + (None, None) => return Err(Error::MissingField("filtered_attributes").into()), (Some(a), None) | (None, Some(a)) => common::obj_key_value_list_to_pb(a), (Some(a), Some(l)) => { let mut a = common::obj_key_value_list_to_pb(a); @@ -144,7 +146,7 @@ pub(crate) fn quantile_values_to_json(data: Vec) -> Value<'stat pub(crate) fn quantile_values_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() - .ok_or("Unable to map json value to ValueAtQuantiles")? + .ok_or(Error::InvalidMapping("ValueAtQuantile"))? .iter() .map(|data| { let value = pb::maybe_double_to_pb(data.get("value"))?; @@ -170,7 +172,7 @@ pub(crate) fn int_data_points_to_json(pb: Vec) -> Value<'static> { pub(crate) fn int_data_points_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() - .ok_or("Unable to map json value to otel pb IntDataPoint list")? + .ok_or(Error::InvalidMapping("IntDataPoint"))? .iter() .map(|item| { Ok(IntDataPoint { @@ -220,7 +222,7 @@ pub(crate) fn double_data_points_to_json(pb: Vec) -> Value<'sta #[allow(deprecated)] // handling depricated fields is required by the PB files pub(crate) fn double_data_points_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() - .ok_or("Unable to map json value to otel pb NumberDataPoint list")? + .ok_or(Error::InvalidMapping("NumberDataPoint"))? .iter() .map(|data| { let attributes = common::get_attributes_or_labes(data)?; @@ -268,7 +270,7 @@ pub(crate) fn double_histo_data_points_to_pb( json: Option<&Value<'_>>, ) -> Result> { json.as_array() - .ok_or("Unable to map json value to otel pb HistogramDataPoint list")? + .ok_or(Error::InvalidMapping("HistogramDataPoint"))? .iter() .map(|data| { let attributes = common::get_attributes_or_labes(data)?; @@ -317,7 +319,7 @@ pub(crate) fn double_summary_data_points_to_pb( json: Option<&Value<'_>>, ) -> Result> { json.as_array() - .ok_or("Unable to map json value to otel pb SummaryDataPoint list")? + .ok_or(Error::InvalidMapping("SummaryDataPoint"))? .iter() .map(|data| { let attributes = common::get_attributes_or_labes(data)?; @@ -354,8 +356,8 @@ pub(crate) fn int_histo_data_points_to_json(pb: Vec) -> V pub(crate) fn int_histo_data_points_to_pb( json: Option<&Value<'_>>, ) -> Result> { - json.as_array() - .ok_or("Unable to map json value to otel pb IntHistogramDataPoint list")? + json.try_as_array() + .context(Error::InvalidMapping("IntHistogramDataPoint"))? .iter() .map(|item| { Ok(IntHistogramDataPoint { @@ -460,7 +462,7 @@ pub(crate) fn metrics_data_to_pb(data: &Value<'_>) -> Result { let data_points = double_summary_data_points_to_pb(json.get("data_points"))?; Ok(metric::Data::Summary(Summary { data_points })) } else { - Err("Invalid metric data point type - cannot convert to pb".into()) + Err(Error::InvalidMapping("metric::Data").into()) } } @@ -504,7 +506,7 @@ pub(crate) fn instrumentation_library_metrics_to_pb( ) -> Result> { let data = data .as_array() - .ok_or("Invalid json mapping for InstrumentationLibraryMetrics")?; + .ok_or(Error::InvalidMapping("InstrumentationLibraryMetrics"))?; let mut pb = Vec::with_capacity(data.len()); for data in data { let mut metrics = Vec::new(); @@ -550,7 +552,7 @@ pub(crate) fn resource_metrics_to_json(request: ExportMetricsServiceRequest) -> pub(crate) fn resource_metrics_to_pb(json: Option<&Value<'_>>) -> Result> { json.get_array("metrics") - .ok_or("Invalid json mapping for otel metrics message - cannot convert to pb")? + .ok_or(Error::InvalidMapping("ResourceMetrics"))? .iter() .filter_map(Value::as_object) .map(|item| { @@ -1268,12 +1270,12 @@ mod tests { ] }); assert_eq!( - Ok(vec![ResourceMetrics { + Some(vec![ResourceMetrics { resource: None, instrumentation_library_metrics: vec![], schema_url: "bla".to_string() }]), - resource_metrics_to_pb(Some(&rm)) + resource_metrics_to_pb(Some(&rm)).ok() ); } @@ -1284,12 +1286,12 @@ mod tests { "schema_url": "snot" }]); assert_eq!( - Ok(vec![InstrumentationLibraryMetrics { + Some(vec![InstrumentationLibraryMetrics { instrumentation_library: None, metrics: vec![], schema_url: "snot".to_string() }]), - instrumentation_library_metrics_to_pb(Some(&ilm)) + instrumentation_library_metrics_to_pb(Some(&ilm)).ok() ); } @@ -1302,13 +1304,13 @@ mod tests { // surprise: no data }); assert_eq!( - Ok(Metric { + Some(Metric { name: "badger".to_string(), description: "snot".to_string(), unit: "fahrenheit".to_string(), data: None }), - metric_to_pb(&metric) + metric_to_pb(&metric).ok() ); } } diff --git a/src/connectors/impls/otel/resource.rs b/src/connectors/impls/otel/resource.rs index 2ebf7c1c63..0384322318 100644 --- a/src/connectors/impls/otel/resource.rs +++ b/src/connectors/impls/otel/resource.rs @@ -19,6 +19,11 @@ use crate::connectors::prelude::*; use crate::connectors::utils::pb; use tremor_otelapis::opentelemetry::proto::resource::v1::Resource; +#[derive(Debug, Clone, thiserror::Error)] +pub(crate) enum Error { + #[error("Unable to map json value to Resource pb")] + InvalidResource, +} pub(crate) fn resource_to_json(pb: Resource) -> Value<'static> { literal!({ "attributes": common::key_value_list_to_json(pb.attributes), @@ -27,9 +32,7 @@ pub(crate) fn resource_to_json(pb: Resource) -> Value<'static> { } pub(crate) fn resource_to_pb(json: &Value<'_>) -> Result { - let json = json - .as_object() - .ok_or("Invalid json mapping for Resource")?; + let json = json.as_object().ok_or(Error::InvalidResource)?; Ok(Resource { dropped_attributes_count: pb::maybe_int_to_pbu32(json.get("dropped_attributes_count")) .unwrap_or_default(), diff --git a/src/connectors/impls/otel/server.rs b/src/connectors/impls/otel/server.rs index 39dff27e40..b58a33f729 100644 --- a/src/connectors/impls/otel/server.rs +++ b/src/connectors/impls/otel/server.rs @@ -15,7 +15,7 @@ use std::net::ToSocketAddrs; use super::{common::OtelDefaults, logs, metrics, trace}; -use crate::{connectors::prelude::*, errors::already_created_error}; +use crate::connectors::prelude::*; use async_std::channel::{bounded, Receiver, Sender}; use tokio::task::JoinHandle; use tremor_otelapis::all::{self, OpenTelemetryEvents}; @@ -92,6 +92,16 @@ impl ConnectorBuilder for Builder { } } +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Missing host for otel server")] + MissingHost, + #[error("Missing port for otel server")] + MissingPort, + #[error("Bad bind address")] + BadAddr, +} + #[async_trait::async_trait] impl Connector for Server { fn codec_requirements(&self) -> CodecReq { @@ -106,7 +116,10 @@ impl Connector for Server { let source = OtelSource { origin_uri: self.origin_uri.clone(), config: self.config.clone(), - rx: self.rx.take().ok_or_else(already_created_error)?, + rx: self + .rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, }; Ok(Some(builder.spawn(source, ctx))) } @@ -120,20 +133,12 @@ impl Connector for Server { } async fn connect(&mut self, ctx: &ConnectorContext, _attempt: &Attempt) -> Result { - let host = self - .config - .url - .host_str() - .ok_or("Missing host for otel server")?; - let port = self - .config - .url - .port() - .ok_or("Missing prot for otel server")?; + let host = self.config.url.host_str().ok_or(Error::MissingHost)?; + let port = self.config.url.port().ok_or(Error::MissingPort)?; let endpoint = format!("{host}:{port}") .to_socket_addrs()? .next() - .ok_or("badaddr")?; + .ok_or(Error::BadAddr)?; if let Some(previous_handle) = self.accept_task.take() { previous_handle.abort(); @@ -157,7 +162,11 @@ struct OtelSource { #[async_trait::async_trait()] impl Source for OtelSource { - async fn pull_data(&mut self, pull_id: &mut u64, ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result { let (data, remote) = match self.rx.recv().await? { OpenTelemetryEvents::Metrics(metrics, remote) if self.config.metrics => { (metrics::resource_metrics_to_json(metrics), remote) diff --git a/src/connectors/impls/otel/trace.rs b/src/connectors/impls/otel/trace.rs index 8f65c0a1b2..e99ced7e88 100644 --- a/src/connectors/impls/otel/trace.rs +++ b/src/connectors/impls/otel/trace.rs @@ -15,7 +15,7 @@ #![allow(dead_code)] use super::{ - common::{self, EMPTY}, + common::{self, Error, EMPTY}, id, resource::{self, resource_to_pb}, }; @@ -25,10 +25,7 @@ use crate::connectors::utils::pb::{ }; use tremor_otelapis::opentelemetry::proto::{ collector::trace::v1::ExportTraceServiceRequest, - trace::v1::{ - span::{Event, Link}, - InstrumentationLibrarySpans, ResourceSpans, Span, Status, - }, + trace::v1::{span, InstrumentationLibrarySpans, ResourceSpans, Span, Status}, }; #[allow(deprecated)] @@ -45,7 +42,7 @@ pub(crate) fn status_to_json<'event>(data: Option) -> Value<'event> { ) } -pub(crate) fn span_events_to_json(pb: Vec) -> Value<'static> { +pub(crate) fn span_events_to_json(pb: Vec) -> Value<'static> { pb.into_iter() .map(|data| { literal!({ @@ -58,12 +55,12 @@ pub(crate) fn span_events_to_json(pb: Vec) -> Value<'static> { .collect() } -pub(crate) fn span_events_to_pb(json: Option<&Value<'_>>) -> Result> { +pub(crate) fn span_events_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() .unwrap_or(&EMPTY) .iter() .map(|json| { - Ok(Event { + Ok(span::Event { name: maybe_string_to_pb(json.get("name"))?, time_unix_nano: maybe_int_to_pbu64(json.get("time_unix_nano"))?, attributes: common::maybe_key_value_list_to_pb(json.get("attributes")) @@ -75,7 +72,7 @@ pub(crate) fn span_events_to_pb(json: Option<&Value<'_>>) -> Result> .collect() } -pub(crate) fn span_links_to_json(pb: Vec) -> Value<'static> { +pub(crate) fn span_links_to_json(pb: Vec) -> Value<'static> { pb.into_iter() .map(|data| { literal!({ @@ -89,12 +86,12 @@ pub(crate) fn span_links_to_json(pb: Vec) -> Value<'static> { .collect() } -pub(crate) fn span_links_to_pb(json: Option<&Value<'_>>) -> Result> { +pub(crate) fn span_links_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() .unwrap_or(&EMPTY) .iter() .map(|json| { - Ok(Link { + Ok(span::Link { span_id: id::hex_span_id_to_pb(json.get("span_id"))?, trace_id: id::hex_trace_id_to_pb(json.get("trace_id"))?, trace_state: maybe_string_to_pb(json.get("trace_state"))?, @@ -109,9 +106,7 @@ pub(crate) fn status_to_pb(json: Option<&Value<'_>>) -> Result> { if json.is_none() { return Ok(None); } - let json = json - .as_object() - .ok_or("Unable to map json value to pb trace status")?; + let json = json.as_object().ok_or(Error::InvalidMapping("Status"))?; // This is generated code in the pb stub code deriving from otel proto files #[allow(deprecated)] @@ -188,7 +183,7 @@ pub(crate) fn instrumentation_library_spans_to_pb( data: Option<&Value<'_>>, ) -> Result> { data.as_array() - .ok_or("Invalid json mapping for InstrumentationLibrarySpans")? + .ok_or(Error::InvalidMapping("InstrumentationLibrarySpans"))? .iter() .filter_map(Value::as_object) .map(|data| { @@ -235,7 +230,7 @@ pub(crate) fn resource_spans_to_json(request: ExportTraceServiceRequest) -> Valu pub(crate) fn resource_spans_to_pb(json: Option<&Value<'_>>) -> Result> { json.get_array("trace") - .ok_or("Invalid json mapping for otel trace message - cannot convert to pb")? + .ok_or(Error::InvalidMapping("ResourceSpans"))? .iter() .filter_map(Value::as_object) .map(|json| { @@ -302,7 +297,7 @@ mod tests { #[test] fn span_event() -> Result<()> { - let pb = vec![Event { + let pb = vec![span::Event { time_unix_nano: 0, name: "badger".into(), attributes: vec![], @@ -323,7 +318,7 @@ mod tests { assert_eq!(pb, back_again); // Empty span events - let pb: Vec = vec![]; + let pb: Vec = vec![]; let json = span_events_to_json(vec![]); let back_again = span_events_to_pb(Some(&json))?; let expected: Value = literal!([]); @@ -341,7 +336,7 @@ mod tests { let trace_id_json = id::random_trace_id_value(nanotime); let trace_id_pb = id::hex_trace_id_to_pb(Some(&trace_id_json))?; - let pb = vec![Link { + let pb = vec![span::Link { attributes: vec![], dropped_attributes_count: 42, span_id: span_id_pb.clone(), @@ -555,12 +550,12 @@ mod tests { ] }); assert_eq!( - Ok(vec![ResourceSpans { + Some(vec![ResourceSpans { resource: None, instrumentation_library_spans: vec![], schema_url: "schema_url".to_string() }]), - resource_spans_to_pb(Some(&resource_spans)) + resource_spans_to_pb(Some(&resource_spans)).ok() ); } @@ -571,12 +566,12 @@ mod tests { "schema_url": "schema_url" }]); assert_eq!( - Ok(vec![InstrumentationLibrarySpans { + Some(vec![InstrumentationLibrarySpans { instrumentation_library: None, spans: vec![], schema_url: "schema_url".to_string() }]), - instrumentation_library_spans_to_pb(Some(&ils)) + instrumentation_library_spans_to_pb(Some(&ils)).ok() ); } @@ -596,7 +591,7 @@ mod tests { "dropped_attributes_count": 0 }); assert_eq!( - Ok(Span { + Some(Span { trace_id: b"aaaaaaaaaaaaaaaa".to_vec(), span_id: b"aaaaaaaa".to_vec(), trace_state: String::new(), @@ -613,7 +608,7 @@ mod tests { dropped_links_count: 0, status: None }), - span_to_pb(&span) + span_to_pb(&span).ok() ); } @@ -624,13 +619,13 @@ mod tests { "name": "urknall" }]); assert_eq!( - Ok(vec![Event { + Some(vec![span::Event { time_unix_nano: 1, name: "urknall".to_string(), attributes: vec![], dropped_attributes_count: 0 }]), - span_events_to_pb(Some(&span_events)) + span_events_to_pb(Some(&span_events)).ok() ); } } diff --git a/src/connectors/impls/s3.rs b/src/connectors/impls/s3.rs index dc81a9d495..d45a03efe8 100644 --- a/src/connectors/impls/s3.rs +++ b/src/connectors/impls/s3.rs @@ -150,3 +150,13 @@ mod auth; pub(crate) mod reader; pub(crate) mod streamer; + +#[derive(Clone, thiserror::Error, Debug)] +enum Error { + #[error("Source sender not initialized")] + NoSource, + #[error("No s3 client available")] + NoClient, + #[error("Failed to start upload for `{0}`")] + UploadStart(String), +} diff --git a/src/connectors/impls/s3/reader.rs b/src/connectors/impls/s3/reader.rs index a24ad405c7..24876fb17a 100644 --- a/src/connectors/impls/s3/reader.rs +++ b/src/connectors/impls/s3/reader.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. use super::auth; +use super::Error; use crate::connectors::prelude::*; +use anyhow::Context as ErrorContext; use aws_sdk_s3::{primitives::ByteStream, types::Object, Client as S3Client}; -use std::error::Error as StdError; use std::sync::Arc; use tokio::task::{self, JoinHandle}; @@ -146,17 +147,7 @@ impl Connector for S3Reader { .bucket(self.config.bucket.clone()) .send() .await - .map_err(|e| { - let msg = if let Some(err) = e.source() { - format!( - "Failed to access Bucket \"{}\": {err}.", - &self.config.bucket - ) - } else { - format!("Failed to access Bucket \"{}\".", &self.config.bucket) - }; - Error::from(ErrorKind::S3Error(msg)) - })?; + .with_context(|| format!("Failed to access Bucket `{}`.", &self.config.bucket))?; let (tx_key, rx_key) = async_channel::bounded(qsize()); @@ -166,10 +157,7 @@ impl Connector for S3Reader { let rx = rx_key.clone(); let bucket = self.config.bucket.clone(); - let tx = self - .tx - .clone() - .ok_or_else(|| ErrorKind::S3Error("source sender not initialized".to_string()))?; + let tx = self.tx.clone().ok_or(Error::NoSource)?; let origin_uri = EventOriginUri { scheme: URL_SCHEME.to_string(), host: hostname(), diff --git a/src/connectors/impls/s3/streamer.rs b/src/connectors/impls/s3/streamer.rs index 14978f92f8..2166f5c873 100644 --- a/src/connectors/impls/s3/streamer.rs +++ b/src/connectors/impls/s3/streamer.rs @@ -20,6 +20,7 @@ use crate::connectors::{ ObjectStorageSinkImpl, ObjectStorageUpload, YoloSink, }, }; +use anyhow::Context as ErrorContext; use aws_sdk_s3::{ types::{CompletedMultipartUpload, CompletedPart}, Client as S3Client, @@ -27,6 +28,8 @@ use aws_sdk_s3::{ use tremor_common::time::nanotime; use tremor_pipeline::{EventId, OpMeta}; +use super::Error; + pub(crate) const CONNECTOR_TYPE: &str = "s3_streamer"; const MORE_THEN_FIVEMBS: usize = 5 * 1024 * 1024 + 100; // Some extra bytes to keep aws happy. @@ -165,9 +168,7 @@ impl S3ObjectStorageSinkImpl { } fn get_client(&self) -> Result<&S3Client> { - self.client - .as_ref() - .ok_or_else(|| ErrorKind::S3Error("no s3 client available".to_string()).into()) + Ok(self.client.as_ref().ok_or(Error::NoClient)?) } } @@ -241,10 +242,7 @@ impl ObjectStorageSinkImpl for S3ObjectStorageSinkImpl { .bucket(bucket) .send() .await - .map_err(|e| { - let msg = format!("Failed to access Bucket `{bucket}`: {e}"); - Error::from(ErrorKind::S3Error(msg)) - })?; + .with_context(|| format!("Failed to access Bucket `{bucket}`"))?; Ok(true) } @@ -264,12 +262,9 @@ impl ObjectStorageSinkImpl for S3ObjectStorageSinkImpl { //let upload = CurrentUpload::new(resp.) - let upload_id = resp.upload_id.ok_or_else(|| { - ErrorKind::S3Error(format!( - "Failed to start upload for s3://{}: upload id not found in response.", - &object_id - )) - })?; + let upload_id = resp + .upload_id + .ok_or_else(|| Error::UploadStart(object_id.to_string()))?; let upload = S3Upload::new(object_id.clone(), upload_id, event); Ok(upload) diff --git a/src/connectors/impls/stdio.rs b/src/connectors/impls/stdio.rs index a330059385..6e163f3112 100644 --- a/src/connectors/impls/stdio.rs +++ b/src/connectors/impls/stdio.rs @@ -127,7 +127,11 @@ impl StdStreamSource { #[async_trait::async_trait()] impl Source for StdStreamSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { let reply = if self.done { SourceReply::Finished } else { @@ -184,6 +188,12 @@ impl StdStreamConnector { const REF_IN_PORTS: &'static [Port<'static>; 3] = &Self::IN_PORTS; } +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("{0} is not a valid port, use one of `in`, `stdout` or `stderr`")] + InvalidPort(String), +} + #[async_trait::async_trait()] impl Sink for StdStreamSink { async fn on_event( @@ -193,18 +203,14 @@ impl Sink for StdStreamSink { _ctx: &SinkContext, serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { for (value, meta) in event.value_meta_iter() { let data = serializer.serialize(value, meta, event.ingest_ns).await?; for chunk in data { match input { "in" | "stdout" => self.stdout.write_all(&chunk).await?, "stderr" => self.stderr.write_all(&chunk).await?, - _ => { - return Err( - "{} is not a valid port, use one of `in`, `stdout` or `stderr`".into(), - ) - } + port => return Err(Error::InvalidPort(port.into()).into()), } } } diff --git a/src/connectors/impls/tcp/client.rs b/src/connectors/impls/tcp/client.rs index fb27205e45..ee5494acda 100644 --- a/src/connectors/impls/tcp/client.rs +++ b/src/connectors/impls/tcp/client.rs @@ -19,19 +19,13 @@ #![allow(clippy::module_name_repetitions)] use super::TcpReader; -use crate::{ - channel::{bounded, Receiver, Sender}, - errors::already_created_error, -}; -use crate::{ - connectors::{ - prelude::*, - utils::{ - socket::{tcp_client_socket, TcpSocketOptions}, - tls::TLSClientConfig, - }, +use crate::channel::{bounded, Receiver, Sender}; +use crate::connectors::{ + prelude::*, + utils::{ + socket::{tcp_client_socket, Error, TcpSocketOptions}, + tls::TLSClientConfig, }, - errors::err_connector_def, }; use either::Either; use std::sync::{atomic::AtomicBool, Arc}; @@ -84,11 +78,11 @@ impl ConnectorBuilder for Builder { ) -> Result> { let config = Config::new(config)?; if config.url.port().is_none() { - return Err(err_connector_def(id, Self::MISSING_PORT)); + return Err(error_connector_def(id, Self::MISSING_PORT).into()); } let host = match config.url.host_str() { Some(host) => host.to_string(), - None => return Err(err_connector_def(id, Self::MISSING_HOST)), + None => return Err(error_connector_def(id, Self::MISSING_HOST).into()), }; let (tls_connector, tls_domain) = match config.tls.as_ref() { Some(Either::Right(true)) => { @@ -144,7 +138,9 @@ impl Connector for TcpClient { // this source is wired up to the ending channel that is forwarding data received from the TCP (or TLS) connection let source = ChannelSource::from_channel( self.source_tx.clone(), - self.source_rx.take().ok_or_else(already_created_error)?, + self.source_rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, // we don't need to know if the source is connected. Worst case if nothing is connected is that the receiving task is blocked. Arc::new(AtomicBool::new(false)), ); @@ -197,10 +193,7 @@ impl TcpClientSink { /// writing to the client socket async fn write(&mut self, data: Vec>) -> Result<()> { - let stream = self - .wrapped_stream - .as_mut() - .ok_or_else(|| Error::from(ErrorKind::NoSocket))?; + let stream = self.wrapped_stream.as_mut().ok_or(Error::NoSocket)?; for chunk in data { let slice: &[u8] = chunk.as_slice(); stream.write_all(slice).await?; @@ -211,7 +204,7 @@ impl TcpClientSink { #[async_trait::async_trait()] impl Sink for TcpClientSink { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let buf_size = self.config.buf_size; // connect TCP stream @@ -315,7 +308,7 @@ impl Sink for TcpClientSink { } /// when writing is done - async fn on_stop(&mut self, ctx: &SinkContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SinkContext) -> anyhow::Result<()> { if let Some(stream) = self.tcp_stream.as_mut() { if let Err(e) = stream.shutdown().await { error!("{ctx} stopping: {e}...",); diff --git a/src/connectors/impls/tcp/server.rs b/src/connectors/impls/tcp/server.rs index 944f0312f6..820e90c3ec 100644 --- a/src/connectors/impls/tcp/server.rs +++ b/src/connectors/impls/tcp/server.rs @@ -12,21 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. use super::{TcpDefaults, TcpReader, TcpWriter}; -use crate::{ - channel::{bounded, Receiver, Sender}, - errors::{already_created_error, empty_error}, -}; -use crate::{ - connectors::{ - prelude::*, - sink::channel_sink::ChannelSinkMsg, - utils::{ - socket::{tcp_server_socket, TcpSocketOptions}, - tls::TLSServerConfig, - ConnectionMeta, - }, +use crate::channel::{bounded, empty_e, Receiver, Sender}; +use crate::connectors::{ + prelude::*, + sink::channel_sink::ChannelSinkMsg, + utils::{ + socket::{tcp_server_socket, TcpSocketOptions}, + tls::TLSServerConfig, + ConnectionMeta, }, - errors::err_connector_def, }; use rustls::ServerConfig; use std::sync::{atomic::AtomicBool, Arc}; @@ -80,7 +74,7 @@ impl ConnectorBuilder for Builder { ) -> crate::errors::Result> { let config = Config::new(config)?; if config.url.port().is_none() { - return Err(err_connector_def(id, "Missing port for TCP server")); + return Err(error_connector_def(id, "Missing port for TCP server").into()); } let tls_server_config = if let Some(tls_config) = config.tls.as_ref() { Some(tls_config.to_server_config()?) @@ -137,7 +131,9 @@ impl Connector for TcpServer { resolve_connection_meta, builder.reply_tx(), self.sink_tx.clone(), - self.sink_rx.take().ok_or_else(already_created_error)?, + self.sink_rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, self.sink_is_connected.clone(), ); Ok(Some(builder.spawn(sink, ctx))) @@ -181,7 +177,7 @@ impl TcpServerSource { #[async_trait::async_trait()] impl Source for TcpServerSource { #[allow(clippy::too_many_lines)] - async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { let path = vec![self.config.url.port_or_dflt().to_string()]; let accept_ctx = ctx.clone(); let buf_size = self.config.buf_size; @@ -257,7 +253,7 @@ impl Source for TcpServerSource { let tls_reader = TcpReader::tls_server( tls_read_stream, vec![0; buf_size], - ctx.alias.clone(), + ctx.alias().clone(), origin_uri.clone(), meta, reader_runtime, @@ -293,7 +289,7 @@ impl Source for TcpServerSource { let tcp_reader = TcpReader::new( read_stream, vec![0; buf_size], - ctx.alias.clone(), + ctx.alias().clone(), origin_uri.clone(), meta, reader_runtime, @@ -316,11 +312,15 @@ impl Source for TcpServerSource { Ok(true) } - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { - Ok(self.connection_rx.recv().await.ok_or_else(empty_error)?) + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + Ok(self.connection_rx.recv().await.ok_or_else(empty_e)?) } - async fn on_stop(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_stop(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { if let Some(accept_task) = self.accept_task.take() { // stop acceptin' new connections accept_task.abort(); diff --git a/src/connectors/impls/udp/client.rs b/src/connectors/impls/udp/client.rs index 5b95afbbfa..f13161babf 100644 --- a/src/connectors/impls/udp/client.rs +++ b/src/connectors/impls/udp/client.rs @@ -17,7 +17,7 @@ use crate::connectors::{ prelude::*, - utils::socket::{udp_socket, UdpSocketOptions}, + utils::socket::{udp_socket, Error, UdpSocketOptions}, }; use tokio::net::UdpSocket; @@ -54,7 +54,7 @@ impl ConnectorBuilder for Builder { ) -> Result> { let config: Config = Config::new(config)?; if config.url.port().is_none() { - return Err("Missing port for UDP client".into()); + return Err(Error::MissingPort.into()); } Ok(Box::new(UdpClient { config })) @@ -104,7 +104,7 @@ impl UdpClientSink { #[async_trait::async_trait()] impl Sink for UdpClientSink { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let connect_addrs = tokio::net::lookup_host(( self.config.url.host_or_local(), self.config.url.port_or_dflt(), @@ -117,7 +117,7 @@ impl Sink for UdpClientSink { // chose default bind if unspecified by checking the first resolved connect addr let is_ipv4 = connect_addrs .first() - .ok_or_else(|| format!("unable to resolve {}", self.config.url))? + .ok_or_else(|| Error::UnableToConnect(self.config.url.to_string()))? .is_ipv4(); if is_ipv4 { Url::parse(super::UDP_IPV4_UNSPECIFIED)? @@ -143,10 +143,7 @@ impl Sink for UdpClientSink { serializer: &mut EventSerializer, _start: u64, ) -> Result { - let socket = self - .socket - .as_ref() - .ok_or_else(|| Error::from(ErrorKind::NoSocket))?; + let socket = self.socket.as_ref().ok_or(Error::NoSocket)?; for (value, meta) in event.value_meta_iter() { let data = serializer.serialize(value, meta, event.ingest_ns).await?; if let Err(e) = Self::send_event(socket, data).await { diff --git a/src/connectors/impls/udp/server.rs b/src/connectors/impls/udp/server.rs index 011d789bf8..0c53c7fb21 100644 --- a/src/connectors/impls/udp/server.rs +++ b/src/connectors/impls/udp/server.rs @@ -15,7 +15,7 @@ //! The UDP server will close the udp spcket on stop use crate::connectors::{ prelude::*, - utils::socket::{udp_socket, UdpSocketOptions}, + utils::socket::{udp_socket, Error, UdpSocketOptions}, }; use tokio::net::UdpSocket; @@ -103,17 +103,18 @@ impl UdpServerSource { #[async_trait::async_trait] impl Source for UdpServerSource { - async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { let listener = udp_socket(&self.config.url, &self.config.socket_options).await?; self.listener = Some(listener); Ok(true) } - async fn pull_data(&mut self, _pull_id: &mut u64, ctx: &SourceContext) -> Result { - let socket = self - .listener - .as_ref() - .ok_or_else(|| Error::from(ErrorKind::NoSocket))?; + async fn pull_data( + &mut self, + _pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result { + let socket = self.listener.as_ref().ok_or(Error::NoSocket)?; match socket.recv(&mut self.buffer).await { Ok(bytes_read) => { if bytes_read == 0 { diff --git a/src/connectors/impls/unix_socket/client.rs b/src/connectors/impls/unix_socket/client.rs index 4b505a2ea1..f683b57e3f 100644 --- a/src/connectors/impls/unix_socket/client.rs +++ b/src/connectors/impls/unix_socket/client.rs @@ -15,8 +15,8 @@ use super::UnixSocketReader; use crate::channel::{bounded, Receiver, Sender}; use crate::{ - connectors::prelude::*, - errors::{Kind as ErrorKind, Result}, + connectors::{prelude::*, utils::socket::Error}, + errors::Result, }; use std::{path::PathBuf, sync::Arc}; use tokio::{ @@ -82,7 +82,7 @@ impl Connector for Client { self.source_tx.clone(), self.source_rx .take() - .ok_or_else(crate::errors::already_created_error)?, + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, Arc::default(), // we don't need to know if the source is connected. Worst case if nothing is connected is that the receiving task is blocked. ); Ok(Some(builder.spawn(source, ctx))) @@ -115,10 +115,7 @@ impl UnixSocketSink { } async fn write(&mut self, data: Vec>) -> Result<()> { - let stream = self - .writer - .as_mut() - .ok_or_else(|| Error::from(ErrorKind::NoSocket))?; + let stream = self.writer.as_mut().ok_or(Error::NoSocket)?; for chunk in data { let slice: &[u8] = chunk.as_slice(); stream.write_all(slice).await?; @@ -131,7 +128,7 @@ impl UnixSocketSink { #[async_trait::async_trait()] impl Sink for UnixSocketSink { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let path = PathBuf::from(&self.config.path); if !path.exists() { return Err(std::io::Error::new( diff --git a/src/connectors/impls/unix_socket/server.rs b/src/connectors/impls/unix_socket/server.rs index 1f1cba55d0..bab1812b51 100644 --- a/src/connectors/impls/unix_socket/server.rs +++ b/src/connectors/impls/unix_socket/server.rs @@ -29,12 +29,9 @@ //! //! We try to route the event to the connection with `stream_id` `123`. use super::{UnixSocketReader, UnixSocketWriter}; +use crate::channel::{bounded, empty_e, Receiver, Sender}; +use crate::connectors::prelude::*; use crate::connectors::sink::channel_sink::ChannelSinkMsg; -use crate::{ - channel::{bounded, Receiver, Sender}, - errors::empty_error, -}; -use crate::{connectors::prelude::*, errors::already_created_error}; use std::sync::Arc; use std::{path::PathBuf, sync::atomic::AtomicBool}; use tokio::{io::split, net::UnixListener, task::JoinHandle, time::timeout}; @@ -133,7 +130,9 @@ impl Connector for UnixSocketServer { resolve_connection_meta, builder.reply_tx(), self.sink_tx.clone(), - self.sink_rx.take().ok_or_else(already_created_error)?, + self.sink_rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, self.sink_is_connected.clone(), ); Ok(Some(builder.spawn(sink, ctx))) @@ -170,7 +169,7 @@ impl UnixSocketSource { #[async_trait::async_trait()] impl Source for UnixSocketSource { - async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { if let Some(listener_task) = self.listener_task.take() { listener_task.abort(); } @@ -255,11 +254,15 @@ impl Source for UnixSocketSource { })); Ok(true) } - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { - Ok(self.connection_rx.recv().await.ok_or_else(empty_error)?) + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + Ok(self.connection_rx.recv().await.ok_or_else(empty_e)?) } - async fn on_stop(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_stop(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { if let Some(listener_task) = self.listener_task.take() { // stop acceptin' new connections listener_task.abort(); diff --git a/src/connectors/impls/wal.rs b/src/connectors/impls/wal.rs index fb677deb00..b9a599446c 100644 --- a/src/connectors/impls/wal.rs +++ b/src/connectors/impls/wal.rs @@ -112,7 +112,11 @@ impl qwal::Entry for Payload { #[async_trait::async_trait] impl Source for WalSource { - async fn pull_data(&mut self, pull_id: &mut u64, _ctx: &SourceContext) -> Result { + async fn pull_data( + &mut self, + pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { // This is a busy loop until we get data to avoid hogging the cpu // TODO: improve this by adding notifyer on write loop { @@ -130,12 +134,22 @@ impl Source for WalSource { } } - async fn ack(&mut self, _stream_id: u64, pull_id: u64, _ctx: &SourceContext) -> Result<()> { + async fn ack( + &mut self, + _stream_id: u64, + pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { self.wal.lock().await.ack(pull_id).await?; Ok(()) } - async fn fail(&mut self, _stream_id: u64, _pull_id: u64, _ctx: &SourceContext) -> Result<()> { + async fn fail( + &mut self, + _stream_id: u64, + _pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { self.wal.lock().await.revert().await?; Ok(()) } @@ -166,7 +180,7 @@ impl Sink for WalSink { _ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { self.wal.lock().await.push(Payload(event)).await?; Ok(SinkReply::NONE) } diff --git a/src/connectors/impls/ws/client.rs b/src/connectors/impls/ws/client.rs index 4ab569d2f6..22d7000a43 100644 --- a/src/connectors/impls/ws/client.rs +++ b/src/connectors/impls/ws/client.rs @@ -16,16 +16,13 @@ #![allow(clippy::module_name_repetitions)] use super::{WsReader, WsWriter}; -use crate::{ - connectors::{ - prelude::*, - utils::{ - socket::{tcp_client_socket, TcpSocketOptions}, - tls::TLSClientConfig, - ConnectionMeta, - }, +use crate::connectors::{ + prelude::*, + utils::{ + socket::{tcp_client_socket, Error, TcpSocketOptions}, + tls::TLSClientConfig, + ConnectionMeta, }, - errors::{already_created_error, err_connector_def}, }; use either::Either; use futures::StreamExt; @@ -72,11 +69,11 @@ impl ConnectorBuilder for Builder { let host = config .url .host() - .ok_or_else(|| err_connector_def(id, Self::MISSING_HOST))? + .ok_or_else(|| error_connector_def(id, Self::MISSING_HOST))? .to_string(); // TODO: do we really need to make the port required when we have a default defined on the URL? if config.url.port().is_none() { - return Err(err_connector_def(id, Self::MISSING_PORT)); + return Err(error_connector_def(id, Self::MISSING_PORT).into()); }; let (tls_connector, tls_domain) = match config.tls.as_ref() { @@ -153,7 +150,7 @@ impl WsClientSink { #[async_trait::async_trait] impl Sink for WsClientSink { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { let tcp_stream = tcp_client_socket(&self.config.url, &self.config.socket_options).await?; let (local_addr, peer_addr) = (tcp_stream.local_addr()?, tcp_stream.peer_addr()?); @@ -223,10 +220,7 @@ impl Sink for WsClientSink { _start: u64, ) -> Result { let ingest_ns = event.ingest_ns; - let writer = self - .wrapped_stream - .as_mut() - .ok_or_else(|| Error::from(ErrorKind::NoSocket))?; + let writer = self.wrapped_stream.as_mut().ok_or(Error::NoSocket)?; for (value, meta) in event.value_meta_iter() { let data = serializer.serialize(value, meta, ingest_ns).await?; if let Err(e) = writer.write(data, Some(meta)).await { @@ -258,7 +252,9 @@ impl Connector for WsClient { // we don't need to know if the source is connected. Worst case if nothing is connected is that the receiving task is blocked. let source = ChannelSource::from_channel( self.source_tx.clone(), - self.source_rx.take().ok_or_else(already_created_error)?, + self.source_rx + .take() + .ok_or_else(|| ConnectorError::AlreadyCreated(ctx.alias().clone()))?, // we don't need to know if the source is connected. Worst case if nothing is connected is that the receiving task is blocked. Arc::new(AtomicBool::new(false)), ); diff --git a/src/connectors/impls/ws/server.rs b/src/connectors/impls/ws/server.rs index aadd9dd3ce..8ce4451eed 100644 --- a/src/connectors/impls/ws/server.rs +++ b/src/connectors/impls/ws/server.rs @@ -33,6 +33,16 @@ use tokio_tungstenite::accept_async; const URL_SCHEME: &str = "tremor-ws-server"; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Source runtime not initialized")] + NoSource, + #[error("Sink runtime not initialized")] + NoSink, + #[error("Invalid URL")] + InvalidUrl, +} + #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub(crate) struct Config { @@ -160,14 +170,8 @@ impl Connector for WsServer { // TODO: this can be simplified as the connect can be moved into the source let path = vec![self.config.url.port_or_dflt().to_string()]; - let source_runtime = self - .source_runtime - .clone() - .ok_or("Source runtime not initialized")?; - let sink_runtime = self - .sink_runtime - .clone() - .ok_or("sink runtime not initialized")?; + let source_runtime = self.source_runtime.clone().ok_or(Error::NoSource)?; + let sink_runtime = self.sink_runtime.clone().ok_or(Error::NoSink)?; // cancel last accept task if necessary, this will drop the previous listener if let Some(previous_handle) = self.accept_task.take() { @@ -184,7 +188,7 @@ impl Connector for WsServer { self.config .url .set_port(Some(port)) - .map_err(|()| "Invalid URL")?; + .map_err(|()| Error::InvalidUrl)?; } let listener = tcp_server_socket( &self.config.url, diff --git a/src/connectors/prelude.rs b/src/connectors/prelude.rs index dc32dca4ba..977cd82059 100644 --- a/src/connectors/prelude.rs +++ b/src/connectors/prelude.rs @@ -13,8 +13,9 @@ // limitations under the License. pub(crate) use crate::{ - channel::{bounded, Receiver, Sender}, + channel::{bounded, ChannelError, Receiver, Sender}, connectors::{ + error_connector_def, sink::{ channel_sink::{ChannelSink, ChannelSinkRuntime}, AsyncSinkReply, ContraflowData, EventSerializer, ReplySender, Sink, SinkAck, SinkAddr, @@ -27,9 +28,9 @@ pub(crate) use crate::{ spawn_task, utils::reconnect::Attempt, CodecReq, Connector, ConnectorBuilder, ConnectorContext, ConnectorType, Context, - StreamDone, StreamIdGen, ACCEPT_TIMEOUT, + Error as ConnectorError, StreamDone, StreamIdGen, ACCEPT_TIMEOUT, }, - errors::{err_connector_def, Error, Kind as ErrorKind, Result}, + errors::{ErrorKind, Result}, qsize, system::KillSwitch, utils::hostname, @@ -46,7 +47,7 @@ pub(crate) use tremor_common::{ }; pub(crate) use tremor_config::Impl; pub use tremor_config::NameWithConfig; -pub use tremor_pipeline::{CbAction, EventIdGenerator, EventOriginUri, DEFAULT_STREAM_ID}; +pub use tremor_pipeline::{CbAction, EventOriginUri, DEFAULT_STREAM_ID}; pub(crate) use tremor_script::prelude::*; /// default buf size used for reading from files and streams (sockets etc) /// diff --git a/src/connectors/sink.rs b/src/connectors/sink.rs index 784e524081..d78ceb255b 100644 --- a/src/connectors/sink.rs +++ b/src/connectors/sink.rs @@ -19,7 +19,7 @@ pub(crate) mod channel_sink; /// Utility for limiting concurrency (by sending `CB::Close` messages when a maximum concurrency value is reached) pub(crate) mod concurrency_cap; -use super::{utils::metrics::SinkReporter, ConnectionLostNotifier, Msg, QuiescenceBeacon}; +use super::{utils::metrics::SinkReporter, ConnectionLostNotifier, Error, Msg, QuiescenceBeacon}; use crate::{ channel::{bounded, unbounded, Receiver, Sender, UnboundedReceiver, UnboundedSender}, config::Connector as ConnectorConfig, @@ -143,14 +143,14 @@ pub(crate) trait Sink: Send { ctx: &SinkContext, serializer: &mut EventSerializer, start: u64, - ) -> Result; + ) -> anyhow::Result; /// called when receiving a signal async fn on_signal( &mut self, _signal: Event, _ctx: &SinkContext, _serializer: &mut EventSerializer, - ) -> Result { + ) -> anyhow::Result { Ok(SinkReply::default()) } @@ -179,7 +179,7 @@ pub(crate) trait Sink: Send { // lifecycle stuff /// called when started - async fn on_start(&mut self, _ctx: &SinkContext) -> Result<()> { + async fn on_start(&mut self, _ctx: &SinkContext) -> anyhow::Result<()> { Ok(()) } @@ -190,30 +190,30 @@ pub(crate) trait Sink: Send { /// The intended result of this function is to re-establish a connection. It might reuse a working connection. /// /// Return `Ok(true)` if the connection could be successfully established. - async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { Ok(true) } /// called when paused - async fn on_pause(&mut self, _ctx: &SinkContext) -> Result<()> { + async fn on_pause(&mut self, _ctx: &SinkContext) -> anyhow::Result<()> { Ok(()) } /// called when resumed - async fn on_resume(&mut self, _ctx: &SinkContext) -> Result<()> { + async fn on_resume(&mut self, _ctx: &SinkContext) -> anyhow::Result<()> { Ok(()) } /// called when stopped - async fn on_stop(&mut self, _ctx: &SinkContext) -> Result<()> { + async fn on_stop(&mut self, _ctx: &SinkContext) -> anyhow::Result<()> { Ok(()) } // connectivity stuff /// called when sink lost connectivity - async fn on_connection_lost(&mut self, _ctx: &SinkContext) -> Result<()> { + async fn on_connection_lost(&mut self, _ctx: &SinkContext) -> anyhow::Result<()> { Ok(()) } /// called when sink re-established connectivity - async fn on_connection_established(&mut self, _ctx: &SinkContext) -> Result<()> { + async fn on_connection_established(&mut self, _ctx: &SinkContext) -> anyhow::Result<()> { Ok(()) } @@ -251,23 +251,23 @@ pub(crate) trait SinkRuntime: Send + Sync { #[derive(Clone)] pub(crate) struct SinkContextInner { /// the connector unique identifier - pub(crate) uid: SinkUId, + uid: SinkUId, /// the connector alias - pub(crate) alias: alias::Connector, + alias: alias::Connector, /// the connector type - pub(crate) connector_type: ConnectorType, + connector_type: ConnectorType, /// check if we are paused or should stop reading/writing - pub(crate) quiescence_beacon: QuiescenceBeacon, + quiescence_beacon: QuiescenceBeacon, /// notifier the connector runtime if we lost a connection - pub(crate) notifier: ConnectionLostNotifier, + notifier: ConnectionLostNotifier, /// sender for raft requests /// Application Context - pub(crate) app_ctx: AppContext, + app_ctx: AppContext, - pub(crate) killswitch: KillSwitch, + killswitch: KillSwitch, } #[derive(Clone)] pub(crate) struct SinkContext(Arc); @@ -285,18 +285,15 @@ impl SinkContext { KillSwitch::dummy(), ) } + pub(crate) fn killswitch(&self) -> KillSwitch { self.0.killswitch.clone() } + pub(crate) fn uid(&self) -> SinkUId { self.0.uid } - pub(crate) fn notifier(&self) -> &ConnectionLostNotifier { - &self.0.notifier - } - pub(crate) fn app_ctx(&self) -> &AppContext { - &self.0.app_ctx - } + pub(crate) fn new( uid: SinkUId, alias: alias::Connector, @@ -343,7 +340,7 @@ impl Context for SinkContext { fn connector_type(&self) -> &ConnectorType { &self.0.connector_type } - fn raft(&self) -> &raft::Cluster { + fn raft(&self) -> &raft::ClusterInterface { &self.0.app_ctx.raft } fn app_ctx(&self) -> &AppContext { @@ -372,7 +369,7 @@ pub(crate) enum SinkMsg { pipelines: Vec<(DeployEndpoint, pipeline::Addr)>, }, /// Connect to the outside world and send the result back - Connect(Sender>, Attempt), + Connect(Sender>, Attempt), /// the connection to the outside world wasl ost ConnectionLost, /// connection established @@ -385,7 +382,7 @@ pub(crate) enum SinkMsg { /// resume the sink Resume, /// stop the sink - Stop(Sender>), + Stop(Sender>), /// drain this sink and notify the connector via the provided sender Drain(Sender), } @@ -503,21 +500,17 @@ impl EventSerializer { codec_config: Option, default_codec: CodecReq, postprocessor_configs: Vec, - connector_type: &ConnectorType, + _connector_type: &ConnectorType, alias: &alias::Connector, ) -> Result { let codec_config = match default_codec { CodecReq::Structured => { if codec_config.is_some() { - return Err(format!( - "The {connector_type} connector {alias} can not be configured with a codec.", - ) - .into()); + return Err(Error::UnsupportedCodec(alias.clone()).into()); } tremor_codec::Config::from("null") } - CodecReq::Required => codec_config - .ok_or_else(|| format!("Missing codec for {connector_type} connector {alias}"))?, + CodecReq::Required => codec_config.ok_or_else(|| Error::MissingCodec(alias.clone()))?, CodecReq::Optional(opt) => { codec_config.unwrap_or_else(|| tremor_codec::Config::from(opt)) } diff --git a/src/connectors/sink/channel_sink.rs b/src/connectors/sink/channel_sink.rs index b85efde41a..f6399fb3b3 100644 --- a/src/connectors/sink/channel_sink.rs +++ b/src/connectors/sink/channel_sink.rs @@ -14,7 +14,7 @@ //! Sink implementation that keeps track of multiple streams and keeps channels to send to each stream -use crate::channel::{bounded, Receiver, Sender}; +use crate::channel::{bounded, send_e, Receiver, Sender}; use crate::{ connectors::{prelude::*, Context, StreamDone}, errors::Result, @@ -257,7 +257,11 @@ where /// The writer should receive an error when the channel is empty, so we safely drain all messages. /// We will get a double `RemoveStream` message, but this is fine async fn unregister_stream_writer(&mut self, stream: u64) -> Result<()> { - Ok(self.tx.send(ChannelSinkMsg::RemoveStream(stream)).await?) + Ok(self + .tx + .send(ChannelSinkMsg::RemoveStream(stream)) + .await + .map_err(send_e)?) } } @@ -346,7 +350,8 @@ where } stream_sink_tx .send(ChannelSinkMsg::RemoveStream(stream)) - .await?; + .await + .map_err(send_e)?; Result::Ok(()) }) } @@ -378,7 +383,7 @@ where ctx: &SinkContext, serializer: &mut EventSerializer, start: u64, - ) -> Result { + ) -> anyhow::Result { // clean up // make sure channels for the given event are added to avoid stupid errors // due to channels not yet handled @@ -461,7 +466,7 @@ where signal: Event, ctx: &SinkContext, serializer: &mut EventSerializer, - ) -> Result { + ) -> anyhow::Result { match signal.kind.as_ref() { Some(SignalKind::Tick) => { self.handle_channels(ctx, serializer, true); diff --git a/src/connectors/source.rs b/src/connectors/source.rs index e60b58c75f..7eb2b15fb3 100644 --- a/src/connectors/source.rs +++ b/src/connectors/source.rs @@ -17,7 +17,7 @@ /// A simple source that is fed with `SourceReply` via a channel. pub mod channel_source; -use crate::errors::empty_error; +use crate::channel::empty_e; use crate::errors::Result; use crate::pipeline; use crate::pipeline::InputTarget; @@ -48,7 +48,7 @@ use tremor_pipeline::{ }; use tremor_script::{ast::DeployEndpoint, prelude::BaseExpr, EventPayload, ValueAndMeta}; -use super::utils::metrics::SourceReporter; +use super::{utils::metrics::SourceReporter, Error}; #[derive(Debug)] /// Messages a Source can receive @@ -63,7 +63,7 @@ pub(crate) enum SourceMsg { pipeline: (DeployEndpoint, pipeline::Addr), }, /// Connect to the outside world and send the result back - Connect(Sender>, Attempt), + Connect(Sender>, Attempt), /// connectivity is lost in the connector ConnectionLost, /// connectivity is re-established @@ -77,7 +77,7 @@ pub(crate) enum SourceMsg { /// resume the source Resume, /// stop the source - Stop(Sender>), + Stop(Sender>), /// drain the source - bears a sender for sending out a SourceDrained status notification Drain(Sender), #[cfg(test)] @@ -150,7 +150,11 @@ pub(crate) trait Source: Send { /// `pull_id` can be modified, but users need to beware that it needs to remain unique per event stream. The modified `pull_id` /// will be used in the `EventId` and will be passed backl into the `ack`/`fail` methods. This allows sources to encode /// information into the `pull_id` to keep track of internal state. - async fn pull_data(&mut self, pull_id: &mut u64, ctx: &SourceContext) -> Result; + async fn pull_data( + &mut self, + pull_id: &mut u64, + ctx: &SourceContext, + ) -> anyhow::Result; /// This callback is called when the data provided from /// pull_event did not create any events, this is needed for /// linked sources that require a 1:1 mapping between requests @@ -160,7 +164,7 @@ pub(crate) trait Source: Send { _pull_id: u64, _stream: u64, _ctx: &SourceContext, - ) -> Result<()> { + ) -> anyhow::Result<()> { Ok(()) } @@ -174,7 +178,7 @@ pub(crate) trait Source: Send { /////////////////////////// /// called when the source is started. This happens only once in the whole source lifecycle, before any other callbacks - async fn on_start(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_start(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } @@ -185,21 +189,21 @@ pub(crate) trait Source: Send { /// The intended result of this function is to re-establish a connection. It might reuse a working connection. /// /// Return `Ok(true)` if the connection could be successfully established. - async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, _ctx: &SourceContext, _attempt: &Attempt) -> anyhow::Result { Ok(true) } /// called when the source is explicitly paused as result of a user/operator interaction /// in contrast to `on_cb_trigger` which happens automatically depending on downstream pipeline or sink connector logic. - async fn on_pause(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_pause(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } /// called when the source is explicitly resumed from being paused - async fn on_resume(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_resume(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } /// called when the source is stopped. This happens only once in the whole source lifecycle, as the very last callback - async fn on_stop(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_stop(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } @@ -208,35 +212,45 @@ pub(crate) trait Source: Send { /// Expected reaction is to pause receiving messages, which is handled automatically by the runtime /// Source implementations might want to close connections or signal a pause to the upstream entity it connects to if not done in the connector (the default) // TODO: add info of Cb event origin (port, origin_uri)? - async fn on_cb_trigger(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_trigger(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } /// Called when we receive a `open` Circuit breaker event from any connected pipeline /// This means we can start/continue polling this source for messages /// Source implementations might want to start establishing connections if not done in the connector (the default) - async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } // guaranteed delivery callbacks /// an event has been acknowledged and can be considered delivered /// multiple acks for the same set of ids are always possible - async fn ack(&mut self, _stream_id: u64, _pull_id: u64, _ctx: &SourceContext) -> Result<()> { + async fn ack( + &mut self, + _stream_id: u64, + _pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { Ok(()) } /// an event has failed along its way and can be considered failed /// multiple fails for the same set of ids are always possible - async fn fail(&mut self, _stream_id: u64, _pull_id: u64, _ctx: &SourceContext) -> Result<()> { + async fn fail( + &mut self, + _stream_id: u64, + _pull_id: u64, + _ctx: &SourceContext, + ) -> anyhow::Result<()> { Ok(()) } // connectivity stuff /// called when connector lost connectivity - async fn on_connection_lost(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_connection_lost(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } /// called when connector re-established connectivity - async fn on_connection_established(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_connection_established(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { Ok(()) } @@ -274,26 +288,26 @@ pub(crate) trait StreamReader: Send { #[derive(Clone)] pub(crate) struct SourceContext { /// connector uid - pub(crate) uid: SourceUId, + uid: SourceUId, /// connector alias - pub(crate) alias: alias::Connector, + alias: alias::Connector, /// connector type - pub(crate) connector_type: ConnectorType, + connector_type: ConnectorType, /// The Quiescence Beacon - pub(crate) quiescence_beacon: QuiescenceBeacon, + quiescence_beacon: QuiescenceBeacon, /// tool to notify the connector when the connection is lost - pub(crate) notifier: ConnectionLostNotifier, + notifier: ConnectionLostNotifier, /// Application Context - pub(crate) app_ctx: AppContext, + app_ctx: AppContext, /// kill switch - pub(crate) killswitch: KillSwitch, + killswitch: KillSwitch, /// Precomputed prefix for logging - pub(crate) prefix: alias::SourceContext, + prefix: alias::SourceContext, } impl SourceContext { @@ -345,7 +359,7 @@ impl Context for SourceContext { fn connector_type(&self) -> &ConnectorType { &self.connector_type } - fn raft(&self) -> &raft::Cluster { + fn raft(&self) -> &raft::ClusterInterface { &self.app_ctx.raft } fn app_ctx(&self) -> &AppContext { @@ -427,23 +441,20 @@ pub(crate) fn builder( config: &ConnectorConfig, connector_default_codec: CodecReq, source_metrics_reporter: SourceReporter, + alias: &alias::Connector, ) -> Result { let preprocessor_configs = config.preprocessors.clone().unwrap_or_default(); let codec_config = match connector_default_codec { CodecReq::Structured => { if config.codec.is_some() { - return Err(format!( - "The connector {} can not be configured with a codec.", - config.connector_type - ) - .into()); + return Err(Error::UnsupportedCodec(alias.clone()).into()); } tremor_codec::Config::from("null") } CodecReq::Required => config .codec .clone() - .ok_or_else(|| format!("Missing codec for connector {}", config.connector_type))?, + .ok_or_else(|| Error::MissingCodec(alias.clone()))?, CodecReq::Optional(opt) => config .codec .clone() @@ -872,7 +883,12 @@ where &mut self.pipelines_err } else { error!("{} Tried to connect to invalid port: {}", &self.ctx, &port); - tx.send(Err("Connecting to invalid port".into())).await?; + tx.send(Err(ConnectorError::InvalidPort( + self.ctx.alias().clone(), + port.clone(), + ) + .into())) + .await?; return Ok(Control::Continue); }; // We can not move this to the system flow since we need to know about transactionality @@ -985,7 +1001,11 @@ where } /// handle data from the source - async fn handle_source_reply(&mut self, data: Result, pull_id: u64) -> Result<()> { + async fn handle_source_reply( + &mut self, + data: anyhow::Result, + pull_id: u64, + ) -> Result<()> { let data = match data { Ok(d) => d, Err(e) => { @@ -1262,7 +1282,7 @@ where // and handle it the same as rx let src_future = self.source.pull_data(&mut pull_id, &self.ctx); match futures::future::select(Box::pin(rx.recv()), src_future).await { - Either::Left((msg, _o)) => Either::Left(msg.ok_or_else(empty_error)?), + Either::Left((msg, _o)) => Either::Left(msg.ok_or_else(empty_e)?), Either::Right((data, _o)) => Either::Right(data), } }; diff --git a/src/connectors/source/channel_source.rs b/src/connectors/source/channel_source.rs index bf47faa2bf..b4ad9d889a 100644 --- a/src/connectors/source/channel_source.rs +++ b/src/connectors/source/channel_source.rs @@ -12,17 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - channel::{bounded, Receiver, Sender}, - errors::empty_error, - qsize, +use crate::connectors::{ + source::{Source, SourceContext, SourceReply, SourceReplySender, StreamDone, StreamReader}, + Context, }; use crate::{ - connectors::{ - source::{Source, SourceContext, SourceReply, SourceReplySender, StreamDone, StreamReader}, - Context, - }, - errors::Result, + channel::{bounded, empty_e, Receiver, Sender}, + qsize, }; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -74,8 +70,12 @@ impl ChannelSource { #[async_trait::async_trait()] impl Source for ChannelSource { - async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { - Ok(self.rx.recv().await.ok_or_else(empty_error)?) + async fn pull_data( + &mut self, + _pull_id: &mut u64, + _ctx: &SourceContext, + ) -> anyhow::Result { + Ok(self.rx.recv().await.ok_or_else(empty_e)?) } /// this source is not handling acks/fails @@ -87,7 +87,7 @@ impl Source for ChannelSource { true } - async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> Result<()> { + async fn on_cb_restore(&mut self, _ctx: &SourceContext) -> anyhow::Result<()> { // we will only know if we are connected to some pipelines if we receive a CBAction::Restore contraflow event // we will not send responses to out/err if we are not connected and this is determined by this variable self.is_connected.store(true, Ordering::Release); diff --git a/src/connectors/tests.rs b/src/connectors/tests.rs index ecc6693f36..d5219064d1 100644 --- a/src/connectors/tests.rs +++ b/src/connectors/tests.rs @@ -58,8 +58,8 @@ mod bench; use super::{prelude::KillSwitch, sink::SinkMsg}; use crate::{ + channel::empty_e, channel::{bounded, unbounded, Receiver, UnboundedReceiver}, - errors::empty_error, system::flow::AppContext, }; use crate::{ @@ -69,6 +69,7 @@ use crate::{ instance::State, pipeline, qsize, Event, }; +use anyhow::anyhow; use log::{debug, info}; use std::time::Duration; use std::{collections::HashMap, time::Instant}; @@ -182,8 +183,8 @@ impl ConnectorHarness { // start the connector let (tx, mut rx) = bounded(1); self.addr.start(tx).await?; - let cr = rx.recv().await.ok_or_else(empty_error)?; - cr.res?; + let cr = rx.recv().await.ok_or_else(empty_e)?; + cr?; // send a `CBAction::Restore` to the connector, so it starts pulling data self.send_to_source(SourceMsg::Cb(CbAction::Restore, EventId::default()))?; @@ -218,9 +219,9 @@ impl ConnectorHarness { debug!("Stopping harness..."); self.addr.stop(tx).await?; debug!("Waiting for stop result..."); - let cr = rx.recv().await.ok_or_else(empty_error)?; + let cr = rx.recv().await.ok_or_else(empty_e)?; debug!("Stop result received."); - cr.res?; + cr?; //self.handle.abort(); let out_events = self .pipes @@ -302,10 +303,9 @@ impl ConnectorHarness { T: Into>, { let port = port.into(); - Ok(self - .pipes + self.pipes .get_mut(&port) - .ok_or_else(|| format!("No pipeline connected to port {port}"))?) + .ok_or_else(|| anyhow!("No pipeline connected to port {port}")) } /// get the out pipeline - if any @@ -343,7 +343,6 @@ impl ConnectorHarness { // this is only used in integration tests, // otherwise this throws an error when compiled for non-integration tests - #[allow(dead_code)] pub(crate) async fn signal_tick_to_sink(&self) -> Result<()> { self.addr .send_sink(SinkMsg::Signal { @@ -424,7 +423,7 @@ impl TestPipeline { pub(crate) async fn get_contraflow(&mut self) -> Result { match timeout(Duration::from_secs(20), self.rx_cf.recv()) .await? - .ok_or("No contraflow")? + .ok_or_else(|| anyhow!("No contraflow"))? { pipeline::CfMsg::Insight(event) => Ok(event), } @@ -463,14 +462,14 @@ impl TestPipeline { } } Ok(None) => { - return Err(empty_error()); + return Err(empty_e().into()); } Err(_) => { - return Err(format!("Did not receive an event for {TIMEOUT:?}").into()); + return Err(anyhow!("Did not receive an event for {TIMEOUT:?}")); } } if start.elapsed() > TIMEOUT { - return Err(format!("Did not receive an event for {TIMEOUT:?}").into()); + return Err(anyhow!("Did not receive an event for {TIMEOUT:?}")); } } } @@ -485,13 +484,13 @@ impl TestPipeline { Ok(Some(msg)) => match *msg { pipeline::Msg::Signal(_signal) => (), pipeline::Msg::Event { event, .. } => { - return Err( - format!("Expected no event for {duration:?}, got: {event:?}").into(), - ); + return Err(anyhow!( + "Expected no event for {duration:?}, got: {event:?}" + )); } }, Ok(None) => { - return Err(empty_error()); + return Err(empty_e().into()); } } if start.elapsed() > duration { diff --git a/src/connectors/tests/bench.rs b/src/connectors/tests/bench.rs index 86ad57c64d..e9b293c45b 100644 --- a/src/connectors/tests/bench.rs +++ b/src/connectors/tests/bench.rs @@ -62,7 +62,7 @@ async fn stop_after_events() -> Result<()> { .send_sink(SinkMsg::Event { event, port: IN }) .await?; } - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }); // the bench connector should shut the world down diff --git a/src/connectors/tests/clickhouse/more_complex_test.rs b/src/connectors/tests/clickhouse/more_complex_test.rs index 70dd052e36..f06ae99299 100644 --- a/src/connectors/tests/clickhouse/more_complex_test.rs +++ b/src/connectors/tests/clickhouse/more_complex_test.rs @@ -44,28 +44,26 @@ // As the `DateTime64` encoding is not implemented in `clickhouse-rs`, the // columns `k`, `l`, `m` and `n` commented out everywhere in the file. -use std::{ - net::{Ipv4Addr, Ipv6Addr}, - time::{Duration, Instant}, +use super::utils; +use crate::{ + connectors::{ + impls::clickhouse, + tests::{free_port, ConnectorHarness}, + }, + errors::Result, }; - use chrono::{DateTime, NaiveDateTime, Utc}; use chrono_tz::Tz; use clickhouse_rs::Pool; +use std::{ + net::{Ipv4Addr, Ipv6Addr}, + time::{Duration, Instant}, +}; use testcontainers::{clients, core::Port, images::generic::GenericImage, RunnableImage}; use tremor_common::ports::IN; use tremor_pipeline::{CbAction, Event, EventId}; use tremor_value::literal; -use super::utils; -use crate::{ - connectors::{ - impls::clickhouse, - tests::{free_port, ConnectorHarness}, - }, - errors::{Error, Result}, -}; - macro_rules! assert_row_equals { ( $row:expr, @@ -317,9 +315,7 @@ async fn test() -> Result<()> { if start.elapsed() > wait_for { let max_time = wait_for.as_secs(); error!("We waited for more than {max_time}"); - return Err(Error::from( - "Timeout while waiting for all the data to be available", - )); + anyhow::bail!("Timeout while waiting for all the data to be available",); } tokio::time::sleep(delay).await; diff --git a/src/connectors/tests/clickhouse/simple_test.rs b/src/connectors/tests/clickhouse/simple_test.rs index 140b422d76..f5e39189ef 100644 --- a/src/connectors/tests/clickhouse/simple_test.rs +++ b/src/connectors/tests/clickhouse/simple_test.rs @@ -162,9 +162,7 @@ async fn simple_insertion() -> Result<()> { if start.elapsed() > wait_for { let max_time = wait_for.as_secs(); error!("We waited for more than {max_time}"); - return Err(Error::from( - "Timeout while waiting for all the data to be available", - )); + anyhow::bail!("Timeout while waiting for all the data to be available",); } tokio::time::sleep(delay).await; diff --git a/src/connectors/tests/clickhouse/utils.rs b/src/connectors/tests/clickhouse/utils.rs index 986f214333..b31b3d09c4 100644 --- a/src/connectors/tests/clickhouse/utils.rs +++ b/src/connectors/tests/clickhouse/utils.rs @@ -31,7 +31,7 @@ pub(super) async fn wait_for_ok(port: u16) -> Result<()> { if start.elapsed() > wait_for { let max_time = wait_for.as_secs(); error!("We waited for more than {max_time}"); - return Err(e.chain_err(|| "Waiting for the ClickHouse container timed out.")); + return Err(e.context("Waiting for the ClickHouse container timed out.")); } tokio::time::sleep(Duration::from_secs(1)).await; diff --git a/src/connectors/tests/elastic.rs b/src/connectors/tests/elastic.rs index 7bca6e90fb..ee0045f571 100644 --- a/src/connectors/tests/elastic.rs +++ b/src/connectors/tests/elastic.rs @@ -1233,9 +1233,7 @@ async fn wait_for_es(elastic: &Elasticsearch) -> Result<()> { .await { if start.elapsed() > wait_for { - return Err( - Error::from(e).chain_err(|| "Waiting for elasticsearch container timed out.") - ); + return Err(Error::from(e).context("Waiting for elasticsearch container timed out.")); } tokio::time::sleep(Duration::from_secs(1)).await; } diff --git a/src/connectors/tests/file.rs b/src/connectors/tests/file.rs index a3a38af1ab..48450012ff 100644 --- a/src/connectors/tests/file.rs +++ b/src/connectors/tests/file.rs @@ -24,7 +24,7 @@ async fn file_connector() -> Result<()> { let input_path = Path::new(file!()) .parent() - .ok_or("bad path")? + .expect("bad path") .join("../../..") .join("tests") .join("data") diff --git a/src/connectors/tests/file_non_existent.rs b/src/connectors/tests/file_non_existent.rs index 4895da57dd..ed0c1db4f1 100644 --- a/src/connectors/tests/file_non_existent.rs +++ b/src/connectors/tests/file_non_existent.rs @@ -23,7 +23,7 @@ async fn file_connector() -> Result<()> { let input_path = Path::new(file!()) .parent() - .ok_or("bad path")? + .expect("bad path") .join("data") .join("non_existent.txt"); let defn = literal!({ diff --git a/src/connectors/tests/file_xz.rs b/src/connectors/tests/file_xz.rs index c56a718221..f81ed518c2 100644 --- a/src/connectors/tests/file_xz.rs +++ b/src/connectors/tests/file_xz.rs @@ -24,7 +24,7 @@ async fn file_connector_xz() -> Result<()> { let input_path = Path::new(file!()) .parent() - .ok_or("bad path")? + .expect("bad path") .join("../../..") .join("tests") .join("data") diff --git a/src/connectors/tests/http/client.rs b/src/connectors/tests/http/client.rs index 997194fee0..f38e11545f 100644 --- a/src/connectors/tests/http/client.rs +++ b/src/connectors/tests/http/client.rs @@ -111,7 +111,7 @@ impl TestHttpServer { // Return the service to hyper. async move { Ok::<_, Infallible>(service) } }); - let addr = (host, port).to_socket_addrs()?.next().ok_or("no address")?; + let addr = (host, port).to_socket_addrs()?.next().expect("no address"); let listener = hyper::Server::bind(&addr).serve(make_service); listener.await?; @@ -683,7 +683,7 @@ async fn missing_tls_config_https() -> Result<()> { .map(|e| e.to_string()) .unwrap_or_default(); - assert_eq!("Invalid Definition for connector \"missing_tls_config_https\": missing tls config with 'https' url. Set 'tls' to 'true' or provide a full tls config.", res); + assert_eq!("[missing_tls_config_https] Invalid definition: missing tls config with 'https' url. Set 'tls' to 'true' or provide a full tls config.", res); Ok(()) } @@ -700,7 +700,7 @@ async fn missing_config() -> Result<()> { .map(|e| e.to_string()) .unwrap_or_default(); - assert!(res.contains("Missing Configuration")); + assert!(dbg!(res).contains("Missing configuration")); Ok(()) } diff --git a/src/connectors/tests/http/server.rs b/src/connectors/tests/http/server.rs index 6b65edba65..d601546663 100644 --- a/src/connectors/tests/http/server.rs +++ b/src/connectors/tests/http/server.rs @@ -67,7 +67,7 @@ where } let (_out, _err) = connector.stop().await?; - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }); let response = timeout( Duration::from_secs(5), @@ -105,12 +105,12 @@ async fn http_server_test_basic() -> Result<()> { // send an empty body, return request data in the body as json - let mut result = Err("todo".into()); + let mut result = Err(anyhow::anyhow!("todo")); let mut final_port = 0; while let Err(e) = result { if start.elapsed() > timeout { - return Err(format!("HTTP Server not listening after {timeout:?}: {e}").into()); + anyhow::bail!("HTTP Server not listening after {timeout:?}: {e}"); } let (connector, url, port) = harness_dflt("http").await?; final_port = port; @@ -378,7 +378,7 @@ async fn https_server_test() -> Result<()> { }; c_addr.send_sink(SinkMsg::Event { event, port: IN }).await?; } - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }); let tls_config = TLSClientConfig { @@ -415,7 +415,7 @@ async fn https_server_test() -> Result<()> { .uri(&url) .body(hyper::Body::empty())?; if start.elapsed() > max_timeout { - return Err(format!("Timeout waiting for HTTPS server to boot up: {e}").into()); + anyhow::bail!("Timeout waiting for HTTPS server to boot up: {e}"); } response = timeout(one_sec, client.request(req)).await; } diff --git a/src/connectors/tests/kafka/consumer.rs b/src/connectors/tests/kafka/consumer.rs index ad90291ce7..28679d1ed3 100644 --- a/src/connectors/tests/kafka/consumer.rs +++ b/src/connectors/tests/kafka/consumer.rs @@ -106,7 +106,7 @@ async fn transactional_retry() -> Result<()> { value: Some("snot"), })); if producer.send(record, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to kafka".into()); + anyhow::bail!("Unable to send record to kafka"); } let e1 = harness.out()?.get_event().await?; @@ -142,7 +142,7 @@ async fn transactional_retry() -> Result<()> { .partition(0) .timestamp(12); if producer.send(record2, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } let e2 = harness.out()?.get_event().await?; assert_eq!(Value::null(), e2.data.suffix().value()); @@ -191,7 +191,7 @@ async fn transactional_retry() -> Result<()> { .partition(2) .timestamp(123); if producer.send(record3, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // next event @@ -207,7 +207,6 @@ async fn transactional_retry() -> Result<()> { "partition": 2, "timestamp": 123_000_000 } - }), e4.data.suffix().meta() ); @@ -219,7 +218,7 @@ async fn transactional_retry() -> Result<()> { .partition(2) .timestamp(1234); if producer.send(record4, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } let e5 = harness.err()?.get_event().await?; @@ -336,7 +335,7 @@ async fn custom_no_retry() -> Result<()> { value: Some("snot"), })); if producer.send(record, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to kafka".into()); + anyhow::bail!("Unable to send record to kafka"); } let e1 = harness.out()?.get_event().await?; @@ -372,7 +371,7 @@ async fn custom_no_retry() -> Result<()> { .partition(0) .timestamp(12); if producer.send(record2, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // get second event @@ -408,7 +407,7 @@ async fn custom_no_retry() -> Result<()> { .partition(2) .timestamp(123); if producer.send(record3, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // we only get the next event, no previous one @@ -436,7 +435,7 @@ async fn custom_no_retry() -> Result<()> { .partition(2) .timestamp(1234); if producer.send(record4, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } let e5 = harness.err()?.get_event().await?; @@ -541,7 +540,7 @@ async fn performance() -> Result<()> { value: Some("snot"), })); if producer.send(record, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to kafka".into()); + anyhow::bail!("Unable to send record to kafka"); } let e1 = harness.out()?.get_event().await?; @@ -577,7 +576,7 @@ async fn performance() -> Result<()> { .partition(0) .timestamp(12); if producer.send(record2, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // get second event @@ -614,7 +613,7 @@ async fn performance() -> Result<()> { .partition(2) .timestamp(123); if producer.send(record3, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // we only get the next event, no previous one @@ -642,7 +641,7 @@ async fn performance() -> Result<()> { .partition(2) .timestamp(1234); if producer.send(record4, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } let e5 = harness.err()?.get_event().await?; @@ -828,7 +827,7 @@ async fn connector_kafka_consumer_pause_resume() -> Result<()> { .partition(1) .timestamp(0); if producer.send(record, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } debug!("BEFORE GET EVENT 1"); let e1 = harness.out()?.get_event().await?; @@ -848,7 +847,7 @@ async fn connector_kafka_consumer_pause_resume() -> Result<()> { .partition(0) .timestamp(1); if producer.send(record2, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // we didn't receive shit because we are paused assert!(harness @@ -929,7 +928,7 @@ async fn transactional_store_offset_handling() -> Result<()> { .partition(0) .timestamp(1); if producer.send(record1, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // send message 2 let record2 = FutureRecord::to(topic) @@ -938,7 +937,7 @@ async fn transactional_store_offset_handling() -> Result<()> { .partition(0) .timestamp(2); if producer.send(record2, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // send message 3 @@ -948,7 +947,7 @@ async fn transactional_store_offset_handling() -> Result<()> { .partition(0) .timestamp(3); if producer.send(record3, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // receive events @@ -1129,7 +1128,7 @@ async fn transactional_commit_offset_handling() -> Result<()> { .partition(0) .timestamp(1); if producer.send(record1, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // send message 2 let record2 = FutureRecord::to(topic) @@ -1138,7 +1137,7 @@ async fn transactional_commit_offset_handling() -> Result<()> { .partition(0) .timestamp(2); if producer.send(record2, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // send message 3 @@ -1148,7 +1147,7 @@ async fn transactional_commit_offset_handling() -> Result<()> { .partition(0) .timestamp(3); if producer.send(record3, PRODUCE_TIMEOUT).await.is_err() { - return Err("Unable to send record to Kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // receive events @@ -1278,7 +1277,7 @@ async fn transactional_commit_offset_handling() -> Result<()> { .partition(0) .timestamp(5678); if producer.send(record5, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } let record6 = FutureRecord::to(topic) .key("6") @@ -1286,7 +1285,7 @@ async fn transactional_commit_offset_handling() -> Result<()> { .partition(0) .timestamp(5679); if producer.send(record6, PRODUCE_TIMEOUT).await.is_err() { - return Err("Could not send to kafka".into()); + anyhow::bail!("Unable to send record to Kafka"); } // discard old repeated events (We don't know why they come again) @@ -1343,7 +1342,7 @@ async fn create_topic( match r { Err((topic, err)) => { error!("Error creating topic {}: {}", &topic, err); - return Err(err.to_string().into()); + return Err(err.into()); } Ok(topic) => { info!("Created topic {}", topic); diff --git a/src/connectors/tests/kafka/producer.rs b/src/connectors/tests/kafka/producer.rs index 4f251096db..b15f1eb71d 100644 --- a/src/connectors/tests/kafka/producer.rs +++ b/src/connectors/tests/kafka/producer.rs @@ -143,7 +143,7 @@ async fn connector_kafka_producer() -> Result<()> { return Err(e.into()); } None => { - return Err("Topic Stream unexpectedly finished.".into()); + anyhow::bail!("Topic Stream unexpectedly finished.") } }; assert!(harness.get_pipe(IN)?.get_contraflow_events().is_empty()); @@ -185,7 +185,7 @@ async fn connector_kafka_producer() -> Result<()> { return Err(e.into()); } None => { - return Err("EOF on kafka topic".into()); + anyhow::bail!("EOF on kafka topic") } } diff --git a/src/connectors/tests/s3.rs b/src/connectors/tests/s3.rs index 07bcdaf075..f44490609b 100644 --- a/src/connectors/tests/s3.rs +++ b/src/connectors/tests/s3.rs @@ -40,7 +40,7 @@ async fn wait_for_s3(port: u16) -> Result<()> { while let Err(e) = s3_client.list_buckets().send().await { if start.elapsed() > wait_for { - return Err(Error::from(e).chain_err(|| "Waiting for mock-s3 container timed out")); + return Err(Error::from(e).context("Waiting for mock-s3 container timed out")); } tokio::time::sleep(Duration::from_millis(200)).await; @@ -54,7 +54,7 @@ async fn wait_for_bucket(bucket: &str, client: Client) -> Result<()> { while let Err(e) = client.head_bucket().bucket(bucket).send().await { if start.elapsed() > wait_for { - return Err(Error::from(e).chain_err(|| "Waiting for bucket to become alive timed out")); + return Err(Error::from(e).context("Waiting for bucket to become alive timed out")); } tokio::time::sleep(Duration::from_millis(200)).await; diff --git a/src/connectors/tests/tcp.rs b/src/connectors/tests/tcp.rs index d6a856a88b..151d12ed72 100644 --- a/src/connectors/tests/tcp.rs +++ b/src/connectors/tests/tcp.rs @@ -65,13 +65,14 @@ impl EchoServer { Ok(Ok(bytes_read)) => { debug!("[ECHO SERVER] Received {bytes_read} bytes."); buf.truncate(bytes_read); - stream.write_all(&buf).await.map_err(|e| { - Error::from(format!("Error writing to tcp connection: {e}")) - })?; + stream + .write_all(&buf) + .await + .map_err(|e| Error::msg(format!("Error writing to tcp connection: {e}")))?; } Ok(Err(e)) => { error!("[ECHO SERVER] Error: {e}"); - return Err(Error::from(format!( + return Err(Error::msg(format!( "Error reading from tcp connection: {e}" ))); } diff --git a/src/connectors/tests/ws.rs b/src/connectors/tests/ws.rs index 1efb4399db..f9511e8e75 100644 --- a/src/connectors/tests/ws.rs +++ b/src/connectors/tests/ws.rs @@ -13,10 +13,11 @@ // limitations under the License. use super::{free_port::find_free_tcp_port, setup_for_tls, ConnectorHarness}; -use crate::channel::{bounded, Receiver, Sender, TryRecvError}; +use crate::channel::{bounded, Receiver, Sender}; use crate::connectors::impls::ws::WsDefaults; use crate::connectors::{impls::ws, utils::tls::TLSClientConfig}; -use crate::errors::{Result, ResultExt}; +use crate::errors::Result; +use anyhow::{Context, Error}; use futures::SinkExt; use futures::StreamExt; use rustls::ServerName; @@ -33,6 +34,7 @@ use std::{ }; use tokio::{ net::{TcpListener, TcpStream}, + sync::mpsc::error::TryRecvError, task, }; use tokio_rustls::TlsConnector; @@ -82,7 +84,7 @@ impl TestClient>> { async fn send(&mut self, data: &str) -> Result<()> { let status = self.client.send(Message::Text(data.to_string())).await; if status.is_err() { - Err("Failed to send to ws server".into()) + Err(Error::msg("Failed to send to ws server")) } else { Ok(()) } @@ -98,7 +100,7 @@ impl TestClient>> { Some(Ok(Message::Binary(data))) => Ok(ExpectMessage::Binary(data)), Some(Ok(other)) => Ok(ExpectMessage::Unexpected(other)), Some(Err(e)) => Err(e.into()), - None => Err("EOF".into()), // stream end + None => Err(Error::msg("EOF")), // stream end } } @@ -144,13 +146,13 @@ impl TestClient>> { fn send(&mut self, data: &str) -> Result<()> { self.client .send(Message::Text(data.into())) - .chain_err(|| "Failed to send to ws server") + .context("Failed to send to ws server") } fn port(&mut self) -> Result { match self.client.get_ref() { MaybeTlsStream::Plain(client) => Ok(client.local_addr()?.port()), - _otherwise => Err("Unable to retrieve local port".into()), + _otherwise => Err(anyhow::anyhow!("Unable to retrieve local port")), } } @@ -250,7 +252,7 @@ impl TestServer { Ok(Message::Binary(data)) => return Ok(ExpectMessage::Binary(data)), Ok(other) => return Ok(ExpectMessage::Unexpected(other)), Err(TryRecvError::Empty) => continue, - Err(_e) => return Err("Failed to receive text message".into()), + Err(_e) => anyhow::bail!("Failed to receive text message"), } } } @@ -308,10 +310,7 @@ async fn ws_server_text_routing() -> Result<()> { match TestClient::new(&format!("localhost:{free_port}")) { Err(e) => { if start.elapsed() > timeout { - return Err(format!( - "Timeout waiting for the ws server to start listening: {e}." - ) - .into()); + anyhow::bail!("Timeout waiting for the ws server to start listening: {e}."); } tokio::time::sleep(Duration::from_secs(1)).await; } @@ -489,10 +488,7 @@ async fn wss_server_text_routing() -> Result<()> { match TestClient::new_tls(url.as_str(), free_port).await { Err(e) => { if start.elapsed() > timeout { - return Err(format!( - "Timeout waiting for the ws server to start listening: {e}." - ) - .into()); + anyhow::bail!("Timeout waiting for the ws server to start listening: {e}."); } tokio::time::sleep(Duration::from_secs(1)).await; } @@ -579,10 +575,7 @@ async fn wss_server_binary_routing() -> Result<()> { match TestClient::new_tls(url.as_str(), free_port).await { Err(e) => { if start.elapsed() > timeout { - return Err(format!( - "Timeout waiting for the ws server to start listening: {e}." - ) - .into()); + anyhow::bail!("Timeout waiting for the ws server to start listening: {e}."); } tokio::time::sleep(Duration::from_secs(1)).await; } diff --git a/src/connectors/utils/object_storage.rs b/src/connectors/utils/object_storage.rs index d32624c2ae..acb0ddde32 100644 --- a/src/connectors/utils/object_storage.rs +++ b/src/connectors/utils/object_storage.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::connectors::prelude::*; -use crate::errors::{err_object_storage, Result}; +use crate::errors::Result; /// mode of operation for object storage connectors #[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq)] @@ -78,44 +78,39 @@ impl std::fmt::Display for ObjectId { pub(crate) const NAME: &str = "name"; pub(crate) const BUCKET: &str = "bucket"; +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("Metadata `${0}.{1}` is invalid or missing.")] + MetadataError(String, &'static str), + #[error("No upload in progress")] + NoUpload, +} + pub(crate) trait ObjectStorageCommon { fn connector_type(&self) -> &str; fn default_bucket(&self) -> Option<&String>; fn get_bucket_name(&self, meta: Option<&Value>) -> Result { let res = match (meta.get(BUCKET), self.default_bucket()) { - (Some(meta_bucket), _) => meta_bucket.as_str().ok_or_else(|| { - err_object_storage(format!( - "Metadata `${}.{BUCKET}` is not a string.", - self.connector_type() - )) - }), + (Some(meta_bucket), _) => meta_bucket + .as_str() + .ok_or_else(|| Error::MetadataError(self.connector_type().to_string(), BUCKET)), (None, Some(default_bucket)) => Ok(default_bucket.as_str()), - (None, None) => Err(err_object_storage(format!( - "Metadata `${}.{BUCKET}` missing", - self.connector_type() - ))), + (None, None) => Err(Error::MetadataError( + self.connector_type().to_string(), + BUCKET, + )), }; - res.map(ToString::to_string) + Ok(res.map(ToString::to_string)?) } fn get_object_id(&self, meta: Option<&Value<'_>>) -> Result { let name = meta .get(NAME) - .ok_or_else(|| { - err_object_storage(format!( - "`${}.{NAME}` metadata is missing", - self.connector_type() - )) - })? + .ok_or_else(|| Error::MetadataError(self.connector_type().to_string(), NAME))? .as_str() - .ok_or_else(|| { - err_object_storage(format!( - "`${}.{NAME}` metadata is not a string", - self.connector_type() - )) - })?; + .ok_or_else(|| Error::MetadataError(self.connector_type().to_string(), NAME))?; let bucket = self.get_bucket_name(meta)?; Ok(ObjectId::new(bucket, name)) } @@ -222,7 +217,7 @@ where Upload: ObjectStorageUpload + Send + Sync, Impl: ObjectStorageSinkImpl + Send + Sync, { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { self.buffers = Buffer::new(self.sink_impl.buffer_size()); self.sink_impl.connect(ctx).await?; @@ -270,7 +265,7 @@ where Ok(SinkReply::NONE) } - async fn on_stop(&mut self, ctx: &SinkContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SinkContext) -> anyhow::Result<()> { // Commit the final upload. self.fail_or_finish_upload(ctx).await?; Ok(()) @@ -351,9 +346,7 @@ where .sink_impl .upload_data( data, - self.current_upload - .as_mut() - .ok_or_else(|| err_object_storage("No upload available"))?, + self.current_upload.as_mut().ok_or(Error::NoUpload)?, ctx, ) .await?; @@ -413,7 +406,7 @@ where Upload: ObjectStorageUpload + Send + Sync, Buffer: ObjectStorageBuffer + Send + Sync, { - async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> anyhow::Result { // clear out the previous upload self.current_upload = None; self.buffer = Buffer::new(self.sink_impl.buffer_size()); @@ -434,7 +427,7 @@ where ctx: &SinkContext, serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { for (value, meta) in event.value_meta_iter() { let sink_meta = ctx.extract_meta(meta); let object_id = self.sink_impl.get_object_id(sink_meta)?; @@ -474,9 +467,7 @@ where self.sink_impl .upload_data( data, - self.current_upload - .as_mut() - .ok_or_else(|| err_object_storage("No upload available"))?, + self.current_upload.as_mut().ok_or(Error::NoUpload)?, ctx, ) .await, @@ -494,7 +485,7 @@ where Ok(SinkReply::ack_or_none(event.transactional)) } - async fn on_stop(&mut self, ctx: &SinkContext) -> Result<()> { + async fn on_stop(&mut self, ctx: &SinkContext) -> anyhow::Result<()> { if let Some(current_upload) = self.current_upload.take() { let final_part = self.buffer.reset(); ctx.swallow_err( diff --git a/src/connectors/utils/pb.rs b/src/connectors/utils/pb.rs index f72153d5f4..0a46570992 100644 --- a/src/connectors/utils/pb.rs +++ b/src/connectors/utils/pb.rs @@ -12,53 +12,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::connectors::prelude::*; -use crate::errors::{Error, ErrorKind, Result}; +use crate::errors::Result; use simd_json::StaticNode; use std::collections::BTreeMap; use tremor_common::base64::encode; use tremor_otelapis::opentelemetry::proto::metrics::v1; +#[derive(Debug, Clone, thiserror::Error)] +pub(crate) enum Error { + #[error("Invalid json to mapping for `{0}`")] + InvalidMapping(&'static str), +} + pub(crate) fn maybe_string_to_pb(data: Option<&Value<'_>>) -> Result { if let Some(s) = data.as_str() { Ok(s.to_string()) } else if data.is_none() { Ok(String::new()) } else { - Err("Expected an json string to convert to pb string".into()) + Err(Error::InvalidMapping("String").into()) } } pub(crate) fn maybe_int_to_pbu64(data: Option<&Value<'_>>) -> Result { - data.map_or_else( - || Err("Expected an json u64 to convert to pb u64".into()), - |data| { - data.as_u64() - .ok_or_else(|| Error::from("not coercable to u64")) - }, - ) + Ok(data.try_as_u64()?) } pub(crate) fn maybe_int_to_pbi32(data: Option<&Value<'_>>) -> Result { - data.as_i32() - .ok_or_else(|| Error::from("not coercable to i32")) + Ok(data.try_as_i32()?) } pub(crate) fn maybe_int_to_pbi64(data: Option<&Value<'_>>) -> Result { - data.as_i64() - .ok_or_else(|| Error::from("not coercable to i64")) + Ok(data.try_as_i64()?) } pub(crate) fn maybe_int_to_pbu32(data: Option<&Value<'_>>) -> Result { - data.as_u32() - .ok_or_else(|| Error::from("not coercable to u32")) + Ok(data.try_as_u32()?) } pub(crate) fn maybe_double_to_pb(data: Option<&Value<'_>>) -> Result { - data.as_f64() - .ok_or_else(|| Error::from("not coercable to f64")) + Ok(data.try_as_f64()?) } pub(crate) trait FromValue: Sized { @@ -70,25 +64,18 @@ pub(crate) trait FromValue: Sized { impl FromValue for v1::exemplar::Value { fn from_value(data: &Value<'_>) -> Result { - Ok(v1::exemplar::Value::AsDouble( - data.as_f64() - .ok_or_else(|| Error::from("not coercable to f64"))?, - )) + Ok(v1::exemplar::Value::AsDouble(data.try_as_f64()?)) } } impl FromValue for v1::number_data_point::Value { fn from_value(data: &Value<'_>) -> Result { - Ok(v1::number_data_point::Value::AsDouble( - data.as_f64() - .ok_or_else(|| Error::from("not coercable to f64"))?, - )) + Ok(v1::number_data_point::Value::AsDouble(data.try_as_f64()?)) } } pub(crate) fn maybe_bool_to_pb(data: Option<&Value<'_>>) -> Result { - data.as_bool() - .ok_or_else(|| Error::from("not coercable to bool")) + Ok(data.try_as_bool()?) } pub(crate) fn maybe_from_value(data: Option<&Value<'_>>) -> Result> { @@ -96,8 +83,7 @@ pub(crate) fn maybe_from_value(data: Option<&Value<'_>>) -> Result } pub(crate) fn f64_repeated_to_pb(json: Option<&Value<'_>>) -> Result> { - json.as_array() - .ok_or("Unable to map json value to repeated f64 pb")? + json.try_as_array()? .iter() .map(Some) .map(maybe_double_to_pb) @@ -105,8 +91,7 @@ pub(crate) fn f64_repeated_to_pb(json: Option<&Value<'_>>) -> Result> { } pub(crate) fn u64_repeated_to_pb(json: Option<&Value<'_>>) -> Result> { - json.as_array() - .ok_or("Unable to map json value to repeated u64 pb")? + json.try_as_array()? .iter() .map(Some) .map(maybe_int_to_pbu64) @@ -174,10 +159,7 @@ pub(crate) fn value_to_prost_struct(json: &Value<'_>) -> Result { - Ok(()) + Ok::<(), anyhow::Error>(()) } // TODO: should we error on those? // Err(Inner::STOP_ALL) => Err(Error::from("failed to pause from state STOP_ALL")), // Err(Inner::STOP_READING) => Err(Error::from("failed to pause from state STOP_READING")), - Err(e) => Err(Error::from(format!("Invalid state {e}"))), - }?; - Ok(()) + Err(e) => Err(Error::InvalidState(e).into()), + } } /// Resume both reading and writing. @@ -118,16 +123,16 @@ impl QuiescenceBeacon { Ordering::Relaxed, ) { Ok(_) | Err(Inner::PAUSED | Inner::RUNNING | Inner::STOP_ALL | Inner::STOP_READING) => { - Ok(()) + Ok::<(), anyhow::Error>(()) } // TODO: should we error on those? // Err(Inner::STOP_READING) => Err(Error::from("Can't resume from STOP_READING")), // Err(Inner::STOP_ALL) => Err(Error::from("Can't resume from STOP_ALL")), - Err(e) => Err(Error::from(format!("Invalid state {e}"))), + Err(e) => Err(Error::InvalidState(e).into()), }?; self.0.resume_event.notify(Self::MAX_LISTENERS); // we might have been paused, so notify here - Ok(()) + Ok::<(), anyhow::Error>(()) } /// notify consumers of this beacon that reading and writing should be stopped diff --git a/src/connectors/utils/reconnect.rs b/src/connectors/utils/reconnect.rs index 8d48d15bb2..022733b025 100644 --- a/src/connectors/utils/reconnect.rs +++ b/src/connectors/utils/reconnect.rs @@ -14,12 +14,13 @@ /// reconnect logic and execution for connectors use crate::{ + channel::empty_e, config::Reconnect, connectors::{ sink::SinkMsg, source::SourceMsg, Addr, Connectivity, Connector, ConnectorContext, Context, Msg, }, - errors::{empty_error, Result}, + errors::Result, }; use futures::future::FutureExt; use rand::{rngs::SmallRng, Rng, SeedableRng}; @@ -189,9 +190,7 @@ pub(crate) struct ReconnectRuntime { interval_ms: Option, strategy: Box, addr: Addr, - notifier: ConnectionLostNotifier, retry_task: Option>, - alias: alias::Connector, } /// Notifier that connector implementations @@ -214,28 +213,11 @@ impl ConnectionLostNotifier { } impl ReconnectRuntime { - pub(crate) fn notifier(&self) -> ConnectionLostNotifier { - self.notifier.clone() - } /// constructor - pub(crate) fn new( - connector_addr: &Addr, - notifier: ConnectionLostNotifier, - config: &Reconnect, - ) -> Self { - Self::inner( - connector_addr.clone(), - connector_addr.alias.clone(), - notifier, - config, - ) + pub(crate) fn new(connector_addr: &Addr, config: &Reconnect) -> Self { + Self::inner(connector_addr.clone(), config) } - fn inner( - addr: Addr, - alias: alias::Connector, - notifier: ConnectionLostNotifier, - config: &Reconnect, - ) -> Self { + fn inner(addr: Addr, config: &Reconnect) -> Self { let strategy: Box = match config { Reconnect::None => Box::new(FailFast {}), Reconnect::Retry { @@ -255,12 +237,14 @@ impl ReconnectRuntime { interval_ms: None, strategy, addr, - notifier, retry_task: None, - alias, } } + pub(crate) fn alias(&self) -> &alias::Connector { + &self.addr.alias + } + /// Use the given connector to attempt to establish a connection. /// /// Will issue a retry after the configured interval (based upon the number of the attempt) @@ -274,7 +258,7 @@ impl ReconnectRuntime { let source_res = if self.addr.has_source() { self.addr .send_source(SourceMsg::Connect(tx.clone(), self.attempt))?; - rx.recv().map(|r| r.ok_or_else(empty_error)?).await + rx.recv().map(|r| r.ok_or_else(empty_e)?).await } else { Ok(true) }; @@ -282,7 +266,7 @@ impl ReconnectRuntime { self.addr .send_sink(SinkMsg::Connect(tx, self.attempt)) .await?; - rx.recv().map(|r| r.ok_or_else(empty_error)?).await + rx.recv().map(|r| r.ok_or_else(empty_e)?).await } else { Ok(true) }; @@ -326,14 +310,15 @@ impl ReconnectRuntime { pub(crate) async fn enqueue_retry(&mut self, _ctx: &ConnectorContext) -> bool { if let ShouldRetry::No(msg) = self.strategy.should_reconnect(&self.attempt) { - warn!("[Connector::{}] Not reconnecting: {}", &self.alias, msg); + warn!("[Connector::{}] Not reconnecting: {}", self.alias(), msg); false } else { // compute next interval let interval = self.strategy.next_interval(self.interval_ms, &self.attempt); info!( "[Connector::{}] Reconnecting after {} ms", - &self.alias, interval + &self.alias(), + interval ); self.interval_ms = Some(interval); @@ -347,7 +332,7 @@ impl ReconnectRuntime { if spawn_retry { let duration = Duration::from_millis(interval); let sender = self.addr.sender.clone(); - let alias = self.alias.clone(); + let alias = self.alias().clone(); self.retry_task = Some(task::spawn(async move { tokio::time::sleep(duration).await; if sender.send(Msg::Reconnect).await.is_err() { @@ -398,7 +383,7 @@ mod tests { #[async_trait::async_trait] impl Connector for FakeConnector { async fn connect(&mut self, _ctx: &ConnectorContext, _attempt: &Attempt) -> Result { - self.answer.ok_or_else(|| "Blergh!".into()) + self.answer.ok_or_else(|| anyhow::anyhow!("Blergh!")) } fn codec_requirements(&self) -> CodecReq { @@ -456,7 +441,7 @@ mod tests { sender: tx.clone(), }; let config = Reconnect::None; - let mut runtime = ReconnectRuntime::inner(addr, alias.clone(), notifier, &config); + let mut runtime = ReconnectRuntime::inner(addr, &config); let mut connector = FakeConnector { answer: Some(false), }; @@ -465,7 +450,7 @@ mod tests { alias, connector_type: "fake".into(), quiescence_beacon: qb, - notifier: runtime.notifier(), + notifier, app_ctx: AppContext::default(), }; // failing attempt @@ -494,7 +479,7 @@ mod tests { max_retries: Some(3), randomized: true, }; - let mut runtime = ReconnectRuntime::inner(addr, alias.clone(), notifier, &config); + let mut runtime = ReconnectRuntime::inner(addr, &config); let mut connector = FakeConnector { answer: Some(false), }; @@ -503,7 +488,7 @@ mod tests { alias, connector_type: "fake".into(), quiescence_beacon: qb, - notifier: runtime.notifier(), + notifier, app_ctx: AppContext::default(), }; // 1st failing attempt @@ -515,7 +500,7 @@ mod tests { assert!(matches!( timeout(Duration::from_secs(5), rx.recv()) .await? - .ok_or_else(empty_error)?, + .ok_or_else(empty_e)?, Msg::Reconnect )); @@ -528,7 +513,7 @@ mod tests { assert!(matches!( timeout(Duration::from_secs(5), rx.recv()) .await? - .ok_or_else(empty_error)?, + .ok_or_else(empty_e)?, Msg::Reconnect )); diff --git a/src/connectors/utils/socket.rs b/src/connectors/utils/socket.rs index e5701a13d6..1e7d27359d 100644 --- a/src/connectors/utils/socket.rs +++ b/src/connectors/utils/socket.rs @@ -13,12 +13,26 @@ // limitations under the License. use crate::connectors::prelude::*; -use crate::errors::{Error, Result}; +use crate::errors::Result; use socket2::{Domain, Protocol, SockAddr, Socket, Type}; use tokio::net::{lookup_host, UdpSocket}; use tokio::net::{TcpListener, TcpStream}; use tremor_common::url::{Defaults, Url}; +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("unable to resolve {0}")] + UnableToConnect(String), + #[error("no socket")] + NoSocket, + #[error("could not resolve to any addresses")] + CouldNotResolve, + #[error("invalid address {0}:{1}")] + InvalidAddress(String, u16), + #[error("missing port")] + MissingPort, +} + #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields, rename_all = "UPPERCASE")] pub(crate) struct UdpSocketOptions { @@ -58,7 +72,7 @@ pub(crate) async fn udp_socket( // the bind operation is also not awaited or anything in `UdpSocket::bind`, so this is fine here let socket_addr = sock_addr .as_socket() - .ok_or_else(|| format!("Invalid address {}:{}", host_port.0, host_port.1))?; + .ok_or_else(|| Error::InvalidAddress(host_port.0.to_string(), host_port.1))?; let sock = Socket::new( Domain::for_address(socket_addr), Type::DGRAM, @@ -79,10 +93,7 @@ pub(crate) async fn udp_socket( } } } - Err(last_err.map_or_else( - || Error::from("could not resolve to any addresses"), - Error::from, - )) + Err(last_err.map_or_else(|| Error::CouldNotResolve.into(), anyhow::Error::from)) } #[derive(Debug, Clone, Deserialize)] @@ -128,7 +139,7 @@ pub(crate) async fn tcp_server_socket( // the bind operation is also not awaited or anything in `UdpSocket::bind`, so this is fine here let socket_addr = sock_addr .as_socket() - .ok_or_else(|| format!("Invalid address {}:{}", host_port.0, host_port.1))?; + .ok_or_else(|| Error::InvalidAddress(host_port.0.to_string(), host_port.1))?; let sock = Socket::new( Domain::for_address(socket_addr), Type::STREAM, @@ -150,10 +161,7 @@ pub(crate) async fn tcp_server_socket( } } } - Err(last_err.map_or_else( - || Error::from("could not resolve to any addresses"), - Error::from, - )) + Err(last_err.map_or_else(|| Error::CouldNotResolve.into(), anyhow::Error::from)) } pub(crate) async fn tcp_client_socket( @@ -166,7 +174,7 @@ pub(crate) async fn tcp_client_socket( let sock_addr = SockAddr::from(addr); let socket_addr = sock_addr .as_socket() - .ok_or_else(|| format!("Invalid address {}:{}", host_port.0, host_port.1))?; + .ok_or_else(|| Error::InvalidAddress(host_port.0.to_string(), host_port.1))?; let sock = Socket::new( Domain::for_address(socket_addr), Type::STREAM, @@ -187,8 +195,5 @@ pub(crate) async fn tcp_client_socket( } } } - Err(last_err.map_or_else( - || Error::from("could not resolve to any addresses"), - Error::from, - )) + Err(last_err.map_or_else(|| Error::CouldNotResolve.into(), anyhow::Error::from)) } diff --git a/src/connectors/utils/tls.rs b/src/connectors/utils/tls.rs index daa4db2186..777c609d0c 100644 --- a/src/connectors/utils/tls.rs +++ b/src/connectors/utils/tls.rs @@ -14,7 +14,7 @@ //! TLS utilities -use crate::errors::{Error, Kind as ErrorKind, Result}; +use crate::errors::Result; use futures::Future; use hyper::server::{ accept::Accept, @@ -68,28 +68,24 @@ pub(crate) struct TLSClientConfig { /// Path to the private key to use for TLS with client-side certificate pub(crate) key: Option, } - +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Invalid certificate in {0}")] + InvalidCertificate(String), +} /// Load the passed certificates file fn load_certs(path: &Path) -> Result> { let certfile = tremor_common::file::open(path)?; let mut reader = BufReader::new(certfile); - certs(&mut reader) - .map_err(|_| { - Error::from(ErrorKind::TLSError(format!( - "Invalid certificate in {}", - path.display() - ))) - }) + Ok(certs(&mut reader) + .map_err(|_| Error::InvalidCertificate(path.display().to_string())) .and_then(|certs| { if certs.is_empty() { - Err(Error::from(ErrorKind::TLSError(format!( - "No valid TLS certificates found in {}", - path.display() - )))) + Err(Error::InvalidCertificate(path.display().to_string())) } else { Ok(certs.into_iter().map(Certificate).collect()) } - }) + })?) } /// Load the passed private key file @@ -99,12 +95,8 @@ fn load_keys(path: &Path) -> Result { let keyfile = tremor_common::file::open(path)?; let mut reader = BufReader::new(keyfile); - let certs = pkcs8_private_keys(&mut reader).map_err(|_e| { - Error::from(ErrorKind::TLSError(format!( - "Invalid PKCS8 Private key in {}", - path.display() - ))) - })?; + let certs = pkcs8_private_keys(&mut reader) + .map_err(|_e| Error::InvalidCertificate(path.display().to_string()))?; let mut keys: Vec = certs.into_iter().map(PrivateKey).collect(); // only attempt to load as RSA keys if file has no pkcs8 keys @@ -112,23 +104,15 @@ fn load_keys(path: &Path) -> Result { let keyfile = tremor_common::file::open(path)?; let mut reader = BufReader::new(keyfile); keys = rsa_private_keys(&mut reader) - .map_err(|_e| { - Error::from(ErrorKind::TLSError(format!( - "Invalid RSA Private key in {}", - path.display() - ))) - })? + .map_err(|_e| Error::InvalidCertificate(path.display().to_string()))? .into_iter() .map(PrivateKey) .collect(); } - keys.into_iter().next().ok_or_else(|| { - Error::from(ErrorKind::TLSError(format!( - "No valid private keys (RSA or PKCS8) found in {}", - path.display() - ))) - }) + keys.into_iter() + .next() + .ok_or_else(|| Error::InvalidCertificate(path.display().to_string()).into()) } impl TLSServerConfig { pub(crate) fn to_server_config(&self) -> Result { @@ -162,12 +146,8 @@ impl TLSClientConfig { Item::X509Certificate(cert) => Some(Certificate(cert)), _ => None, }); - cert.and_then(|cert| roots.add(&cert).ok()).ok_or_else(|| { - Error::from(ErrorKind::TLSError(format!( - "Invalid certificate in {}", - cafile.display() - ))) - })?; + cert.and_then(|cert| roots.add(&cert).ok()) + .ok_or_else(|| Error::InvalidCertificate(cafile.display().to_string()))?; roots } else { SYSTEM_ROOT_CERTS.clone() diff --git a/src/errors.rs b/src/errors.rs index 526200de0c..5dbe442ddc 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -12,480 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -//NOTE: error_chain -#![allow(deprecated, missing_docs, clippy::large_enum_variant)] - -use error_chain::error_chain; -use hdrhistogram::{self, serialization as hdr_s}; -use tokio::sync::broadcast; -use tremor_common::ports::Port; -use value_trait::prelude::*; - -pub type Kind = ErrorKind; - -impl From> for Error { - fn from(e: sled::transaction::TransactionError<()>) -> Self { - Self::from(format!("Sled Transaction Error: {e:?}")) - } -} - -impl From for Error { - fn from(e: hdr_s::DeserializeError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From> for Error { - fn from(e: Box) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From> for Error { - fn from(e: Box) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From for Error { - fn from(e: hdrhistogram::errors::CreationError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From for Error { - fn from(e: hdrhistogram::RecordError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From for Error { - fn from(e: hdrhistogram::serialization::V2SerializeError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From for Error { - fn from(e: http_types::Error) -> Self { - Self::from(format!("{e}")) - } -} - -impl From for Error { - fn from(e: glob::PatternError) -> Self { - Self::from(format!("{e}")) - } -} - -impl From> for Error { - fn from(e: crate::channel::SendError) -> Self { - Self::from(format!("{e}")) - } -} - -impl From for Error { - fn from(e: crate::channel::TryRecvError) -> Self { - Self::from(format!("{e}")) - } -} - -impl From> for Error { - fn from(e: async_channel::SendError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From> for Error { - fn from(e: async_std::channel::SendError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From> for Error { - fn from(e: async_std::channel::TrySendError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl From> for Error -where - T: std::fmt::Debug, -{ - fn from(e: broadcast::error::SendError) -> Self { - Self::from(format!("{e:?}")) - } -} -impl From for Error { - fn from(e: broadcast::error::TryRecvError) -> Self { - Self::from(format!("{e:?}")) - } -} -impl From for Error { - fn from(e: broadcast::error::RecvError) -> Self { - Self::from(format!("{e:?}")) - } -} - -impl

From> for Error { - fn from(e: std::sync::PoisonError

) -> Self { - Self::from(format!("Poison Error: {e:?}")) - } -} - -impl From> for Error { - fn from(e: aws_sdk_s3::error::SdkError) -> Self { - Self::from(ErrorKind::S3Error(format!("{e:?}"))) - } -} - -impl From for Error { - fn from(e: TryTypeError) -> Self { - ErrorKind::TypeError(e.expected, e.got).into() - } -} - -#[cfg(test)] -impl PartialEq for Error { - fn eq(&self, _other: &Self) -> bool { - // This might be Ok since we try to compare Result in tests - false - } -} - -error_chain! { - links { - Script(tremor_script::errors::Error, tremor_script::errors::ErrorKind); - Pipeline(tremor_pipeline::errors::Error, tremor_pipeline::errors::ErrorKind); - Codec(tremor_codec::errors::Error, tremor_codec::errors::ErrorKind); - Interceptor(tremor_interceptor::errors::Error, tremor_interceptor::errors::ErrorKind); - } - foreign_links { - AddrParseError(std::net::AddrParseError); - AnyhowError(anyhow::Error); - AsyncChannelRecvError(async_std::channel::RecvError); - AsyncChannelTryRecvError(async_std::channel::TryRecvError); - Base64Error(tremor_common::base64::DecodeError); - ChannelReceiveError(std::sync::mpsc::RecvError); - Clickhouse(clickhouse_rs::errors::Error); - Common(tremor_common::Error); - Config(tremor_config::Error); - CronError(cron::error::Error); - DnsError(trust_dns_resolver::error::ResolveError); - ElasticError(elasticsearch::Error); - ElasticTransportBuildError(elasticsearch::http::transport::BuildError); - EnvVarError(std::env::VarError); - FromUtf8Error(std::string::FromUtf8Error); - GoogleAuthError(gouth::Error); - GrokError(grok::Error); - HeaderToStringError(http::header::ToStrError); - Hex(hex::FromHexError); - Http(http::Error); - HttpHeaderError(http::header::InvalidHeaderValue); - Hyper(hyper::Error); - InvalidHeaderName(reqwest::header::InvalidHeaderName); - InvalidMetadataValue(tonic::metadata::errors::InvalidMetadataValue); - InvalidMethod(http::method::InvalidMethod); - InvalidStatusCode(http::status::InvalidStatusCode); - InvalidTLSClientName(rustls::client::InvalidDnsNameError); - Io(std::io::Error); - JoinError(tokio::task::JoinError); - JsonAccessError(value_trait::AccessError); - JsonError(simd_json::Error); - KafkaError(rdkafka::error::KafkaError); - SerdeJsonError(serde_json::Error); - MimeParsingError(mime::FromStrError); - ModeParseError(file_mode::ModeParseError); - OneShotRecv(tokio::sync::oneshot::error::RecvError); - ParseFloatError(std::num::ParseFloatError); - ParseIntError(std::num::ParseIntError); - RaftAPIError(crate::raft::api::client::Error); - RegexError(regex::Error); - ReqwestError(reqwest::Error); - RustlsError(rustls::Error); - S3ByteStream(aws_smithy_http::byte_stream::error::Error); - S3Endpoint(aws_smithy_http::endpoint::error::InvalidEndpointError); - Serenity(serenity::Error); - Sled(sled::Error); - Timeout(tokio::time::error::Elapsed); - TonicStatusError(tonic::Status); - TonicTransportError(tonic::transport::Error); - TryFromIntError(std::num::TryFromIntError); - UriParserError(http::uri::InvalidUri); - UrlParserError(url::ParseError); - Utf8Error(std::str::Utf8Error); - ValueError(tremor_value::Error); - WalInfailable(qwal::Error); - WalJson(qwal::Error); - Ws(tokio_tungstenite::tungstenite::Error); - YamlError(serde_yaml::Error) #[doc = "Error during yaml parsing"]; - CheckIsLeaderError(openraft::error::RaftError>); - ClientWriteError(openraft::error::RaftError>); - } - - errors { - RaftNotRunning { - description("Raft is not running") - display("Raft is not running") - } - TypeError(expected: ValueType, found: ValueType) { - description("Type error") - display("Type error: Expected {}, found {}", expected, found) - } - - S3Error(n: String) { - description("S3 Error") - display("S3Error: {}", n) - } - - UnknownOp(n: String, o: String) { - description("Unknown operator") - display("Unknown operator: {}::{}", n, o) - } - UnknownConnectorType(t: String) { - description("Unknown connector type") - display("Unknown connector type {}", t) - } - - // TODO: Old errors, verify if needed - BadOpConfig(e: String) { - description("Operator config has a bad syntax") - display("Operator config has a bad syntax: {}", e) - } - - UnknownNamespace(n: String) { - description("Unknown namespace") - display("Unknown namespace: {}", n) - } - - BadUtF8InString { - description("Bad UTF8 in input string") - display("Bad UTF8 in input string") - - } - InvalidCompression { - description("Data can't be decompressed") - display("The data did not contain a known magic header to identify a supported compression") - } - KvError(s: String) { - description("KV error") - display("{}", s) - } - TLSError(s: String) { - description("TLS error") - display("{}", s) - } - InvalidTremorUrl(msg: String, invalid_url: String) { - description("Invalid Tremor URL") - display("Invalid Tremor URL {}: {}", invalid_url, msg) - } - - MissingConfiguration(s: String) { - description("Missing Configuration") - display("Missing Configuration for {}", s) - } - InvalidConfiguration(configured_thing: String, msg: String) { - description("Invalid Configuration") - display("Invalid Configuration for {}: {}", configured_thing, msg) - } - InvalidConnectorDefinition(connector_id: String, msg: String) { - description("Invalid Connector Definition") - display("Invalid Definition for connector \"{}\": {}", connector_id, msg) - } - InvalidConnect(target: String, port: Port<'static>) { - description("Invalid Connect attempt") - display("Invalid Connect to {} via port {}", target, port) - } - InvalidDisconnect(target: String, entity: String, port: Port<'static>) { - description("Invalid Disonnect attempt") - display("Invalid Disconnect of {} from {} via port {}", entity, target, port) - } - InvalidMetricsData { - description("Invalid Metrics data") - display("Invalid Metrics data") - } - NoSocket { - description("No socket available") - display("No socket available. Probably not connected yet.") - } - FlowFailed(flow: String) { - description("Flow entered failed state") - display("Flow {flow} entered failed state") - } - DeployFlowError(flow: String, err: String) { - description("Error deploying Flow") - display("Error deploying Flow {}: {}", flow, err) - } - DuplicateFlow(flow: String) { - description("Duplicate Flow") - display("Flow with id \"{}\" is already deployed.", flow) - } - ProducerNotAvailable(alias: String) { - description("Producer not available") - display("Kafka Producer not available for Connector {}", alias) - } - FlowNotFound(alias: String) { - description("Deployment not found") - display("Deployment \"{}\" not found", alias) - } - ConnectorNotFound(flow_id: String, alias: String) { - description("Connector not found") - display("Connector \"{}\" not found in Flow \"{}\"", alias, flow_id) - } - - GbqSinkFailed(msg: &'static str) { - description("GBQ Sink failed") - display("GBQ Sink failed: {}", msg) - } - GbqSchemaNotProvided(table: String) { - description("GBQ Schema not provided") - display("GBQ Schema not provided for table {}", table) - } - ClientNotAvailable(name: &'static str, msg: &'static str) { - description("Client not available") - display("{} client not available: {}", name, msg) - } - PubSubError(msg: &'static str) { - description("PubSub error") - display("PubSub error: {}", msg) - } - BigQueryTypeMismatch(expected: &'static str, actual:value_trait::ValueType) { - description("Type in the message does not match BigQuery type") - display("Type in the message does not match BigQuery type. Expected: {}, actual: {:?}", expected, actual) - } - - - NoClickHouseClientAvailable { - description("The ClickHouse adapter has no client available") - display("The ClickHouse adapter has no client available") - } - - ExpectedObjectEvent(found_type: ValueType) { - description("Expected object event") - display("Expected an object event, found a \"{found_type:?}\"") - } - - MissingEventColumn(column: String) { - description("Missing event column") - display("Column \"{column}\" is missing") - } - - UnexpectedEventFormat(column_name: String, expected_type: String, found_type: ValueType) { - description("Unexpected event format") - display("Field \"{column_name}\" is of type \'{found_type:?}\" while it should have type \"{expected_type}\"") - } - - MalformedIpAddr { - description("Malformed IP address") - display("Malformed IP address") - } - - MalformedUuid { - description("Malformed UUID") - display("Malformed UUID") - } - - GclSinkFailed(msg: &'static str) { - description("Google Cloud Logging Sink failed") - display("Google Cloud Logging Sink failed: {}", msg) - } - - GclTypeMismatch(expected: &'static str, actual:value_trait::ValueType) { - description("Type in the message does not match Google Cloud Logging API type") - display("Type in the message does not match Google Cloud Logging API type. Expected: {}, actual: {:?}", expected, actual) - } - GoogleCloudStorageError(msg: String) { - description("Google cloud storage error") - display("Google cloud storage error: {}", msg) - } - ObjectStorageError(msg: String) { - description("Object storage error") - display("{}", msg) - } - PipelineSendError(s: String) { - description("Pipeline send error") - display("Pipeline send error: {}", s) - } - ChannelEmpty { - description("Channel empty") - display("Channel empty") - } - AlreadyCreated { - description("Connector already created") - display("Connector already created") - } - } -} - -pub(crate) fn empty_error() -> Error { - ErrorKind::ChannelEmpty.into() -} -pub(crate) fn already_created_error() -> Error { - ErrorKind::AlreadyCreated.into() -} -#[allow(clippy::needless_pass_by_value)] -pub(crate) fn pipe_send_e(e: crate::channel::SendError) -> Error { - ErrorKind::PipelineSendError(e.to_string()).into() -} -#[allow(clippy::needless_pass_by_value)] -pub(crate) fn connector_send_err(e: crate::channel::SendError) -> Error { - format!("could not send to connector: {e}").into() -} - -pub(crate) fn err_connector_def(c: &C, e: &E) -> Error { - ErrorKind::InvalidConnectorDefinition(c.to_string(), e.to_string()).into() -} - -pub(crate) fn err_gcs(msg: impl Into) -> Error { - ErrorKind::GoogleCloudStorageError(msg.into()).into() -} - -pub(crate) fn err_object_storage(msg: impl Into) -> Error { - ErrorKind::ObjectStorageError(msg.into()).into() -} - -#[cfg(test)] -mod test { - use super::*; - use matches::assert_matches; - - #[test] - fn test_err_conector_def() { - let r = err_connector_def("snot", "badger").0; - assert_matches!(r, ErrorKind::InvalidConnectorDefinition(_, _)); - } - - #[test] - fn test_type_error() { - let r = Error::from(TryTypeError { - expected: ValueType::Object, - got: ValueType::String, - }) - .0; - assert_matches!( - r, - ErrorKind::TypeError(ValueType::Object, ValueType::String) - ); - } - #[test] - fn aws_error() { - use aws_sdk_s3::error::SdkError; - #[derive(Debug)] - struct DemoError(); - impl std::fmt::Display for DemoError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "DemoError") - } - } - impl std::error::Error for DemoError {} - let e: SdkError<()> = SdkError::timeout_error(Box::new(DemoError())); - let r = Error::from(e); - assert_matches!(r.0, ErrorKind::S3Error(_)); - } - - #[test] - fn send_error() { - let e = async_channel::SendError(0u8); - let error = Error::from(e); - assert_eq!(error.to_string(), "SendError(..)".to_string(),); - } +pub use anyhow::{Error, Result}; + +/// Error kind +#[derive(Debug, thiserror::Error)] +pub enum ErrorKind { + /// Error when trying to lock a mutex + #[error("Read locking error")] + ReadLock, + /// Error when trying to lock a mutex + #[error("Write locking error")] + WriteLock, } diff --git a/src/functions.rs b/src/functions.rs index 4257c39b7b..8a5563e42b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -14,7 +14,7 @@ use crate::connectors::impls::gcl; use crate::connectors::impls::otel; -use crate::errors::Result; +use crate::errors::{ErrorKind, Result}; use crate::version::VERSION; use tremor_script::registry::Registry; use tremor_script::FN_REGISTRY; @@ -25,7 +25,7 @@ use tremor_script::{tremor_const_fn, tremor_fn}; /// # Errors /// * if we can't load the registry pub fn load() -> Result<()> { - let mut reg = FN_REGISTRY.write()?; + let mut reg = FN_REGISTRY.write().map_err(|_| ErrorKind::WriteLock)?; install(&mut reg) } diff --git a/src/lib.rs b/src/lib.rs index 8475f0557a..5b4695bab8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,7 +103,7 @@ pub async fn load_troy_file(world: &Runtime, file_name: &str) -> Result { let mut src = String::new(); file.read_to_string(&mut src) - .map_err(|e| Error::from(format!("Could not open file {file_name} => {e}")))?; + .map_err(|e| anyhow::Error::from(e).context(format!("Could not open file {file_name}")))?; world.load_troy(file_name, &src).await } @@ -112,7 +112,7 @@ pub async fn load_troy_file(world: &Runtime, file_name: &str) -> Result { #[doc(hidden)] macro_rules! log_error { ($maybe_error:expr, $($args:tt)+) => ( - if let Err(e) = $maybe_error { + if let Err(e) = &$maybe_error { error!($($args)+, e = e); true } else { diff --git a/src/pipeline.rs b/src/pipeline.rs index fd735f26bc..f48de2d180 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. use crate::{ - channel::{bounded, unbounded, Receiver, Sender, UnboundedReceiver, UnboundedSender}, + channel::{bounded, send_e, unbounded, Receiver, Sender, UnboundedReceiver, UnboundedSender}, connectors::{self, sink::SinkMsg, source::SourceMsg}, - errors::{pipe_send_e, Result}, instance::State, primerge::PriorityMerge, qsize, system::flow::AppContext, + Result, }; use futures::StreamExt; use std::{fmt, time::Duration}; @@ -40,6 +40,16 @@ type Inputs = halfbrown::HashMap; type Dests = halfbrown::HashMap, Vec<(DeployEndpoint, OutputTarget)>>; type EventSet = Vec<(Port<'static>, Event)>; +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Connector has no sink")] + NoSink, + #[error("input port doesn't exist")] + BadInput, + #[error("output port doesn't exist")] + BadOutput, +} + /// Address for a pipeline #[derive(Clone)] pub struct Addr { @@ -69,16 +79,16 @@ impl Addr { /// send a contraflow insight message back down the pipeline pub(crate) fn send_insight(&self, event: Event) -> Result<()> { use CfMsg::Insight; - self.cf_addr.send(Insight(event)).map_err(pipe_send_e) + self.cf_addr.send(Insight(event)).map_err(send_e) } /// send a data-plane message to the pipeline pub(crate) async fn send(&self, msg: Box) -> Result<()> { - self.addr.send(msg).await.map_err(pipe_send_e) + self.addr.send(msg).await.map_err(send_e) } pub(crate) async fn send_mgmt(&self, msg: MgmtMsg) -> Result<()> { - self.mgmt_addr.send(msg).await.map_err(pipe_send_e) + self.mgmt_addr.send(msg).await.map_err(send_e) } pub(crate) async fn stop(&self) -> Result<()> { @@ -145,7 +155,7 @@ impl TryFrom for OutputTarget { type Error = crate::errors::Error; fn try_from(addr: connectors::Addr) -> Result { - Ok(Self::Sink(addr.sink.ok_or("Connector has no sink")?)) + Ok(Self::Sink(addr.sink.ok_or(Error::NoSink)?)) } } @@ -235,7 +245,6 @@ pub(crate) enum MgmtMsg { } #[cfg(test)] -#[allow(dead_code)] mod report { use tremor_common::ports::Port; @@ -576,11 +585,7 @@ pub(crate) async fn pipeline_task( info!("{ctx} Connecting '{endpoint}' to port '{port}'"); if !pipeline.input_exists(&port) { error!("{ctx} Error connecting input pipeline '{port}' as it does not exist",); - if tx - .send(Err("input port doesn't exist".into())) - .await - .is_err() - { + if tx.send(Err(Error::BadInput.into())).await.is_err() { error!("{ctx} Error sending status report."); } continue; @@ -601,11 +606,7 @@ pub(crate) async fn pipeline_task( // add error statement for port out in pipeline, currently no error messages if !pipeline.output_exists(&port) { error!("{ctx} Error connecting output pipeline {port}"); - if tx - .send(Err("output port doesn't exist".into())) - .await - .is_err() - { + if tx.send(Err(Error::BadOutput.into())).await.is_err() { error!("{ctx} Error sending status report."); } continue; @@ -687,8 +688,9 @@ mod tests { use super::*; use crate::{ + channel::empty_e, connectors::{prelude::SinkAddr, source::SourceAddr}, - errors::empty_error, + errors::ErrorKind, pipeline::report::{InputReport, OutputReport}, }; use std::time::Instant; @@ -701,6 +703,7 @@ mod tests { use tremor_script::{aggr_registry, lexer::Location, NodeMeta, FN_REGISTRY}; use tremor_value::Value; + #[allow(clippy::too_many_lines)] #[tokio::test(flavor = "multi_thread")] async fn report() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); @@ -708,8 +711,11 @@ mod tests { let trickle = r#"select event from in into out;"#; let aggr_reg = aggr_registry(); - let query = - tremor_pipeline::query::Query::parse(&trickle, &*FN_REGISTRY.read()?, &aggr_reg)?; + let query = tremor_pipeline::query::Query::parse( + &trickle, + &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?, + &aggr_reg, + )?; let addr = spawn( AppContext::default(), alias::Pipeline::new("test-pipe1"), @@ -739,7 +745,7 @@ mod tests { target: OutputTarget::Pipeline(Box::new(addr2.clone())), }) .await?; - rx.recv().await.ok_or_else(empty_error)??; + rx.recv().await.ok_or_else(empty_e)??; let (tx, mut rx) = bounded(1); addr2 .send_mgmt(MgmtMsg::ConnectInput { @@ -750,7 +756,7 @@ mod tests { is_transactional: true, }) .await?; - rx.recv().await.ok_or_else(empty_error)??; + rx.recv().await.ok_or_else(empty_e)??; let (tx, mut rx) = bounded(1); addr2 .send_mgmt(MgmtMsg::ConnectOutput { @@ -760,7 +766,7 @@ mod tests { target: OutputTarget::Pipeline(Box::new(addr3.clone())), }) .await?; - rx.recv().await.ok_or_else(empty_error)??; + rx.recv().await.ok_or_else(empty_e)??; let (tx, mut rx) = bounded(1); addr3 .send_mgmt(MgmtMsg::ConnectInput { @@ -771,11 +777,11 @@ mod tests { is_transactional: false, }) .await?; - rx.recv().await.ok_or_else(empty_error)??; + rx.recv().await.ok_or_else(empty_e)??; // get a status report from every single one let (tx, mut rx) = bounded(1); addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let mut report1 = rx.recv().await.ok_or_else(empty_error)?; + let mut report1 = rx.recv().await.ok_or_else(empty_e)?; assert!(report1.inputs.is_empty()); let mut output1 = report1 .outputs @@ -783,10 +789,10 @@ mod tests { .expect("nothing at port `out`"); assert!(report1.outputs.is_empty()); assert_eq!(1, output1.len()); - let output1 = output1.pop().ok_or("no data")?; + let output1 = output1.pop().expect("no data"); assert_eq!(output1, OutputReport::pipeline("snot2", IN)); addr2.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let mut report2 = rx.recv().await.ok_or_else(empty_error)?; + let mut report2 = rx.recv().await.ok_or_else(empty_e)?; let input2 = report2.inputs.pop().expect("no input at port in"); assert_eq!(input2, InputReport::pipeline("snot", OUT)); let mut output2 = report2 @@ -795,11 +801,11 @@ mod tests { .expect("no outputs on out port"); assert!(report2.outputs.is_empty()); assert_eq!(1, output2.len()); - let output2 = output2.pop().ok_or("no data")?; + let output2 = output2.pop().expect("no data"); assert_eq!(output2, OutputReport::pipeline("snot3", IN)); addr3.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let mut report3 = rx.recv().await.ok_or_else(empty_error)?; + let mut report3 = rx.recv().await.ok_or_else(empty_e)?; assert!(report3.outputs.is_empty()); let input3 = report3.inputs.pop().expect("no inputs"); assert_eq!(input3, InputReport::pipeline("snot2", OUT)); @@ -818,8 +824,11 @@ mod tests { let trickle = r#"select event from in into out;"#; let aggr_reg = aggr_registry(); let pipeline_id = alias::Pipeline::new("test-pipe"); - let query = - tremor_pipeline::query::Query::parse(&trickle, &*FN_REGISTRY.read()?, &aggr_reg)?; + let query = tremor_pipeline::query::Query::parse( + &trickle, + &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?, + &aggr_reg, + )?; let addr = spawn( AppContext::default(), pipeline_id, @@ -829,14 +838,14 @@ mod tests { let (tx, mut rx) = bounded(1); addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let report = rx.recv().await.ok_or_else(empty_error)?; + let report = rx.recv().await.ok_or_else(empty_e)?; assert_eq!(State::Initializing, report.state); assert!(report.inputs.is_empty()); assert!(report.outputs.is_empty()); addr.start().await?; addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let report = rx.recv().await.ok_or_else(empty_error)?; + let report = rx.recv().await.ok_or_else(empty_e)?; assert_eq!(State::Running, report.state); assert!(report.inputs.is_empty()); assert!(report.outputs.is_empty()); @@ -855,11 +864,11 @@ mod tests { is_transactional: true, }) .await?; - rx.recv().await.ok_or_else(empty_error)??; + rx.recv().await.ok_or_else(empty_e)??; let (tx, mut rx) = bounded(1); addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let report = rx.recv().await.ok_or_else(empty_error)?; + let report = rx.recv().await.ok_or_else(empty_e)?; assert_eq!(1, report.inputs.len()); assert_eq!( report::InputReport::source("source_01", OUT), @@ -878,11 +887,11 @@ mod tests { target, }) .await?; - rx.recv().await.ok_or_else(empty_error)??; + rx.recv().await.ok_or_else(empty_e)??; let (tx, mut rx) = bounded(1); addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let report = rx.recv().await.ok_or_else(empty_error)?; + let report = rx.recv().await.ok_or_else(empty_e)?; assert_eq!(1, report.outputs.len()); assert_eq!( Some(&vec![report::OutputReport::sink("sink_01", IN)]), @@ -896,9 +905,9 @@ mod tests { }; addr.send(Box::new(Msg::Event { event, input: IN })).await?; - let mut sink_msg = sink_rx.recv().await.ok_or_else(empty_error)?; + let mut sink_msg = sink_rx.recv().await.ok_or_else(empty_e)?; while let SinkMsg::Signal { .. } = sink_msg { - sink_msg = sink_rx.recv().await.ok_or_else(empty_error)?; + sink_msg = sink_rx.recv().await.ok_or_else(empty_e)?; } match sink_msg { SinkMsg::Event { event, port: _ } => { @@ -915,7 +924,7 @@ mod tests { .await?; let start = Instant::now(); - let mut sink_msg = sink_rx.recv().await.ok_or_else(empty_error)?; + let mut sink_msg = sink_rx.recv().await.ok_or_else(empty_e)?; while !matches!(sink_msg, SinkMsg::Signal { signal: @@ -930,12 +939,12 @@ mod tests { "Timed out waiting for drain signal on the sink" ); - sink_msg = sink_rx.recv().await.ok_or_else(empty_error)?; + sink_msg = sink_rx.recv().await.ok_or_else(empty_e)?; } let event_id = EventId::from_id(1, 1, 1); addr.send_insight(Event::cb_ack(0, event_id.clone(), OpMeta::default()))?; - let source_msg = source_rx.recv().await.ok_or_else(empty_error)?; + let source_msg = source_rx.recv().await.ok_or_else(empty_e)?; if let SourceMsg::Cb(cb_action, cb_id) = source_msg { assert_eq!(event_id, cb_id); assert_eq!(CbAction::Ack, cb_action); @@ -946,14 +955,14 @@ mod tests { // test pause and resume addr.pause().await?; addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let report = rx.recv().await.ok_or_else(empty_error)?; + let report = rx.recv().await.ok_or_else(empty_e)?; assert_eq!(State::Paused, report.state); assert_eq!(1, report.inputs.len()); assert_eq!(1, report.outputs.len()); addr.resume().await?; addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; - let report = rx.recv().await.ok_or_else(empty_error)?; + let report = rx.recv().await.ok_or_else(empty_e)?; assert_eq!(State::Running, report.state); assert_eq!(1, report.inputs.len()); assert_eq!(1, report.outputs.len()); diff --git a/src/raft.rs b/src/raft.rs index 928bcc0144..73dc060f68 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -28,25 +28,11 @@ pub mod store; #[cfg(test)] mod test; -use crate::raft::node::Addr; - -pub(crate) use self::manager::Cluster; -use api::client::Error; +pub(crate) use self::manager::{Cluster, ClusterInterface}; use network::raft_network_impl::Network; -use openraft::{ - error::{Fatal, InitializeError, RaftError}, - metrics::WaitError, - storage::Adaptor, - Config, ConfigError, Raft, TokioRuntime, -}; -use redb::{CommitError, DatabaseError, StorageError, TableError, TransactionError}; -use std::{ - fmt::{Display, Formatter}, - io::Cursor, - sync::Mutex, -}; +use openraft::{storage::Adaptor, Config, Raft, TokioRuntime}; +use std::io::Cursor; use store::{TremorRequest, TremorResponse}; -use tokio::task::JoinError; /// A `NodeId` pub type NodeId = u64; @@ -74,180 +60,38 @@ pub(crate) type StateMachineStore = Adaptor; pub(crate) type TremorRaftImpl = Raft; /// A raft cluster error -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum ClusterError { - /// Generic error - Other(String), - /// Database error - Database(DatabaseError), - /// Transaction error - Transaction(TransactionError), - /// Transaction error - Table(TableError), - /// StorageError - Storage(StorageError), - /// Commit Error - Commit(CommitError), - /// IO error - Io(std::io::Error), - /// Store error - Store(store::Error), - /// Raft error during initialization - Initialize(RaftError>), - /// MsgPack encode error - MsgPackEncode(rmp_serde::encode::Error), - /// MsgPack decode error - MsgPackDecode(rmp_serde::decode::Error), - /// Config error - Config(ConfigError), - /// Client error - Client(Error), - /// Join error - JoinError(JoinError), - /// Fatal error - Fatal(Fatal), - /// Wait error - WaitError(WaitError), - // TODO: this is a horrible hack - /// Runtime error - Runtime(Mutex), -} - -impl From for ClusterError { - fn from(e: store::Error) -> Self { - ClusterError::Store(e) - } -} - -impl From for ClusterError { - fn from(e: std::io::Error) -> Self { - ClusterError::Io(e) - } -} - -impl From for ClusterError { - fn from(e: redb::DatabaseError) -> Self { - ClusterError::Database(e) - } -} - -impl From for ClusterError { - fn from(e: redb::TransactionError) -> Self { - ClusterError::Transaction(e) - } -} - -impl From for ClusterError { - fn from(e: redb::TableError) -> Self { - ClusterError::Table(e) - } -} - -impl From for ClusterError { - fn from(e: redb::StorageError) -> Self { - ClusterError::Storage(e) - } -} - -impl From for ClusterError { - fn from(e: redb::CommitError) -> Self { - ClusterError::Commit(e) - } -} - -impl From<&str> for ClusterError { - fn from(e: &str) -> Self { - ClusterError::Other(e.to_string()) - } -} - -impl From for ClusterError { - fn from(e: String) -> Self { - ClusterError::Other(e) - } -} - -impl From>> for ClusterError { - fn from(e: RaftError>) -> Self { - ClusterError::Initialize(e) - } -} - -impl From for ClusterError { - fn from(e: ConfigError) -> Self { - ClusterError::Config(e) - } -} - -impl From for ClusterError { - fn from(e: Error) -> Self { - Self::Client(e) - } -} - -impl From for ClusterError { - fn from(e: crate::Error) -> Self { - ClusterError::Runtime(Mutex::new(e)) - } -} - -impl From for ClusterError { - fn from(e: rmp_serde::encode::Error) -> Self { - ClusterError::MsgPackEncode(e) - } -} - -impl From for ClusterError { - fn from(e: rmp_serde::decode::Error) -> Self { - ClusterError::MsgPackDecode(e) - } -} - -impl From for ClusterError { - fn from(e: JoinError) -> Self { - ClusterError::JoinError(e) - } -} - -impl From> for ClusterError { - fn from(e: Fatal) -> Self { - ClusterError::Fatal(e) - } -} - -impl From for ClusterError { - fn from(e: WaitError) -> Self { - ClusterError::WaitError(e) - } -} + /// The raft node isn't running + #[error("The raft node isn't running")] + RaftNotRunning, + /// Bad bind address + #[error("Bad bind address")] + BadAddr, + /// Error shutting down local raft node + #[error("Error shutting down local raft node")] + Shutdown, + /// No join endpoints provided + #[error("No join endpoints provided")] + NoEndpoints, +} + +type ClusterResult = crate::Result; + +/// We need this since openraft and anyhow hate eachother +/// anyhow refuses to implement `std::error::Error` on it's error type +/// and openraft requires the error type to implement `std::error::Error` +/// so instead of forking we're doing the silly dance +#[derive(Debug)] +pub(crate) struct SillyError(anyhow::Error); -impl Display for ClusterError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - ClusterError::Other(e) => e.fmt(f), - ClusterError::Database(e) => e.fmt(f), - ClusterError::Transaction(e) => e.fmt(f), - ClusterError::Table(e) => e.fmt(f), - ClusterError::Storage(e) => e.fmt(f), - ClusterError::Commit(e) => e.fmt(f), - ClusterError::Io(e) => e.fmt(f), - ClusterError::Store(e) => e.fmt(f), - ClusterError::Initialize(e) => e.fmt(f), - ClusterError::Config(e) => e.fmt(f), - ClusterError::Runtime(e) => write!(f, "{:?}", e.lock()), - ClusterError::MsgPackEncode(e) => e.fmt(f), - ClusterError::MsgPackDecode(e) => e.fmt(f), - ClusterError::Client(e) => e.fmt(f), - ClusterError::JoinError(e) => e.fmt(f), - ClusterError::Fatal(e) => e.fmt(f), - ClusterError::WaitError(e) => e.fmt(f), - } +impl std::fmt::Display for SillyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) } } -impl std::error::Error for ClusterError {} - -type ClusterResult = Result; +impl std::error::Error for SillyError {} /// Removes a node from a cluster /// # Errors diff --git a/src/raft/api.rs b/src/raft/api.rs index 3c7559cad9..e390edf4ce 100644 --- a/src/raft/api.rs +++ b/src/raft/api.rs @@ -48,6 +48,8 @@ use std::{collections::BTreeSet, num::ParseIntError, sync::Arc, time::Duration}; use tokio::{task::JoinHandle, time::timeout}; use tremor_common::alias; +use super::store::ResponseError; + pub(crate) type APIRequest = Arc; /// Tyape alias for API results pub type APIResult = Result; @@ -226,7 +228,7 @@ where } /// API errors -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, thiserror::Error)] pub enum APIError { /// We need to send this API request to the leader_url ForwardToLeader { @@ -266,6 +268,8 @@ pub enum APIError { Timeout, /// recv error Recv, + /// Response error + Response(#[from] ResponseError), /// fallback error type Other(String), } @@ -286,6 +290,7 @@ impl IntoResponse for APIError { | APIError::Fatal(_) | APIError::Runtime(_) | APIError::Recv + | APIError::Response(_) | APIError::App(_) => StatusCode::INTERNAL_SERVER_ERROR, APIError::NoLeader | APIError::NoQuorum(_) => StatusCode::SERVICE_UNAVAILABLE, APIError::Timeout => StatusCode::GATEWAY_TIMEOUT, @@ -320,30 +325,23 @@ impl IntoResponse for APIError { } } -#[async_trait::async_trait] -trait ToAPIResult +pub(crate) async fn to_api_result( + r: Result>>, + uri: &Uri, + state: &APIRequest, +) -> APIResult where T: serde::Serialize + serde::Deserialize<'static>, { - async fn to_api_result(self, uri: &Uri, req: &APIRequest) -> APIResult; -} - -#[async_trait::async_trait()] -impl + Send> ToAPIResult - for Result>> -{ - // we need the request context here to construct the redirect url properly - async fn to_api_result(self, uri: &Uri, state: &APIRequest) -> APIResult { - match self { - Ok(response) => Ok(response), - Err(RaftError::APIError(ClientWriteError::ForwardToLeader(e))) => { - forward_to_leader(e, uri, state).await - } - Err(RaftError::APIError(ClientWriteError::ChangeMembershipError(e))) => { - Err(APIError::ChangeMembership(e)) - } - Err(RaftError::Fatal(e)) => Err(APIError::Fatal(e)), + match r { + Ok(response) => Ok(response), + Err(RaftError::APIError(ClientWriteError::ForwardToLeader(e))) => { + forward_to_leader(e, uri, state).await + } + Err(RaftError::APIError(ClientWriteError::ChangeMembershipError(e))) => { + Err(APIError::ChangeMembership(e)) } + Err(RaftError::Fatal(e)) => Err(APIError::Fatal(e)), } } @@ -396,7 +394,8 @@ impl std::fmt::Display for APIError { .field("node_id", node_id) .field("uri", uri) .finish(), - APIError::Other(s) | Self::Store(s) => write!(f, "{s}"), + + APIError::Other(s) => write!(f, "{s}"), APIError::HTTP { message, status } => write!(f, "HTTP {status} {message}"), APIError::Fatal(e) => write!(f, "Fatal Error: {e}"), APIError::ChangeMembership(e) => write!(f, "Error changing cluster membership: {e}"), @@ -407,10 +406,11 @@ impl std::fmt::Display for APIError { APIError::Timeout => write!(f, "Timeout"), APIError::Recv => write!(f, "Recv error"), APIError::NoLeader => write!(f, "No Leader"), + APIError::Store(e) => write!(f, "Store error: {e}"), + APIError::Response(e) => write!(f, "Response error: {e}"), } } } -impl std::error::Error for APIError {} impl From for APIError { fn from(e: store::Error) -> Self { diff --git a/src/raft/api/apps.rs b/src/raft/api/apps.rs index df3910e2c4..5da50709ef 100644 --- a/src/raft/api/apps.rs +++ b/src/raft/api/apps.rs @@ -14,7 +14,7 @@ use crate::{ instance::IntendedState, raft::{ - api::{APIRequest, APIResult, AppError, ArgsError, ToAPIResult, API_WORKER_TIMEOUT}, + api::{APIRequest, APIResult, AppError, ArgsError, API_WORKER_TIMEOUT}, archive::{get_app, TremorAppDef}, store::{ AppsRequest as AppsCmd, FlowInstance, Instances, StateApp, TremorInstanceState, @@ -33,6 +33,8 @@ use std::fmt::Display; use tokio::time::timeout; use tremor_common::alias; +use super::to_api_result; + pub(crate) fn endpoints() -> Router { Router::::new() .route("/", post(install_app).get(list)) @@ -65,12 +67,7 @@ async fn install_app( app, file: file.clone(), }); - state - .raft - .client_write(request) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result(state.raft.client_write(request).await, &uri, &state).await?; Ok(Json(app_id)) } @@ -102,11 +99,7 @@ async fn uninstall_app( app: app_id.clone(), force: false, }); - state - .raft - .client_write(request) - .await - .to_api_result(&uri, &state) + to_api_result(state.raft.client_write(request).await, &uri, &state) .await .map(|d| d.data) } @@ -226,12 +219,7 @@ async fn start( DeploymentType::AllNodes }, }); - state - .raft - .client_write(request) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result(state.raft.client_write(request).await, &uri, &state).await?; Ok(Json(instance_id)) } @@ -288,12 +276,7 @@ async fn manage_instance( instance: instance_id.clone(), state: body_state, }); - state - .raft - .client_write(request) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result(state.raft.client_write(request).await, &uri, &state).await?; Ok(Json(instance_id)) } @@ -318,12 +301,7 @@ async fn stop_instance( return Err(AppError::AppNotFound(app_id).into()); } let request = TremorRequest::Apps(AppsCmd::Undeploy(instance_id.clone())); - state - .raft - .client_write(request) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result(state.raft.client_write(request).await, &uri, &state).await?; Ok(Json(instance_id)) } diff --git a/src/raft/api/cluster.rs b/src/raft/api/cluster.rs index eb10120730..804b5cddc9 100644 --- a/src/raft/api/cluster.rs +++ b/src/raft/api/cluster.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{APIRequest, API_WORKER_TIMEOUT}; +use super::{to_api_result, APIRequest, API_WORKER_TIMEOUT}; use crate::raft::{ - api::{APIError, APIResult, ToAPIResult}, + api::{APIError, APIResult}, node::Addr, store::{NodesRequest, TremorRequest}, NodeId, @@ -83,14 +83,17 @@ async fn add_node( // when this succeeds the local state machine should have the node addr stored, so the network can access it // in order to establish a network connection debug!("node {addr} not yet known to cluster"); - let response = state - .raft - .client_write(TremorRequest::Nodes(NodesRequest::AddNode { - addr: addr.clone(), - })) - .await - .to_api_result(&uri, &state) - .await?; + let response = to_api_result( + state + .raft + .client_write(TremorRequest::Nodes(NodesRequest::AddNode { + addr: addr.clone(), + })) + .await, + &uri, + &state, + ) + .await?; let node_id: NodeId = NodeId::try_from(response.data)?; debug!("node {addr} added to the cluster as node {node_id}"); Ok(Json(node_id)) @@ -118,12 +121,15 @@ async fn remove_node( } // TODO: how to check if the node is a learner? // remove the node metadata from the state machine - state - .raft - .client_write(TremorRequest::Nodes(NodesRequest::RemoveNode { node_id })) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result( + state + .raft + .client_write(TremorRequest::Nodes(NodesRequest::RemoveNode { node_id })) + .await, + &uri, + &state, + ) + .await?; Ok(Json(())) } @@ -152,13 +158,13 @@ async fn add_learner( // add the node as learner debug!("Adding node {node_id} as learner..."); - state - .raft - .add_learner(node_id, node_addr, true) - .await - .to_api_result(&uri, &state) - .await - .map(|d| Json(d.log_id)) + to_api_result( + state.raft.add_learner(node_id, node_addr, true).await, + &uri, + &state, + ) + .await + .map(|d| Json(d.log_id)) } /// Removes a node from **Learners** only @@ -174,12 +180,15 @@ async fn remove_learner( let mut nodes = BTreeSet::new(); nodes.insert(node_id); - state - .raft - .change_membership(ChangeMembers::RemoveNodes(nodes), true) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result( + state + .raft + .change_membership(ChangeMembers::RemoveNodes(nodes), true) + .await, + &uri, + &state, + ) + .await?; Ok(Json(())) } @@ -198,12 +207,12 @@ async fn promote_voter( let value = if membership.insert(node_id) { // only update state if not already in the membership config // this call always returns TremorResponse { value: None } // see store.rs - state - .raft - .change_membership(membership, true) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result( + state.raft.change_membership(membership, true).await, + &uri, + &state, + ) + .await?; Some(node_id) } else { None @@ -222,12 +231,12 @@ async fn demote_voter( let mut membership = timeout(API_WORKER_TIMEOUT, state.raft_manager.get_last_membership()).await??; let value = if membership.remove(&node_id) { - state - .raft - .change_membership(membership, true) - .await - .to_api_result(&uri, &state) - .await?; + to_api_result( + state.raft.change_membership(membership, true).await, + &uri, + &state, + ) + .await?; Some(node_id) } else { None diff --git a/src/raft/archive.rs b/src/raft/archive.rs index 6809a5af6a..63fb21e949 100644 --- a/src/raft/archive.rs +++ b/src/raft/archive.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::errors::Result; +use crate::errors::{ErrorKind, Result}; use sha2::{Digest, Sha256}; use simd_json::OwnedValue; use std::{ @@ -36,6 +36,22 @@ use tremor_script::{ NodeMeta, FN_REGISTRY, }; +#[derive(thiserror::Error, Debug)] +enum Error { + #[error("Failed to get parent dir")] + NoParentDir, + #[error("Failed to get name")] + NoName, + #[error("`app.json` missing")] + SpecMissing, + #[error("`main.troy` missing")] + NoEntrypoint, + #[error("Archive is empty")] + Empty, + #[error("No module name Specified")] + NoModuleName, +} + /// A tremor app flow with defaults and arguments #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct AppFlow { @@ -70,7 +86,7 @@ impl TremorAppDef { pub async fn package(target: &str, entrypoint: &str, name: Option) -> Result<()> { let mut output = file::create(target).await?; let file = PathBuf::from(entrypoint); - let dir = file.parent().ok_or("Failed to get parent dir")?; + let dir = file.parent().ok_or(Error::NoParentDir)?; let path = dir.to_string_lossy(); info!("Adding {path} to path"); Manager::add_path(&path)?; @@ -80,7 +96,7 @@ pub async fn package(target: &str, entrypoint: &str, name: Option) -> Re .and_then(std::ffi::OsStr::to_str) .map(ToString::to_string) }) - .ok_or("Failed to get name")?; + .ok_or(Error::NoName)?; info!("Building archive for {name}"); output .write_all(&build_archive(&name, entrypoint).await?) @@ -101,7 +117,11 @@ pub(crate) fn build_archive_from_source(name: &str, src: &str) -> Result let aggr_reg = tremor_script::registry::aggr(); let mut hl = highlighter::Term::stderr(); - let mut deploy = match Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg) { + let mut deploy = match Deploy::parse( + &src, + &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?, + &aggr_reg, + ) { Ok(deploy) => deploy, Err(e) => { hl.format_error(&e)?; @@ -109,7 +129,7 @@ pub(crate) fn build_archive_from_source(name: &str, src: &str) -> Result } }; let mut other_warnings = BTreeSet::new(); - let reg = &*FN_REGISTRY.read()?; + let reg = &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?; let helper = Helper::new(reg, &aggr_reg); for stmt in &deploy.deploy.stmts { @@ -175,7 +195,13 @@ pub(crate) fn build_archive_from_source(name: &str, src: &str) -> Result // first hash main.troy hasher.update(src.as_bytes()); // then hash all the modules - for aid in MODULES.read()?.modules().iter().map(|m| m.arena_idx) { + for aid in MODULES + .read() + .map_err(|_| ErrorKind::ReadLock)? + .modules() + .iter() + .map(|m| m.arena_idx) + { if let Some(src) = Arena::get(aid)? { hasher.update(src.as_bytes()); } @@ -202,7 +228,8 @@ pub(crate) fn build_archive_from_source(name: &str, src: &str) -> Result ar.append_data(&mut header, "main.troy", src.as_bytes())?; for (id, paths) in MODULES - .read()? + .read() + .map_err(|_| ErrorKind::ReadLock)? .modules() .iter() .map(|m| (m.arena_idx, m.paths())) @@ -231,10 +258,10 @@ pub fn get_app(src: &[u8]) -> Result { let mut ar = Archive::new(src); let mut entries = ar.entries()?; - let mut app = entries.next().ok_or("Invalid archive: empty")??; + let mut app = entries.next().ok_or(Error::Empty)??; if app.path()?.to_string_lossy() != "app.json" { - return Err("Invalid archive: app.json missing".into()); + return Err(Error::SpecMissing.into()); } let mut content = String::new(); app.read_to_string(&mut content)?; @@ -250,19 +277,17 @@ pub fn extract(src: &[u8]) -> Result<(TremorAppDef, Deploy, Vec)> let mut ar = Archive::new(src); let mut entries = ar.entries()?; - let mut app = entries.next().ok_or("Invalid archive: empty")??; + let mut app = entries.next().ok_or(Error::Empty)??; if app.path()?.to_string_lossy() != "app.json" { - return Err("Invalid archive: app.json missing".into()); + return Err(Error::SpecMissing.into()); } let mut content = String::new(); app.read_to_string(&mut content)?; let app: TremorAppDef = serde_json::from_str(&content)?; - let mut main = entries - .next() - .ok_or("Invalid archive: main.troy missing")??; + let mut main = entries.next().ok_or(Error::NoEntrypoint)??; content.clear(); main.read_to_string(&mut content)?; @@ -277,7 +302,7 @@ pub fn extract(src: &[u8]) -> Result<(TremorAppDef, Deploy, Vec)> .iter() .map(|p| p.to_string_lossy().to_string()) .collect(); - let id = module.pop().ok_or("No module name")?; + let id = module.pop().ok_or(Error::NoModuleName)?; let module = NodeId::new(id.clone(), module.clone(), NodeMeta::dummy()); info!("included library: {}", entry.path()?.to_string_lossy()); @@ -287,7 +312,12 @@ pub fn extract(src: &[u8]) -> Result<(TremorAppDef, Deploy, Vec)> modules.insert(module, aid); } let aggr_reg = tremor_script::registry::aggr(); - let deploy = Deploy::parse_with_cache(&main, &*FN_REGISTRY.read()?, &aggr_reg, &modules)?; + let deploy = Deploy::parse_with_cache( + &main, + &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?, + &aggr_reg, + &modules, + )?; let mut aids = modules.values().collect::>(); aids.push(deploy.aid); Ok((app, deploy, aids)) diff --git a/src/raft/manager.rs b/src/raft/manager.rs index 6bd95217ae..c1ce933857 100644 --- a/src/raft/manager.rs +++ b/src/raft/manager.rs @@ -12,22 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeSet, HashMap}; +use std::{ + collections::{BTreeSet, HashMap}, + convert::Into, +}; use super::{ node::Addr, store::{StateApp, TremorRequest, TremorSet}, - TremorRaftConfig, TremorRaftImpl, + ClusterError, TremorRaftConfig, TremorRaftImpl, }; -use crate::channel::{oneshot, Sender}; use crate::raft::api::APIStoreReq; +use crate::raft::NodeId; use crate::Result; use crate::{ - errors::{Error, ErrorKind}, - raft::NodeId, + channel::{bounded, oneshot, OneShotSender, Sender}, + connectors::prelude::Receiver, }; use openraft::{ - error::{CheckIsLeaderError, ForwardToLeader, RaftError}, + error::{CheckIsLeaderError, Fatal, ForwardToLeader, RaftError}, raft::ClientWriteResponse, }; use simd_json::OwnedValue; @@ -39,6 +42,148 @@ pub(crate) struct Cluster { raft: Option, sender: Option>, } +#[derive(Clone, Default, Debug)] +pub(crate) struct ClusterInterface { + node_id: NodeId, + store_sender: Option>, + cluster_sender: Option>, +} + +type IsLeaderResult = std::result::Result<(), RaftError>>; +enum IFRequest { + IsLeader(OneShotSender), + SetKeyLocal(TremorSet, OneShotSender>>), +} +async fn cluster_interface(raft: TremorRaftImpl, mut rx: Receiver) { + while let Some(msg) = rx.recv().await { + match msg { + IFRequest::IsLeader(tx) => { + let res = raft.is_leader().await; + if tx.send(res).is_err() { + error!("Error sending response to API"); + } + } + IFRequest::SetKeyLocal(set, tx) => { + let res = match raft.client_write(set.into()).await { + Ok(v) => v.data.into_kv_value().map_err(anyhow::Error::from), + Err(e) => Err(e.into()), + }; + if tx.send(res).is_err() { + error!("Error sending response to API"); + } + } + } + } +} + +impl ClusterInterface { + pub(crate) fn id(&self) -> NodeId { + self.node_id + } + async fn send_store(&self, command: APIStoreReq) -> Result<()> { + self.store_sender + .as_ref() + .ok_or(ClusterError::RaftNotRunning)? + .send(command) + .await?; + Ok(()) + } + async fn send_cluster(&self, command: IFRequest) -> Result<()> { + self.cluster_sender + .as_ref() + .ok_or(ClusterError::RaftNotRunning)? + .send(command) + .await?; + Ok(()) + } + pub(crate) async fn get_last_membership(&self) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetLastMembership(tx); + self.send_store(command).await?; + Ok(rx.await?) + } + pub(crate) async fn is_leader(&self) -> IsLeaderResult { + let (tx, rx) = oneshot(); + let command = IFRequest::IsLeader(tx); + self.send_cluster(command) + .await + .map_err(|_| RaftError::Fatal(Fatal::Stopped))?; + rx.await.map_err(|_| RaftError::Fatal(Fatal::Stopped))? + } + + // kv + pub(crate) async fn kv_set(&self, key: String, mut value: Vec) -> Result> { + match self.is_leader().await { + Ok(()) => self.kv_set_local(key, value).await, + Err(RaftError::APIError(CheckIsLeaderError::ForwardToLeader(ForwardToLeader { + leader_node: Some(n), + .. + }))) => { + let client = crate::raft::api::client::Tremor::new(n.api())?; + // TODO: there should be a better way to forward then the client + Ok(simd_json::to_vec( + &client + .write(&crate::raft::api::kv::KVSet { + key, + value: simd_json::from_slice(&mut value)?, + }) + .await?, + )?) + } + Err(e) => Err(e.into()), + } + } + + pub(crate) async fn kv_set_local(&self, key: String, value: Vec) -> Result> { + let (tx, rx) = oneshot(); + let command = IFRequest::SetKeyLocal(TremorSet { key, value }, tx); + self.send_cluster(command).await?; + rx.await? + } + + pub(crate) async fn kv_get(&self, key: String) -> Result> { + match self.is_leader().await { + Ok(()) => self.kv_get_local(key).await, + Err(RaftError::APIError(CheckIsLeaderError::ForwardToLeader(ForwardToLeader { + leader_node: Some(n), + .. + }))) => { + let client = crate::raft::api::client::Tremor::new(n.api())?; + let res = client.read(&key).await; + match res { + Ok(v) => Ok(Some(v)), + Err(e) if e.is_not_found() => Ok(None), + Err(e) => Err(e.into()), + } + } + Err(e) => Err(e.into()), + } + } + pub(crate) async fn kv_get_local(&self, key: String) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::KVGet(key, tx); + self.send_store(command).await?; + Ok(rx + .await? + .map(|mut v| simd_json::from_slice(&mut v)) + .transpose()?) + } +} + +impl From for ClusterInterface { + fn from(c: Cluster) -> Self { + let cluster_sender = c.raft.map(|r| { + let (tx, rx) = bounded(1042); + tokio::spawn(cluster_interface(r, rx)); + tx + }); + Self { + cluster_sender, + node_id: c.node_id, + store_sender: c.sender, + } + } +} impl std::fmt::Debug for Cluster { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -51,23 +196,17 @@ impl std::fmt::Debug for Cluster { } impl Cluster { - pub(crate) fn id(&self) -> NodeId { - self.node_id - } async fn send(&self, command: APIStoreReq) -> Result<()> { self.sender .as_ref() - .ok_or(crate::errors::ErrorKind::RaftNotRunning)? + .ok_or(ClusterError::RaftNotRunning)? .send(command) .await?; Ok(()) } fn raft(&self) -> Result<&TremorRaftImpl> { - Ok(self - .raft - .as_ref() - .ok_or(crate::errors::ErrorKind::RaftNotRunning)?) + Ok(self.raft.as_ref().ok_or(ClusterError::RaftNotRunning)?) } async fn client_write(&self, command: T) -> Result> @@ -77,8 +216,11 @@ impl Cluster { Ok(self.raft()?.client_write(command.into()).await?) } - pub(crate) async fn is_leader(&self) -> Result<()> { - Ok(self.raft()?.is_leader().await?) + pub(crate) async fn is_leader(&self) -> IsLeaderResult { + self.raft() + .map_err(|_| RaftError::Fatal(Fatal::Stopped))? + .is_leader() + .await } pub(crate) fn new(node_id: NodeId, sender: Sender, raft: TremorRaftImpl) -> Self { @@ -136,15 +278,10 @@ impl Cluster { pub(crate) async fn kv_set(&self, key: String, mut value: Vec) -> Result> { match self.is_leader().await { Ok(()) => self.kv_set_local(key, value).await, - Err(Error( - ErrorKind::CheckIsLeaderError(RaftError::APIError( - CheckIsLeaderError::ForwardToLeader(ForwardToLeader { - leader_node: Some(n), - .. - }), - )), - _, - )) => { + Err(RaftError::APIError(CheckIsLeaderError::ForwardToLeader(ForwardToLeader { + leader_node: Some(n), + .. + }))) => { let client = crate::raft::api::client::Tremor::new(n.api())?; // TODO: there should be a better way to forward then the client Ok(simd_json::to_vec( @@ -156,26 +293,21 @@ impl Cluster { .await?, )?) } - Err(e) => Err(e), + Err(e) => Err(e.into()), } } pub(crate) async fn kv_set_local(&self, key: String, value: Vec) -> Result> { let tremor_res = self.client_write(TremorSet { key, value }).await?; - tremor_res.data.into_kv_value() + Ok(tremor_res.data.into_kv_value()?) } pub(crate) async fn kv_get(&self, key: String) -> Result> { match self.is_leader().await { Ok(()) => self.kv_get_local(key).await, - Err(Error( - ErrorKind::CheckIsLeaderError(RaftError::APIError( - CheckIsLeaderError::ForwardToLeader(ForwardToLeader { - leader_node: Some(n), - .. - }), - )), - _, - )) => { + Err(RaftError::APIError(CheckIsLeaderError::ForwardToLeader(ForwardToLeader { + leader_node: Some(n), + .. + }))) => { let client = crate::raft::api::client::Tremor::new(n.api())?; let res = client.read(&key).await; match res { @@ -184,7 +316,7 @@ impl Cluster { Err(e) => Err(e.into()), } } - Err(e) => Err(e), + Err(e) => Err(e.into()), } } pub(crate) async fn kv_get_local(&self, key: String) -> Result> { diff --git a/src/raft/node.rs b/src/raft/node.rs index 46943dbb49..75297c693c 100644 --- a/src/raft/node.rs +++ b/src/raft/node.rs @@ -15,7 +15,7 @@ //! The entirety of a cluster node as a struct use crate::{ channel::{bounded, Sender}, - errors::Result, + errors::{ErrorKind, Result}, qsize, raft::{ api::{self, ServerState}, @@ -56,7 +56,7 @@ impl ClusterNodeKillSwitch { pub fn stop(&self, mode: ShutdownMode) -> ClusterResult<()> { self.sender .try_send(mode) - .map_err(|_| ClusterError::from("Error stopping cluster node")) + .map_err(|_| ClusterError::Shutdown.into()) } } @@ -100,9 +100,13 @@ impl Running { let http_api_addr = server_state.addr().api().to_string(); let app = api::endpoints().with_state(server_state.clone()); - let http_api_server = - axum::Server::bind(&http_api_addr.to_socket_addrs()?.next().ok_or("badaddr")?) - .serve(app.into_make_service()); + let http_api_server = axum::Server::bind( + &http_api_addr + .to_socket_addrs()? + .next() + .ok_or(ClusterError::BadAddr)?, + ) + .serve(app.into_make_service()); let run_handle = task::spawn(async move { let mut tcp_future = Box::pin( @@ -131,7 +135,7 @@ impl Running { warn!("[Node {node_id}] TCP cluster API shutdown."); // Important: this will free and drop the store and thus the rocksdb api_worker_handle.abort(); - raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + raft.shutdown().await.map_err(|_| ClusterError::Shutdown)?; runtime.stop(ShutdownMode::Graceful).await?; runtime_future.await??; } @@ -141,7 +145,7 @@ impl Running { } // Important: this will free and drop the store and thus the rocksdb api_worker_handle.abort(); - raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + raft.shutdown().await.map_err(|_| ClusterError::Shutdown)?; runtime.stop(ShutdownMode::Graceful).await?; runtime_future.await??; @@ -153,7 +157,7 @@ impl Running { // Important: this will free and drop the store and thus the rocksdb api_worker_handle.abort(); // runtime is already down, we only need to stop local raft - raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + raft.shutdown().await.map_err(|_| ClusterError::Shutdown)?; } shutdown_mode = kill_switch_future => { let shutdown_mode = shutdown_mode.unwrap_or(ShutdownMode::Forceful); @@ -161,7 +165,7 @@ impl Running { // Important: this will free and drop the store and thus the rocksdb api_worker_handle.abort(); // tcp and http api stopped listening as we don't poll them no more - raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + raft.shutdown().await.map_err(|_| ClusterError::Shutdown)?; info!("[Node {node_id}] Raft engine did stop."); info!("[Node {node_id}] Stopping the Tremor runtime..."); runtime.stop(shutdown_mode).await?; @@ -171,7 +175,7 @@ impl Running { } info!("[Node {node_id}] Tremor cluster node stopped"); - Ok::<(), ClusterError>(()) + Ok::<(), anyhow::Error>(()) }); Ok(Self { @@ -298,7 +302,7 @@ impl Node { *(runtime .cluster_manager .write() - .map_err(|_| "Failed to set world manager")?) = Some(manager); + .map_err(|_| ErrorKind::WriteLock)?) = Some(manager); let (api_worker_handle, server_state) = api::initialize( node_id, addr, @@ -328,9 +332,7 @@ impl Node { promote_to_voter: bool, ) -> ClusterResult { if endpoints.is_empty() { - return Err(ClusterError::Other( - "No join endpoints provided".to_string(), - )); + return Err(ClusterError::NoEndpoints.into()); } // for now we infinitely try to join until it succeeds @@ -377,7 +379,7 @@ impl Node { *(runtime .cluster_manager .write() - .map_err(|_| "Failed to set world manager")?) = Some(manager); + .map_err(|_| ErrorKind::WriteLock)?) = Some(manager); let (api_worker_handle, server_state) = api::initialize(node_id, addr, raft.clone(), store, store_tx, store_rx); let running = Running::start( @@ -439,7 +441,7 @@ impl Node { *(runtime .cluster_manager .write() - .map_err(|_| "Failed to set world manager")?) = Some(manager); + .map_err(|_| ErrorKind::WriteLock)?) = Some(manager); let mut nodes = BTreeMap::new(); nodes.insert(node_id, addr.clone()); // this is the crucial bootstrapping step @@ -474,9 +476,10 @@ impl Node { ) .await } - Err(e) => Err(ClusterError::Other(format!( - "Error adding myself to the bootstrapped cluster: {e}" - ))), + Err(e) => { + Err(anyhow::Error::from(e) + .context("Error adding myself to the bootstrapped cluster")) + } } } } diff --git a/src/raft/store.rs b/src/raft/store.rs index 9b89ffc691..5c44bac6fb 100644 --- a/src/raft/store.rs +++ b/src/raft/store.rs @@ -22,9 +22,10 @@ use crate::{ archive::TremorAppDef, node::Addr, store::statemachine::{SerializableTremorStateMachine, TremorStateMachine}, - ClusterError, NodeId, TremorRaftConfig, + NodeId, TremorRaftConfig, }, system::{flow::DeploymentType, Runtime}, + Result, }; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use openraft::{ @@ -39,7 +40,6 @@ use redb::{ use serde::{Deserialize, Serialize}; use simd_json::OwnedValue; use std::{ - error::Error as StdError, fmt::{Debug, Display, Formatter}, io::Cursor, ops::RangeBounds, @@ -50,6 +50,8 @@ use std::{ use tokio::sync::RwLock; use tremor_common::alias; +use super::SillyError; + /// Kv Operation #[derive(Serialize, Deserialize, Debug, Clone)] pub enum KvRequest { @@ -202,41 +204,60 @@ pub enum TremorResponse { AppFlowInstanceId(alias::Flow), } +/// Error for a raft response +#[derive(Debug, Clone, thiserror::Error, Serialize)] +pub enum ResponseError { + /// Not a kv value + #[error("Not a kv value")] + NotKv, + /// Not an app id + #[error("Not an app id")] + NotAppId, + /// Not a node id + #[error("Not a node id")] + NotNodeId, + /// Not an app flow instance id + #[error("Not an app flow instance id")] + NotAppFlowInstanceId, +} + +type ResponseResult = std::result::Result; + impl TremorResponse { - pub(crate) fn into_kv_value(self) -> crate::Result> { + pub(crate) fn into_kv_value(self) -> ResponseResult> { match self { TremorResponse::KvValue(v) => Ok(v), - _ => Err(RuntimeError::from("Not a kv value")), + _ => Err(ResponseError::NotKv), } } } impl TryFrom for alias::App { - type Error = RuntimeError; - fn try_from(response: TremorResponse) -> crate::Result { + type Error = ResponseError; + fn try_from(response: TremorResponse) -> ResponseResult { match response { TremorResponse::AppId(id) => Ok(id), - _ => Err(RuntimeError::from("Not an app id")), + _ => Err(ResponseError::NotAppId), } } } impl TryFrom for NodeId { - type Error = RuntimeError; - fn try_from(response: TremorResponse) -> crate::Result { + type Error = ResponseError; + fn try_from(response: TremorResponse) -> ResponseResult { match response { TremorResponse::NodeId(id) => Ok(id), - _ => Err(RuntimeError::from("Not a node id")), + _ => Err(ResponseError::NotNodeId), } } } impl TryFrom for alias::Flow { - type Error = RuntimeError; - fn try_from(response: TremorResponse) -> crate::Result { + type Error = ResponseError; + fn try_from(response: TremorResponse) -> ResponseResult { match response { TremorResponse::AppFlowInstanceId(id) => Ok(id), - _ => Err(RuntimeError::from("Not an app flow instance id")), + _ => Err(ResponseError::NotAppFlowInstanceId), } } } @@ -260,7 +281,7 @@ pub struct Store { db: Arc, } -type StorageResult = Result>; +type StorageResult = anyhow::Result>; /// converts an id to a byte vector for storing in the database. /// Note that we're using big endian encoding to ensure correct sorting of keys @@ -278,10 +299,14 @@ fn bin_to_id(buf: &[u8]) -> Result { } /// The Raft storage error -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { - /// Missing column family - MissingCf(&'static str), + /// invalid cluster store, node_id missing + // #[error("invalid cluster store, node_id missing")] + MissingNodeId, + // #[error("invalid cluster store, node_addr missing")] + /// invalid cluster store, node_addr missing + MissingNodeAddr, /// Invalid utf8 Utf8(FromUtf8Error), /// Invalid utf8 @@ -403,11 +428,11 @@ impl From> for Error { } } -impl StdError for Error {} impl Display for Error { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - Error::MissingCf(cf) => write!(f, "missing column family: `{cf}`"), + Error::MissingNodeId => write!(f, "missing node id"), + Error::MissingNodeAddr => write!(f, "missing node addr"), Error::Utf8(e) => write!(f, "invalid utf8: {e}"), Error::StrUtf8(e) => write!(f, "invalid utf8: {e}"), @@ -438,17 +463,37 @@ impl Display for Error { } } -fn w_err(e: impl StdError + 'static) -> StorageError { - StorageIOError::new(ErrorSubject::Store, ErrorVerb::Write, AnyError::new(&e)).into() -} -fn r_err(e: impl StdError + 'static) -> StorageError { - StorageIOError::new(ErrorSubject::Store, ErrorVerb::Read, AnyError::new(&e)).into() -} -fn logs_r_err(e: impl StdError + 'static) -> StorageError { - StorageIOError::new(ErrorSubject::Logs, ErrorVerb::Read, AnyError::new(&e)).into() -} -fn logs_w_err(e: impl StdError + 'static) -> StorageError { - StorageIOError::new(ErrorSubject::Logs, ErrorVerb::Read, AnyError::new(&e)).into() +fn w_err(e: impl Into) -> StorageError { + StorageIOError::new( + ErrorSubject::Store, + ErrorVerb::Write, + AnyError::new(&SillyError(e.into())), + ) + .into() +} +fn r_err(e: impl Into) -> StorageError { + StorageIOError::new( + ErrorSubject::Store, + ErrorVerb::Read, + AnyError::new(&SillyError(e.into())), + ) + .into() +} +fn logs_r_err(e: impl Into) -> StorageError { + StorageIOError::new( + ErrorSubject::Logs, + ErrorVerb::Read, + AnyError::new(&SillyError(e.into())), + ) + .into() +} +fn logs_w_err(e: impl Into) -> StorageError { + StorageIOError::new( + ErrorSubject::Logs, + ErrorVerb::Read, + AnyError::new(&SillyError(e.into())), + ) + .into() } impl From for StorageError { @@ -477,6 +522,14 @@ impl Store { where T: for<'de> serde::Deserialize<'de>, { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(STORE).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; let read_txn = self.db.begin_read().map_err(r_err)?; let table = read_txn.open_table(STORE).map_err(r_err)?; let r = table.get(key).map_err(r_err)?; @@ -528,6 +581,14 @@ impl RaftLogReader for Store { &mut self, range: RB, ) -> StorageResult>> { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(LOGS).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; let read_txn = self.db.begin_read().map_err(r_err)?; let table = read_txn.open_table(LOGS).map_err(r_err)?; @@ -770,6 +831,15 @@ impl RaftStorage for Store { } async fn get_log_state(&mut self) -> StorageResult> { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(LOGS).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; + let read_txn = self.db.begin_read().map_err(r_err)?; let table = read_txn.open_table(LOGS).map_err(r_err)?; let last = table @@ -833,7 +903,7 @@ impl Store { addr: &Addr, db_path: P, world: Runtime, - ) -> Result { + ) -> Result { let db = Self::init_db(db_path)?; Self::set_self(&db, node_id, addr)?; @@ -850,13 +920,13 @@ impl Store { /// /// This function is safe and never cleans up or resets the current state, /// but creates a new db if there is none. - pub(crate) fn init_db>(db_path: P) -> Result { - let db = Database::create(db_path).map_err(ClusterError::Database)?; + pub(crate) fn init_db>(db_path: P) -> Result { + let db = Database::create(db_path)?; Ok(db) } /// loading constructor - loading the given database - pub(crate) async fn load(db: Arc, world: Runtime) -> Result { + pub(crate) async fn load(db: Arc, world: Runtime) -> Result { let state_machine = Arc::new(RwLock::new( TremorStateMachine::new(db.clone(), world.clone()) .await @@ -867,11 +937,7 @@ impl Store { } /// Store the information about the current node itself in the `db` - fn set_self( - db: &Database, - node_id: crate::raft::NodeId, - addr: &Addr, - ) -> Result<(), ClusterError> { + fn set_self(db: &Database, node_id: crate::raft::NodeId, addr: &Addr) -> Result<()> { let node_id_bytes = id_to_bin(node_id)?; let addr_bytes = rmp_serde::to_vec(addr)?; let write_txn = db.begin_write()?; @@ -884,16 +950,25 @@ impl Store { Ok(()) } - pub(crate) fn get_self(db: &Database) -> Result<(crate::raft::NodeId, Addr), ClusterError> { - let id = Self::get_self_node_id(db)?.ok_or("invalid cluster store, node_id missing")?; - let addr = Self::get_self_addr(db)?.ok_or("invalid cluster store, node_addr missing")?; + pub(crate) fn get_self(db: &Database) -> Result<(crate::raft::NodeId, Addr)> { + let id = Self::get_self_node_id(db)?.ok_or(Error::MissingNodeId)?; + let addr = Self::get_self_addr(db)?.ok_or(Error::MissingNodeAddr)?; Ok((id, addr)) } /// # Errors /// if the store fails to read the RPC address - pub fn get_self_addr(db: &Database) -> Result, Error> { + pub fn get_self_addr(db: &Database) -> Result> { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = db.begin_write()?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(SYSTEM).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; + let read_txn = db.begin_read().map_err(r_err)?; let table = read_txn.open_table(SYSTEM).map_err(r_err)?; @@ -903,12 +978,21 @@ impl Store { /// # Errors /// if the store fails to read the node id - pub fn get_self_node_id(db: &Database) -> Result, Error> { + pub fn get_self_node_id(db: &Database) -> Result> { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = db.begin_write()?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(SYSTEM).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; + let read_txn = db.begin_read().map_err(r_err)?; let table = read_txn.open_table(SYSTEM).map_err(r_err)?; let r = table.get(Store::NODE_ID).map_err(r_err)?; - r.map(|v| bin_to_id(&v.value())).transpose() + Ok(r.map(|v| bin_to_id(&v.value())).transpose()?) } } @@ -922,13 +1006,13 @@ mod tests { // fn init_db_is_idempotent() -> ClusterResult<()> { // let dir = tempfile::tempdir()?; // let db = Store::init_db(dir.path())?; - // let handle = db.cf_handle(Store::STORE).ok_or("no data")?; + // let handle = db.cf_handle(Store::STORE).expect("no data"); // let data = vec![1_u8, 2_u8, 3_u8]; // db.put_cf(handle, "node_id", data.clone())?; // drop(db); // let db2 = Store::init_db(dir.path())?; - // let handle2 = db2.cf_handle(Store::STORE).ok_or("no data")?; + // let handle2 = db2.cf_handle(Store::STORE).expect("no data"); // let res2 = db2.get_cf(handle2, "node_id")?; // assert_eq!(Some(data), res2); // Ok(()) diff --git a/src/raft/store/statemachine.rs b/src/raft/store/statemachine.rs index 6941095319..374fb5b9a8 100644 --- a/src/raft/store/statemachine.rs +++ b/src/raft/store/statemachine.rs @@ -13,8 +13,12 @@ // limitations under the License. use crate::{ - raft::store::{ - self, statemachine::nodes::NodesStateMachine, StorageResult, TremorRequest, TremorResponse, + raft::{ + store::{ + self, statemachine::nodes::NodesStateMachine, StorageResult, TremorRequest, + TremorResponse, + }, + SillyError, }, system::Runtime, }; @@ -23,7 +27,7 @@ use openraft::{ }; use redb::{Database, ReadableTable}; use serde::{Deserialize, Serialize}; -use std::{error::Error, fmt::Debug, sync::Arc}; +use std::{fmt::Debug, sync::Arc}; use super::STATE_MACHINE; @@ -34,27 +38,27 @@ mod nodes; // kv state machine mod kv; -pub(crate) fn r_err(e: E) -> StorageError { +pub(crate) fn r_err>(e: E) -> StorageError { StorageIOError::new( ErrorSubject::StateMachine, ErrorVerb::Read, - AnyError::new(&e), + AnyError::new(&SillyError(e.into())), ) .into() } -pub(crate) fn w_err(e: E) -> StorageError { +pub(crate) fn w_err>(e: E) -> StorageError { StorageIOError::new( ErrorSubject::StateMachine, ErrorVerb::Write, - AnyError::new(&e), + AnyError::new(&SillyError(e.into())), ) .into() } -fn d_err(e: E) -> StorageError { +fn d_err>(e: E) -> StorageError { StorageIOError::new( ErrorSubject::StateMachine, ErrorVerb::Delete, - AnyError::new(&e), + AnyError::new(&SillyError(e.into())), ) .into() } @@ -164,6 +168,15 @@ impl TremorStateMachine { where T: for<'de> serde::Deserialize<'de>, { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(STATE_MACHINE).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; + let read_txn = self.db.begin_read().map_err(r_err)?; let table = read_txn.open_table(STATE_MACHINE).map_err(r_err)?; diff --git a/src/raft/store/statemachine/apps.rs b/src/raft/store/statemachine/apps.rs index ede6bddeea..403af32703 100644 --- a/src/raft/store/statemachine/apps.rs +++ b/src/raft/store/statemachine/apps.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::{ + errors::ErrorKind, instance::IntendedState, raft::{ archive::{extract, get_app, TremorAppDef}, @@ -90,6 +91,16 @@ impl RaftStateMachine for AppsStateMachine { apps: HashMap::new(), world: world.clone(), }; + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = db.begin_write()?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(APPS).map_err(w_err)?; + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(INSTANCES).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; let read_txn = db.begin_read()?; let apps = read_txn.open_table(APPS)?; @@ -99,6 +110,7 @@ impl RaftStateMachine for AppsStateMachine { me.load_archive(archive.value()) .map_err(|e| store::Error::Other(Box::new(e)))?; } + let instances = read_txn.open_table(INSTANCES)?; // load instances @@ -220,6 +232,14 @@ impl RaftStateMachine for AppsStateMachine { } fn as_snapshot(&self) -> StorageResult { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(APPS).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; let read_txn = self.db.begin_read().map_err(r_err)?; let apps = read_txn.open_table(APPS).map_err(r_err)?; let archives = apps @@ -300,7 +320,7 @@ impl AppsStateMachine { } write_txn.commit().map_err(w_err)?; - let app = StateApp { + let app: StateApp = StateApp { app, main, arena_indices, @@ -356,7 +376,10 @@ impl AppsStateMachine { let fake_aggr_reg = AggrRegistry::default(); { - let reg = &*FN_REGISTRY.read().map_err(w_err)?; + let reg = &*FN_REGISTRY + .read() + .map_err(|_| ErrorKind::ReadLock) + .map_err(w_err)?; let mut helper = Helper::new(reg, &fake_aggr_reg); Optimizer::new(&helper) .walk_flow_definition(&mut defn) diff --git a/src/raft/store/statemachine/kv.rs b/src/raft/store/statemachine/kv.rs index fbcc052c57..658719a94c 100644 --- a/src/raft/store/statemachine/kv.rs +++ b/src/raft/store/statemachine/kv.rs @@ -52,6 +52,14 @@ impl KvStateMachine { /// try to obtain the value at the given `key`. /// Returns `Ok(None)` if there is no value for that key. pub(crate) fn get(&self, key: &str) -> StorageResult>> { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(DATA).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; let read_txn = self.db.begin_read().map_err(r_err)?; let table = read_txn.open_table(DATA).map_err(r_err)?; table @@ -85,6 +93,15 @@ impl RaftStateMachine for KvStateMachine { } fn as_snapshot(&self) -> StorageResult { + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = self.db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(DATA).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; + let read_tnx = self.db.begin_read().map_err(w_err)?; let table = read_tnx.open_table(DATA).map_err(w_err)?; let data = table diff --git a/src/raft/store/statemachine/nodes.rs b/src/raft/store/statemachine/nodes.rs index 2dd744be53..d5159c53da 100644 --- a/src/raft/store/statemachine/nodes.rs +++ b/src/raft/store/statemachine/nodes.rs @@ -46,7 +46,7 @@ pub(crate) struct NodesStateMachine { } impl NodesStateMachine { - const NEXT_NODE_ID: &str = "next_node_id"; + const NEXT_NODE_ID: &'static str = "next_node_id"; } #[async_trait::async_trait] @@ -56,6 +56,17 @@ impl RaftStateMachine for NodesStateMachine { Self: std::marker::Sized, { // load known nodes + + // We need to use a write transaction despite just wanting a read transaction due to + // https://github.com/cberner/redb/issues/711 + let bug_fix_txn = db.begin_write().map_err(w_err)?; + { + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(SYSTEM).map_err(w_err)?; + // ALLOW: this is just a workaround + let _argh = bug_fix_txn.open_table(NODES).map_err(w_err)?; + } + bug_fix_txn.commit().map_err(w_err)?; let read_txn = db.begin_read().map_err(r_err)?; let table = read_txn.open_table(SYSTEM).map_err(r_err)?; @@ -66,6 +77,7 @@ impl RaftStateMachine for NodesStateMachine { debug!("No next_node_id stored in db, starting from 0"); 0 }; + let table = read_txn.open_table(NODES).map_err(r_err)?; let known_nodes = table diff --git a/src/raft/test.rs b/src/raft/test.rs index 037e31efc6..e72a4f12ee 100644 --- a/src/raft/test.rs +++ b/src/raft/test.rs @@ -70,7 +70,6 @@ impl TestNode { }) } - #[allow(dead_code)] async fn just_start(path: impl AsRef) -> ClusterResult { let addr = free_node_addr().await?; let running = Node::load_from_store(path, raft_config()?).await?; @@ -82,7 +81,6 @@ impl TestNode { }) } - #[allow(dead_code)] async fn join_as_learner(path: impl AsRef, join_addr: &Addr) -> ClusterResult { let addr = free_node_addr().await?; let mut node = Node::new(path, raft_config()?); @@ -115,9 +113,9 @@ async fn cluster_join_test() -> ClusterResult<()> { let dir0 = tempfile::tempdir()?; let dir1 = tempfile::tempdir()?; let dir2 = tempfile::tempdir()?; - let node0 = TestNode::bootstrap(dir0.path()).await?; - let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; - let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + let node0 = TestNode::bootstrap(dir0.path().join("db")).await?; + let node1 = TestNode::start_and_join(dir1.path().join("db"), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path().join("db"), &node1.addr).await?; // all see the same leader let client0 = node0.client(); @@ -162,9 +160,9 @@ async fn kill_and_restart_voter() -> ClusterResult<()> { let dir1 = tempfile::tempdir()?; let dir2 = tempfile::tempdir()?; - let node0 = TestNode::bootstrap(dir0.path()).await?; - let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; - let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + let node0 = TestNode::bootstrap(dir0.path().join("db")).await?; + let node1 = TestNode::start_and_join(dir1.path().join("db"), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path().join("db"), &node1.addr).await?; let client0 = node0.client(); let metrics = client0.metrics().await?; @@ -181,7 +179,7 @@ async fn kill_and_restart_voter() -> ClusterResult<()> { tokio::time::sleep(Duration::from_millis(500)).await; // restart the node - let node1 = TestNode::just_start(dir1.path()).await?; + let node1 = TestNode::just_start(dir1.path().join("db")).await?; // check that the leader is available // TODO: solidify to guard against timing issues diff --git a/src/raft/test/learner.rs b/src/raft/test/learner.rs index 6ed7bc24b1..aa0779e47c 100644 --- a/src/raft/test/learner.rs +++ b/src/raft/test/learner.rs @@ -25,9 +25,9 @@ async fn add_learner_test() -> ClusterResult<()> { let dir1 = tempfile::tempdir()?; let dir2 = tempfile::tempdir()?; let dir3 = tempfile::tempdir()?; - let node0 = TestNode::bootstrap(dir0.path()).await?; - let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; - let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + let node0 = TestNode::bootstrap(dir0.path().join("db")).await?; + let node1 = TestNode::start_and_join(dir1.path().join("db"), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path().join("db"), &node1.addr).await?; let client0 = node0.client(); let metrics = client0.metrics().await?; let members = metrics @@ -38,7 +38,7 @@ async fn add_learner_test() -> ClusterResult<()> { .expect("No nodes in membership config"); assert_eq!(3, members.len()); - let learner_node = TestNode::join_as_learner(dir3.path(), &node0.addr).await?; + let learner_node = TestNode::join_as_learner(dir3.path().join("db"), &node0.addr).await?; let (learner_node_id, learner_addr) = learner_node.running.node_data(); // learner is known to the cluster let nodemap = client0.get_nodes().await?; @@ -82,9 +82,9 @@ async fn learner_runs_app() -> ClusterResult<()> { let dir1 = tempfile::tempdir()?; let dir2 = tempfile::tempdir()?; let dir3 = tempfile::tempdir()?; - let node0 = TestNode::bootstrap(dir0.path()).await?; - let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; - let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + let node0 = TestNode::bootstrap(dir0.path().join("db")).await?; + let node1 = TestNode::start_and_join(dir1.path().join("db"), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path().join("db"), &node1.addr).await?; let client0 = node0.client(); let metrics = client0.metrics().await?; let members = metrics @@ -95,7 +95,7 @@ async fn learner_runs_app() -> ClusterResult<()> { .expect("No nodes in membership config"); assert_eq!(3, members.len()); - let learner_node = TestNode::join_as_learner(dir3.path(), &node0.addr).await?; + let learner_node = TestNode::join_as_learner(dir3.path().join("db"), &node0.addr).await?; let (_learner_node_id, _learner_addr) = learner_node.running.node_data(); let tmpfile = tempfile::NamedTempFile::new()?; let out_path = tmpfile.into_temp_path(); diff --git a/src/system.rs b/src/system.rs index d7088fee01..ea491156a5 100644 --- a/src/system.rs +++ b/src/system.rs @@ -26,9 +26,10 @@ use self::flow::{DeploymentType, Flow}; use crate::{ channel::{oneshot, Sender}, connectors, - errors::{Error, Kind as ErrorKind, Result}, + errors::{ErrorKind, Result}, instance::IntendedState as IntendedInstanceState, - log_error, raft, + log_error, + raft::{self, ClusterInterface}, }; use tokio::{sync::oneshot, task::JoinHandle, time::timeout}; use tremor_common::alias; @@ -126,13 +127,18 @@ pub struct Runtime { impl Runtime { // pub(crate) fn get_manager(&self) -> Result { // self.cluster_manager - // .read()? + // .read().map_err(|_| ErrorKind::ReadLock)? // .as_ref() // .cloned() // .ok_or_else(|| ErrorKind::RaftNotRunning.into()) // } pub(crate) fn maybe_get_manager(&self) -> Result> { - Ok(self.cluster_manager.read()?.as_ref().cloned()) + Ok(self + .cluster_manager + .read() + .map_err(|_| ErrorKind::ReadLock)? + .as_ref() + .cloned()) } /// Wait for the cluster to be available @@ -156,7 +162,11 @@ impl Runtime { let aggr_reg = tremor_script::registry::aggr(); - let deployable = Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg); + let deployable = Deploy::parse( + &src, + &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?, + &aggr_reg, + ); let mut h = highlighter::Term::stderr(); let deployable = match deployable { Ok(deployable) => { @@ -166,7 +176,9 @@ impl Runtime { Err(e) => { log_error!(h.format_error(&e), "Error: {e}"); - return Err(format!("failed to load troy file: {src}").into()); + return Err( + anyhow::Error::from(e).context(format!("failed to load troy file: {src}")) + ); } }; @@ -205,30 +217,41 @@ impl Runtime { app: app_id, flow: Box::new(flow.clone()), sender: tx, - raft: self.maybe_get_manager()?.unwrap_or_default(), + raft: self + .maybe_get_manager()? + .map(ClusterInterface::from) + .unwrap_or_default(), deployment_type, }) .await?; match rx.await? { Err(e) => { - let err_str = match e { - Error( - ErrorKind::Script(e) - | ErrorKind::Pipeline(tremor_pipeline::errors::ErrorKind::Script(e)), - _, - ) => { - let mut h = crate::ToStringHighlighter::new(); - h.format_error(&tremor_script::errors::Error::from(e))?; - h.finalize()?; - h.to_string() + let err_str = if let Some(e) = e.downcast_ref::() { + let mut h = crate::ToStringHighlighter::new(); + h.format_error(e)?; + h.finalize()?; + h.to_string() + } else { + match e.downcast::() { + Ok(tremor_pipeline::errors::Error( + tremor_pipeline::errors::ErrorKind::Script(e), + _, + )) => { + //ErrorKind::Script(e) + let mut h = crate::ToStringHighlighter::new(); + h.format_error(&(e.into()))?; + h.finalize()?; + h.to_string() + } + Ok(e) => e.to_string(), + Err(e) => e.to_string(), } - err => err.to_string(), }; error!( "Error starting deployment of flow {}: {err_str}", flow.instance_alias ); - Err(ErrorKind::DeployFlowError(flow.instance_alias.clone(), err_str).into()) + Err(flow::Error::Deploy(flow.instance_alias.clone(), err_str).into()) } Ok(flow_id) => Ok(flow_id), } diff --git a/src/system/flow.rs b/src/system/flow.rs index ff89c1aaea..e777f9ee14 100644 --- a/src/system/flow.rs +++ b/src/system/flow.rs @@ -13,13 +13,14 @@ // limitations under the License. use crate::{ + channel::empty_e, channel::{bounded, oneshot, OneShotSender, Sender}, - errors::empty_error, + connectors::AliasableConnectorResult, raft::{self, NodeId}, }; use crate::{ connectors::{self, ConnectorResult, Known}, - errors::{Error, Kind as ErrorKind, Result}, + errors::{ErrorKind, Result}, instance::{IntendedState, State}, log_error, pipeline::{self, InputTarget}, @@ -44,6 +45,26 @@ use tremor_script::{ errors::{error_generic, not_defined_err}, }; +#[derive(thiserror::Error, Debug)] +pub(crate) enum Error { + #[error("failed to deploy flow {0}: {1}")] + Deploy(String, String), + #[error("Pipeline {0} not fund in {1}")] + PipelineNotFound(String, String), + #[error("Connector {0} not fund in {1}")] + ConnectorNotFound(String, String), + #[error("Unknown connector type {1} in flow {0}")] + UnknownConnector(alias::Flow, String), + #[error("illegal state change")] + IllegalStateChange, + #[error("Duplicate flow: {0}")] + Duplicate(alias::Flow), + #[error("Flow not found: {0}")] + NotFound(alias::Flow), + #[error("Flow failed: {0}")] + Failed(alias::Flow), +} + #[derive(Debug)] /// Control Plane message accepted by each binding control plane handler pub(crate) enum Msg { @@ -92,13 +113,13 @@ pub struct StatusReport { #[derive(Debug, Clone, Default)] pub(crate) struct AppContext { pub(crate) id: alias::Flow, - pub(crate) raft: raft::Cluster, + pub(crate) raft: raft::ClusterInterface, pub(crate) metrics: MetricsChannel, } impl AppContext { - pub fn id(&self) -> &alias::App { - self.id.app_id() + pub fn id(&self) -> &alias::Flow { + &self.id } pub fn instance(&self) -> &alias::Instance { self.id.instance_id() @@ -124,13 +145,13 @@ impl Flow { new_state: IntendedState, tx: OneShotSender>, ) -> Result<()> { - self.addr + Ok(self + .addr .send(Msg::ChangeState { intended_state: new_state, reply_tx: tx, }) - .await - .map_err(Error::from) + .await?) } /// request a `StatusReport` from this `Flow` @@ -227,7 +248,10 @@ impl Flow { known_connectors .get(&config.connector_type) .ok_or_else(|| { - ErrorKind::UnknownConnectorType(config.connector_type.to_string()) + Error::UnknownConnector( + alias::Flow::from(alias), + config.connector_type.to_string(), + ) })?; connectors.insert( alias.to_string(), @@ -245,7 +269,9 @@ impl Flow { ast::CreateTargetDefinition::Pipeline(defn) => { let query = { let aggr_reg = tremor_script::aggr_registry(); - let reg = tremor_script::FN_REGISTRY.read()?; + let reg = tremor_script::FN_REGISTRY + .read() + .map_err(|_| ErrorKind::ReadLock)?; let mut helper = Helper::new(®, &aggr_reg); defn.to_query(&create.with, &mut helper)? @@ -305,11 +331,9 @@ async fn link( let pipeline = pipelines .get(to.alias()) - .ok_or(format!( - "Pipeline {} not found, in: {}", - to.alias(), - key_list(pipelines) - ))? + .ok_or_else(|| { + Error::PipelineNotFound(to.alias().to_string(), key_list(pipelines)) + })? .clone(); let (tx, mut rx) = bounded(1); @@ -323,23 +347,19 @@ async fn link( timeout(TIMEOUT, rx.recv()) .await? - .ok_or_else(empty_error)? + .ok_or_else(empty_e)? .map_err(|e| error_generic(link, from, &e))?; } ConnectStmt::PipelineToConnector { from, to, .. } => { - let pipeline = pipelines.get(from.alias()).ok_or(format!( - "Pipeline {} not found in: {}", - from.alias(), - key_list(pipelines) - ))?; + let pipeline = pipelines.get(from.alias()).ok_or_else(|| { + Error::PipelineNotFound(from.alias().to_string(), key_list(pipelines)) + })?; let connector = connectors .get(to.alias()) - .ok_or(format!( - "Connector {} not found: {}", - to.alias(), - key_list(pipelines) - ))? + .ok_or_else(|| { + Error::ConnectorNotFound(to.alias().to_string(), key_list(connectors)) + })? .clone(); // first link the pipeline to the connector @@ -353,7 +373,7 @@ async fn link( pipeline.send_mgmt(msg).await?; timeout(TIMEOUT, rx.recv()) .await? - .ok_or_else(empty_error)? + .ok_or_else(empty_e)? .map_err(|e| error_generic(link, from, &e))?; // then link the connector to the pipeline @@ -368,20 +388,16 @@ async fn link( connector.send(msg).await?; timeout(TIMEOUT, rx.recv()) .await? - .ok_or_else(empty_error)? + .ok_or_else(empty_e)? .map_err(|e| error_generic(link, to, &e))?; } ConnectStmt::PipelineToPipeline { from, to, .. } => { - let from_pipeline = pipelines.get(from.alias()).ok_or(format!( - "Pipeline {} not found in: {}", - from.alias(), - key_list(pipelines) - ))?; - let to_pipeline = pipelines.get(to.alias()).ok_or(format!( - "Pipeline {} not found in: {}", - from.alias(), - key_list(pipelines) - ))?; + let from_pipeline = pipelines.get(from.alias()).ok_or_else(|| { + Error::PipelineNotFound(from.alias().to_string(), key_list(pipelines)) + })?; + let to_pipeline = pipelines.get(to.alias()).ok_or_else(|| { + Error::PipelineNotFound(from.alias().to_string(), key_list(pipelines)) + })?; let (tx_from, mut rx_from) = bounded(1); let msg_from = crate::pipeline::MgmtMsg::ConnectOutput { port: from.port().to_string().into(), @@ -401,12 +417,12 @@ async fn link( from_pipeline.send_mgmt(msg_from).await?; timeout(TIMEOUT, rx_from.recv()) .await? - .ok_or_else(empty_error)? + .ok_or_else(empty_e)? .map_err(|e| error_generic(link, to, &e))?; to_pipeline.send_mgmt(msg_to).await?; timeout(TIMEOUT, rx_to.recv()) .await? - .ok_or_else(empty_error)? + .ok_or_else(empty_e)? .map_err(|e| error_generic(link, from, &e))?; } } @@ -416,9 +432,9 @@ async fn link( /// wrapper for all possible messages handled by the flow task enum MsgWrapper { Msg(Msg), - StartResult(ConnectorResult<()>), - DrainResult(ConnectorResult<()>), - StopResult(ConnectorResult<()>), + StartResult(ConnectorResult), + DrainResult(ConnectorResult), + StopResult(ConnectorResult), } /// How the depoloyment is distributed on a cluster @@ -444,9 +460,9 @@ struct RunningFlow { state_change_senders: HashMap>>>, input_channel: Pin + Send + Sync>>, msg_tx: Sender, - drain_tx: Sender>, - stop_tx: Sender>, - start_tx: Sender>, + drain_tx: Sender, + stop_tx: Sender, + start_tx: Sender, deployment_type: DeploymentType, } @@ -722,10 +738,10 @@ impl RunningFlow { (State::Draining, IntendedState::Running | IntendedState::Paused) | (State::Stopped, _) => { log_error!( - reply_tx.send(Err("illegal state change".into())), + reply_tx.send(Err(Error::IllegalStateChange.into())), "{prefix} Error sending StateChagnge response: {e:?}" ); - return Err("illegal state change".into()); + return Err(Error::IllegalStateChange.into()); } (State::Failed, intended) => { self.insert_state_change_sender(intended_state, reply_tx); @@ -759,10 +775,8 @@ impl RunningFlow { .find(|c| c.alias == connector_alias) .cloned() .ok_or_else(|| { - Error::from(ErrorKind::ConnectorNotFound( - flow_id.to_string(), - connector_alias.connector_alias().to_string(), - )) + Error::UnknownConnector(flow_id.clone(), connector_alias.to_string()) + .into() }); log_error!( reply_tx.send(res), @@ -786,7 +800,7 @@ impl RunningFlow { } } MsgWrapper::StopResult(res) => { - if self.handle_stop_result(res, &prefix) == ControlFlow::Break(()) { + if self.handle_stop_result(&res, &prefix) == ControlFlow::Break(()) { break; } } @@ -823,15 +837,11 @@ impl RunningFlow { Ok(()) } - fn handle_start_result( - &mut self, - conn_res: ConnectorResult<()>, - prefix: impl std::fmt::Display, - ) { - if let Err(e) = conn_res.res { + fn handle_start_result(&mut self, conn_res: ConnectorResult, prefix: impl std::fmt::Display) { + if let Some(e) = conn_res.err() { error!( "{prefix} Error starting Connector {conn}: {e}", - conn = conn_res.alias + conn = e.alias() ); if self.state != State::Failed { // only report failed upon the first connector failure @@ -934,15 +944,16 @@ impl RunningFlow { fn handle_stop_result( &mut self, - conn_res: ConnectorResult<()>, + conn_res: &ConnectorResult, prefix: impl std::fmt::Display, ) -> ControlFlow<()> { - info!("{prefix} Connector {} stopped.", &conn_res.alias); + let alias: &alias::Connector = conn_res.alias(); + + info!("{prefix} Connector {alias} stopped."); log_error!( - conn_res.res, - "{prefix} Error during Stopping in Connector {}: {e}", - &conn_res.alias + conn_res, + "{prefix} Error during Stopping in Connector {alias}: {e}" ); let old = self.expected_stops; @@ -959,15 +970,16 @@ impl RunningFlow { async fn handle_drain_result( &mut self, - conn_res: ConnectorResult<()>, + conn_res: ConnectorResult, prefix: impl std::fmt::Display, ) -> Result> { - info!("{prefix} Connector {} drained.", &conn_res.alias); + let alias: &alias::Connector = conn_res.alias(); + + info!("{prefix} Connector {alias} drained."); log_error!( - conn_res.res, - "{prefix} Error during Draining in Connector {}: {e}", - &conn_res.alias + conn_res, + "{prefix} Error during Draining in Connector {alias}: {e}" ); let old = self.expected_drains; @@ -998,9 +1010,7 @@ impl RunningFlow { if let Some(senders) = self.state_change_senders.get_mut(state) { for sender in senders.drain(..) { log_error!( - sender.send(Err( - ErrorKind::FlowFailed(self.app_ctx.to_string()).into() - )), + sender.send(Err(Error::Failed(self.app_ctx.id().clone()).into())), "Error notifying {state} state handlers of failed state: {e:?}" ); } @@ -1085,7 +1095,7 @@ mod tests { &mut self, _pull_id: &mut u64, _ctx: &SourceContext, - ) -> Result { + ) -> anyhow::Result { Ok(SourceReply::Data { origin_uri: EventOriginUri::default(), data: r#"{"snot":"badger"}"#.as_bytes().to_vec(), @@ -1123,7 +1133,7 @@ mod tests { _ctx: &SinkContext, _serializer: &mut EventSerializer, _start: u64, - ) -> Result { + ) -> anyhow::Result { self.tx.send(event)?; Ok(SinkReply::NONE) } @@ -1184,7 +1194,11 @@ mod tests { "#; let (tx, _rx) = bounded(128); let kill_switch = KillSwitch(tx); - let deployable = Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg)?; + let deployable = Deploy::parse( + &src, + &*FN_REGISTRY.read().map_err(|_| ErrorKind::ReadLock)?, + &aggr_reg, + )?; let deploy = deployable .deploy .stmts @@ -1225,7 +1239,7 @@ mod tests { assert_eq!(String::from("foo"), connectors[0].alias.to_string()); // assert the flow has started and events are flowing - let event = connector_rx.recv().await.ok_or("empty")?; + let event = connector_rx.recv().await.expect("empty"); assert_eq!( &literal!({ "snot": "badger" diff --git a/src/system/flow_supervisor.rs b/src/system/flow_supervisor.rs index a9bad8ba27..2f712cbaf1 100644 --- a/src/system/flow_supervisor.rs +++ b/src/system/flow_supervisor.rs @@ -13,13 +13,13 @@ // limitations under the License. use crate::{ - channel::{bounded, OneShotSender, Sender}, + channel::{bounded, ChannelError, OneShotSender, Sender}, connectors::{self, ConnectorBuilder, ConnectorType}, - errors::{Kind as ErrorKind, Result}, + errors::Result, instance::IntendedState, log_error, qsize, raft, system::{ - flow::{AppContext, Flow}, + flow::{self, AppContext, Flow}, KillSwitch, DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, }, }; @@ -51,7 +51,7 @@ pub(crate) enum Msg { /// result sender sender: OneShotSender>, /// API request sender - raft: raft::Cluster, + raft: raft::ClusterInterface, /// Type of the deployment deployment_type: DeploymentType, }, @@ -112,12 +112,12 @@ impl FlowSupervisor { flow: DeployFlow<'static>, sender: oneshot::Sender>, kill_switch: &KillSwitch, - raft: raft::Cluster, + raft: raft::ClusterInterface, deployment_type: DeploymentType, ) { let id = alias::Flow::new(app_id, &flow.instance_alias); let res = match self.flows.entry(id.clone()) { - Entry::Occupied(_occupied) => Err(ErrorKind::DuplicateFlow(id.to_string()).into()), + Entry::Occupied(_occupied) => Err(flow::Error::Duplicate(id.clone()).into()), Entry::Vacant(vacant) => { let ctx = AppContext { id: id.clone(), @@ -159,7 +159,7 @@ impl FlowSupervisor { self.flows .get(id) .cloned() - .ok_or_else(|| ErrorKind::FlowNotFound(id.to_string()).into()), + .ok_or_else(|| flow::Error::NotFound(id.clone()).into()), ) .map_err(|_| "send error"), "Error sending GetFlow response: {e}" @@ -188,7 +188,7 @@ impl FlowSupervisor { for rx in rxs.drain(..) { log_error!(rx.await?, "Error during Stopping: {e}"); } - Result::Ok(()) + Ok::<(), anyhow::Error>(()) }), ) .await???; @@ -226,8 +226,8 @@ impl FlowSupervisor { }; } info!("Flows drained."); - sender.send(Ok(())).map_err(|_| "Failed to send reply")?; - Result::Ok(()) + sender.send(Ok(())).map_err(|_| ChannelError::Send)?; + Ok::<(), anyhow::Error>(()) }); } } @@ -246,18 +246,18 @@ impl FlowSupervisor { Ok(()) } else { reply_tx - .send(Err(ErrorKind::FlowNotFound(id.to_string()).into())) - .map_err(|_| "can't reply")?; - Err(ErrorKind::FlowNotFound(id.to_string()).into()) + .send(Err(flow::Error::NotFound(id.clone()).into())) + .map_err(|_| ChannelError::Send)?; + Err(flow::Error::NotFound(id.clone()).into()) } } else if let Some(flow) = self.flows.get(&id) { flow.change_state(intended_state, reply_tx).await?; Ok(()) } else { reply_tx - .send(Err(ErrorKind::FlowNotFound(id.to_string()).into())) - .map_err(|_| "can't reply")?; - Err(ErrorKind::FlowNotFound(id.to_string()).into()) + .send(Err(flow::Error::NotFound(id.clone()).into())) + .map_err(|_| ChannelError::Send)?; + Err(flow::Error::NotFound(id.clone()).into()) } } diff --git a/src/utils.rs b/src/utils.rs index 342fcc418a..15b5f93260 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -11,19 +11,14 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -use crate::errors::{Error, Kind as ErrorKind}; /// Fetches a hostname with `tremor-host.local` being the default #[must_use] pub fn hostname() -> String { hostname::get() - .map_err(Error::from) - .and_then(|hostname| { - hostname.into_string().map_err(|os_string| { - ErrorKind::Msg(format!("Invalid hostname: {}", os_string.to_string_lossy())).into() - }) - }) - .unwrap_or_else(|_| "tremor_host.local".to_string()) + .ok() + .and_then(|hostname| hostname.into_string().ok()) + .unwrap_or_else(|| "tremor_host.local".to_string()) } #[must_use] diff --git a/tests/query.rs b/tests/query.rs index e825bf9ed0..6fcea61995 100644 --- a/tests/query.rs +++ b/tests/query.rs @@ -28,7 +28,13 @@ use tremor_value::utils::sorted_serialize; fn to_pipe(query: String) -> Result { let aggr_reg = tremor_script::aggr_registry(); let mut idgen = OperatorUIdGen::new(); - let q = Query::parse(&query, &*FN_REGISTRY.read()?, &aggr_reg)?; + let q = Query::parse( + &query, + &*FN_REGISTRY + .read() + .map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?, + &aggr_reg, + )?; Ok(q.to_executable_graph(&mut idgen, &MetricsChannel::new(128))?) } diff --git a/tests/query_error.rs b/tests/query_error.rs index 528edfc25c..418538d7ec 100644 --- a/tests/query_error.rs +++ b/tests/query_error.rs @@ -27,7 +27,13 @@ use tremor_script::FN_REGISTRY; fn to_executable_graph(query: &str) -> Result { let aggr_reg = tremor_script::aggr_registry(); let mut idgen = OperatorUIdGen::new(); - let q = Query::parse(&query, &*FN_REGISTRY.read()?, &aggr_reg)?; + let q = Query::parse( + &query, + &*FN_REGISTRY + .read() + .map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?, + &aggr_reg, + )?; Ok(q.to_executable_graph(&mut idgen, &MetricsChannel::default())?) } macro_rules! test_cases { @@ -74,30 +80,25 @@ macro_rules! test_cases { let mut err = String::new(); file.read_to_string(&mut err)?; - match s { - Err(Error(ErrorKind::Pipeline(tremor_pipeline::errors::ErrorKind::Script(e)), o)) => { - let e = tremor_script::errors::Error(e, o); - let got = Dumb::error_to_string(&e)?; - assert_eq!(err.trim(), got.trim(), "unexpected error message:\n{}", got); - } - Err(Error(ErrorKind::Script(e), o)) =>{ + let Some(e) = s.err() else { + panic!("Expected error, but got succeess") + }; + + if let Some(e) = e.downcast_ref::() { + let got = Dumb::error_to_string(e)?; + assert_eq!(err.trim(), got.trim(), "unexpected error message:\n{}", got); + } else if let Ok(tremor_pipeline::errors::Error(e, o)) = e.downcast::() { + if let tremor_pipeline::errors::ErrorKind::Script(e) = e { let e = tremor_script::errors::Error(e, o); let got = Dumb::error_to_string(&e)?; assert_eq!(err.trim(), got.trim(), "unexpected error message:\n{}", got); - } - Err(Error(ErrorKind::Pipeline(e), _)) =>{ + } else { let got = format!("{}", e); assert_eq!(err.trim(), got.trim(), "unexpected error message:\n{}", got); } - Err(e) => { - println!("got wrong error: {:?}", e); - assert!(false); - } - _ =>{ - println!("Expected error, but got succeess"); - assert!(false); - } - }; + } else { + panic!("Wrong error") + } }; Ok(()) } diff --git a/tests/query_runtime_error.rs b/tests/query_runtime_error.rs index c64050f1ec..de96f4a8c1 100644 --- a/tests/query_runtime_error.rs +++ b/tests/query_runtime_error.rs @@ -12,27 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. use pretty_assertions::assert_eq; +use serial_test::serial; use std::io::prelude::*; use std::path::PathBuf; use tremor_common::{file, ports::IN, uids::OperatorUIdGen}; - +use tremor_pipeline::errors::{Error as PipelineError, ErrorKind as PipelineErrorKind}; use tremor_pipeline::ExecutableGraph; use tremor_pipeline::{query::Query, MetricsChannel}; use tremor_pipeline::{Event, EventId}; -use tremor_script::FN_REGISTRY; - -use serial_test::serial; -use tremor_pipeline::errors::{Error as PipelineError, ErrorKind as PipelineErrorKind}; use tremor_runtime::errors::*; use tremor_script::highlighter::Dumb; use tremor_script::module::Manager; use tremor_script::utils::*; +use tremor_script::FN_REGISTRY; use tremor_value::utils::sorted_serialize; fn to_pipe(query: &str) -> Result { let aggr_reg = tremor_script::aggr_registry(); let mut idgen = OperatorUIdGen::new(); - let q = Query::parse(&query, &*FN_REGISTRY.read()?, &aggr_reg)?; + let q = Query::parse( + &query, + &*FN_REGISTRY + .read() + .map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?, + &aggr_reg, + )?; Ok(q.to_executable_graph(&mut idgen, &MetricsChannel::new(128))?) } diff --git a/tests/script.rs b/tests/script.rs index dbfbfac259..d2f638279b 100644 --- a/tests/script.rs +++ b/tests/script.rs @@ -47,7 +47,7 @@ macro_rules! test_cases { Manager::clear_path()?; Manager::add_path(&"tremor-script/lib")?; Manager::add_path(&script_dir)?; - let script = Script::parse(&contents, &*FN_REGISTRY.read()?)?; + let script = Script::parse(&contents, &*FN_REGISTRY.read().map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?)?; println!("Loading input: {}", in_file); let in_json = load_event_file(in_file)?; diff --git a/tests/script_error.rs b/tests/script_error.rs index c047c7aa23..f7ead4594d 100644 --- a/tests/script_error.rs +++ b/tests/script_error.rs @@ -43,7 +43,7 @@ macro_rules! test_cases { Manager::clear_path()?; Manager::add_path(&script_dir)?; Manager::add_path(&"tremor-script/lib")?; - let s = Script::parse(&contents, &*FN_REGISTRY.read()?); + let s = Script::parse(&contents, &*FN_REGISTRY.read().map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?); if let Err(e) = s { let got = Dumb::error_to_string(&e)?; println!("{}", got); @@ -83,7 +83,7 @@ macro_rules! ignored_cases { Manager::clear_path()?; Manager::add_path(&script_dir)?; Manager::add_path(&"tremor-script/lib")?; - let s = Script::parse(&contents, &*FN_REGISTRY.read()?); + let s = Script::parse(&contents, &*FN_REGISTRY.read().map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?); if let Err(e) = s { let got = Dumb::error_to_string(&e)?; println!("{}", got); diff --git a/tests/script_runtime_error.rs b/tests/script_runtime_error.rs index 5d8433d4fa..9bf8b7d40d 100644 --- a/tests/script_runtime_error.rs +++ b/tests/script_runtime_error.rs @@ -43,7 +43,7 @@ macro_rules! test_cases { Manager::clear_path()?; Manager::add_path(&script_dir)?; Manager::add_path(&"tremor-script/lib")?; - let script = Script::parse(&contents, &*FN_REGISTRY.read()?)?; + let script = Script::parse(&contents, &*FN_REGISTRY.read().map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?)?; println!("Loading input: {}", in_file); let mut in_json = load_event_file(in_file)?; @@ -96,7 +96,7 @@ macro_rules! ignore_cases { Manager::clear_path()?; Manager::add_path(&script_dir)?; Manager::add_path(&"tremor-script/lib")?; - let script = Script::parse(&contents, &*FN_REGISTRY.read()?)?; + let script = Script::parse(&contents, &*FN_REGISTRY.read().map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?)?; println!("Loading input: {}", in_file); let mut in_json = load_event_file(in_file)?; diff --git a/tests/script_warning.rs b/tests/script_warning.rs index 2b5f2622b5..b7a94c7310 100644 --- a/tests/script_warning.rs +++ b/tests/script_warning.rs @@ -46,7 +46,7 @@ macro_rules! test_cases { let mut err = String::new(); file.read_to_string(&mut err)?; let err = err.trim(); - let s = Script::parse(&contents, &*FN_REGISTRY.read()?)?; + let s = Script::parse(&contents, &*FN_REGISTRY.read().map_err(|_| tremor_runtime::errors::ErrorKind::ReadLock)?)?; let mut h = Dumb::new(); s.format_warnings_with(&mut h)?; h.finalize()?; diff --git a/tremor-cli/Cargo.toml b/tremor-cli/Cargo.toml index 3f82865d67..5d572a0bab 100644 --- a/tremor-cli/Cargo.toml +++ b/tremor-cli/Cargo.toml @@ -9,15 +9,8 @@ version = "0.13.0-rc.16" [package.metadata.docs.rs] default-target = "x86_64-unknown-linux-gnu" -[build-dependencies] -lalrpop = "0.20" -matches = "0.1.10" - [dev-dependencies] temp-dir = "0.1" -criterion = "0.5" -float-cmp = "0.9" -matches = "0.1" pretty_assertions = "1.4" [dependencies] diff --git a/tremor-cli/src/cluster.rs b/tremor-cli/src/cluster.rs index 2b2c631a34..f5c10ebb42 100644 --- a/tremor-cli/src/cluster.rs +++ b/tremor-cli/src/cluster.rs @@ -33,7 +33,7 @@ use tremor_runtime::{ node::{Addr, ClusterNodeKillSwitch, Node}, remove_node, store::TremorInstanceState, - ClusterError, NodeId, + NodeId, }, system::ShutdownMode, }; @@ -159,29 +159,28 @@ impl Cluster { // start db and statemachine // start raft // node_id assigned during bootstrap or join - let running_node = if Path::new(&db_dir).exists() - && !tremor_common::file::is_empty(&db_dir)? - { - if !join.is_empty() { - // TODO: check if join nodes are part of the known cluster, if so, don't error - return Err(Error::from( - "Cannot join another cluster with existing db directory", - )); - } - info!("Loading existing Tremor node state from {db_dir}."); - Node::load_from_store(&db_dir, tremor_runtime::raft::config()?).await? - } else { - // db dir does not exist - let rpc_addr = rpc.ok_or_else(|| ClusterError::from("missing rpc address"))?; - let api_addr = api.ok_or_else(|| ClusterError::from("missing api address"))?; - let addr = Addr::new(api_addr, rpc_addr); + let running_node = + if Path::new(&db_dir).exists() && !tremor_common::file::is_empty(&db_dir)? { + if !join.is_empty() { + // TODO: check if join nodes are part of the known cluster, if so, don't error + return Err(Error::from( + "Cannot join another cluster with existing db directory", + )); + } + info!("Loading existing Tremor node state from {db_dir}."); + Node::load_from_store(&db_dir, tremor_runtime::raft::config()?).await? + } else { + // db dir does not exist + let rpc_addr = rpc.ok_or_else(|| Error::from("missing rpc address"))?; + let api_addr = api.ok_or_else(|| Error::from("missing api address"))?; + let addr = Addr::new(api_addr, rpc_addr); - info!("Bootstrapping cluster node with addr {addr} and db_dir {db_dir}"); - let mut node = Node::new(&db_dir, tremor_runtime::raft::config()?); + info!("Bootstrapping cluster node with addr {addr} and db_dir {db_dir}"); + let mut node = Node::new(&db_dir, tremor_runtime::raft::config()?); - // attempt to join any one of the given endpoints, stop once we joined one - node.try_join(addr, join, !passive).await? - }; + // attempt to join any one of the given endpoints, stop once we joined one + node.try_join(addr, join, !passive).await? + }; // install signal handler let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; diff --git a/tremor-cli/src/errors.rs b/tremor-cli/src/errors.rs index 6884becd5f..f235ceeb92 100644 --- a/tremor-cli/src/errors.rs +++ b/tremor-cli/src/errors.rs @@ -13,9 +13,7 @@ // limitations under the License. //NOTE: error_chain -#![allow(deprecated)] -#![allow(missing_docs)] -#![allow(clippy::large_enum_variant)] +#![allow(deprecated, missing_docs, clippy::large_enum_variant)] use crate::util::SourceKind; use error_chain::error_chain; @@ -47,7 +45,6 @@ error_chain! { links { Script(tremor_script::errors::Error, tremor_script::errors::ErrorKind); Pipeline(tremor_pipeline::errors::Error, tremor_pipeline::errors::ErrorKind); - Runtime(tremor_runtime::errors::Error, tremor_runtime::errors::ErrorKind); Codec(tremor_codec::errors::Error, tremor_codec::errors::ErrorKind); Interceptor(tremor_interceptor::errors::Error, tremor_interceptor::errors::ErrorKind); } diff --git a/tremor-cli/src/test/before.rs b/tremor-cli/src/test/before.rs index 6353d67409..929c2ff9bf 100644 --- a/tremor-cli/src/test/before.rs +++ b/tremor-cli/src/test/before.rs @@ -39,7 +39,7 @@ pub(crate) struct Before { #[serde(rename = "max-await-secs", default = "default_max_await_secs")] until: u64, #[serde(rename = "min-await-secs", default = "default_min_await_secs")] - before_start_delay: u64, + start_delay: u64, } fn default_dir() -> String { @@ -179,9 +179,9 @@ impl Before { tokio::time::sleep(Duration::from_millis(100)).await; } } - if self.before_start_delay > 0 { - let dur = Duration::from_secs(self.before_start_delay); - debug!("Sleeping for {}s ...", self.before_start_delay); + if self.start_delay > 0 { + let dur = Duration::from_secs(self.start_delay); + debug!("Sleeping for {}s ...", self.start_delay); tokio::time::sleep(dur).await; } Ok(()) diff --git a/tremor-codec/src/errors.rs b/tremor-codec/src/errors.rs index 27999c8441..33e3040418 100644 --- a/tremor-codec/src/errors.rs +++ b/tremor-codec/src/errors.rs @@ -34,6 +34,9 @@ impl PartialEq for Error { } } +unsafe impl Send for Error {} +unsafe impl Sync for Error {} + error_chain! { foreign_links { CsvError(csv::Error); diff --git a/tremor-codec/src/lib.rs b/tremor-codec/src/lib.rs index 1974c4e90e..141bffa72e 100644 --- a/tremor-codec/src/lib.rs +++ b/tremor-codec/src/lib.rs @@ -27,7 +27,7 @@ pub mod errors; pub use crate::errors::Error; -use crate::errors::{Kind as ErrorKind, Result}; +use crate::errors::{ErrorKind, Result}; use std::fmt::{Debug, Display}; use tremor_value::Value; mod codec { diff --git a/tremor-interceptor/src/errors.rs b/tremor-interceptor/src/errors.rs index ae1aad0af6..cdd330e667 100644 --- a/tremor-interceptor/src/errors.rs +++ b/tremor-interceptor/src/errors.rs @@ -26,6 +26,9 @@ impl PartialEq for Error { } } +unsafe impl Send for Error {} +unsafe impl Sync for Error {} + error_chain! { links { } diff --git a/tremor-interceptor/src/preprocessor.rs b/tremor-interceptor/src/preprocessor.rs index ff9451e8ea..461a4655a8 100644 --- a/tremor-interceptor/src/preprocessor.rs +++ b/tremor-interceptor/src/preprocessor.rs @@ -195,13 +195,13 @@ mod test { let data = vec![1_u8, 2, 3]; - let encoded = post_p.process(42, 23, &data)?.pop().ok_or("no data")?; + let encoded = post_p.process(42, 23, &data)?.pop().expect("no data"); let mut in_ns = 0u64; let decoded = pre_p .process(&mut in_ns, &encoded, Value::object())? .pop() - .ok_or("no data")? + .expect("no data") .0; assert!(pre_p.finish(None, None)?.is_empty()); diff --git a/tremor-pipeline/Cargo.toml b/tremor-pipeline/Cargo.toml index ebb6347b8e..5c5da91bc0 100644 --- a/tremor-pipeline/Cargo.toml +++ b/tremor-pipeline/Cargo.toml @@ -33,9 +33,6 @@ url = "2" window = { git = "https://github.com/tremor-rs/window.git", tag = "v0.1.1" } [dev-dependencies] -criterion = "0.5" -tempfile = "3.8" -xz2 = "0.1" tokio = { version = "1.32", features = ["full"] } [features] diff --git a/tremor-pipeline/src/errors.rs b/tremor-pipeline/src/errors.rs index b8b76de63f..1619661984 100644 --- a/tremor-pipeline/src/errors.rs +++ b/tremor-pipeline/src/errors.rs @@ -13,9 +13,7 @@ // limitations under the License. //NOTE: error_chain -#![allow(deprecated)] -#![allow(clippy::large_enum_variant)] -#![allow(missing_docs)] +#![allow(deprecated, missing_docs, clippy::large_enum_variant)] use error_chain::error_chain; impl

From> for Error { @@ -35,6 +33,9 @@ impl From> for Error { } } +unsafe impl Send for Error {} +unsafe impl Sync for Error {} + error_chain! { links { Script(tremor_script::errors::Error, tremor_script::errors::ErrorKind); diff --git a/tremor-pipeline/src/op/generic/batch.rs b/tremor-pipeline/src/op/generic/batch.rs index 9d7f394e55..dd8e81aafa 100644 --- a/tremor-pipeline/src/op/generic/batch.rs +++ b/tremor-pipeline/src/op/generic/batch.rs @@ -65,7 +65,7 @@ struct Batch { /// event id for the resulting batched event /// the resulting id will be a new distinct id and will be tracking /// all event ids (min and max) in the batched event - batch_event_id: EventId, + event_id: EventId, is_transactional: bool, event_id_gen: EventIdGenerator, } @@ -85,7 +85,7 @@ if let Some(map) = &node.config { config, max_delay_ns, first_ns: 0, - batch_event_id: idgen.next_id(), + event_id: idgen.next_id(), is_transactional: false, event_id_gen: idgen, })) @@ -114,7 +114,7 @@ impl Operator for Batch { transactional, .. } = event; - self.batch_event_id.track(&id); + self.event_id.track(&id); self.is_transactional = self.is_transactional || transactional; self.data.consume( data, @@ -158,7 +158,7 @@ impl Operator for Batch { ..Event::default() }; self.is_transactional = false; - swap(&mut self.batch_event_id, &mut event.id); + swap(&mut self.event_id, &mut event.id); Ok(event.into()) } else { Ok(EventAndInsights::default()) @@ -197,7 +197,7 @@ impl Operator for Batch { ..Event::default() }; self.is_transactional = false; - swap(&mut self.batch_event_id, &mut event.id); + swap(&mut self.event_id, &mut event.id); EventAndInsights::from(event) } else { EventAndInsights::default() @@ -229,7 +229,7 @@ mod test { max_delay_ns: None, data: empty_payload(), len: 0, - batch_event_id: idgen.next_id(), + event_id: idgen.next_id(), is_transactional: false, event_id_gen: idgen, }; @@ -369,7 +369,7 @@ mod test { max_delay_ns: Some(1_000_000), data: empty_payload(), len: 0, - batch_event_id: idgen.next_id(), + event_id: idgen.next_id(), is_transactional: false, event_id_gen: idgen, }; @@ -445,7 +445,7 @@ mod test { max_delay_ns: Some(100_000), data: empty_payload(), len: 0, - batch_event_id: idgen.next_id(), + event_id: idgen.next_id(), is_transactional: false, event_id_gen: idgen, }; diff --git a/tremor-pipeline/src/op/grouper/bucket.rs b/tremor-pipeline/src/op/grouper/bucket.rs index ccf9d70e9a..90e84654e3 100644 --- a/tremor-pipeline/src/op/grouper/bucket.rs +++ b/tremor-pipeline/src/op/grouper/bucket.rs @@ -103,7 +103,7 @@ op!(BucketGrouperFactory(_uid, node) { #[derive(Debug)] struct Rate { /// the maximum number of events per time range - rate: u64, + max: u64, /// time range in milliseconds, (default: 1000 - 1 second) time_range: u64, /// numbers of window in the time_range (default: 100) @@ -112,12 +112,12 @@ struct Rate { impl Rate { pub fn from_meta(meta: &Value) -> Option { - let rate = meta.get("rate")?.as_u64()?; + let max = meta.get("rate")?.as_u64()?; let time_range = meta.get_u64("time_range").unwrap_or(1000); let windows = meta.get_usize("windows").unwrap_or(100); Some(Self { - rate, + max, time_range, windows, }) @@ -188,7 +188,7 @@ impl Operator for Grouper { TimeWindow::new( rate.windows, rate.time_range / (rate.windows as u64), - rate.rate, + rate.max, ), ); let Some(g) = groups.cache.get_mut(&dimensions) else { @@ -253,7 +253,7 @@ mod test { .on_event(0, operator_id, &Port::In, &mut state, event1.clone()) .expect("could not run pipeline"); - let (port, e) = r.events.pop().ok_or("no data")?; + let (port, e) = r.events.pop().expect("no data"); assert!(r.events.is_empty()); assert_eq!(port, "err"); assert_eq!(e, event1); @@ -270,7 +270,7 @@ mod test { .on_event(0, operator_id, &Port::In, &mut state, event2.clone()) .expect("could not run pipeline"); - let (port, e) = r.events.pop().ok_or("no data")?; + let (port, e) = r.events.pop().expect("no data"); assert!(r.events.is_empty()); assert_eq!(port, "out"); assert_eq!(e, event2); @@ -279,7 +279,7 @@ mod test { .on_event(0, operator_id, &Port::In, &mut state, event2.clone()) .expect("could not run pipeline"); - let (port, e) = r.events.pop().ok_or("no data")?; + let (port, e) = r.events.pop().expect("no data"); assert!(r.events.is_empty()); assert_eq!(port, "out"); assert_eq!(e, event2); @@ -288,7 +288,7 @@ mod test { .on_event(0, operator_id, &Port::In, &mut state, event2.clone()) .expect("could not run pipeline"); - let (port, e) = r.events.pop().ok_or("no data")?; + let (port, e) = r.events.pop().expect("no data"); assert!(r.events.is_empty()); assert_eq!(port, "overflow"); assert_eq!(e, event2); @@ -304,14 +304,14 @@ mod test { .on_event(0, operator_id, &Port::In, &mut state, event3.clone()) .expect("could not run pipeline"); - let (port, e) = r.events.pop().ok_or("no data")?; + let (port, e) = r.events.pop().expect("no data"); assert!(r.events.is_empty()); assert_eq!(port, "out"); assert_eq!(e, event3); let mut m = op.metrics(&Object::with_hasher(ObjectHasher::default()), 0)?; - let overflow = m.pop().ok_or("no data")?; - let pass = m.pop().ok_or("no data")?; + let overflow = m.pop().expect("no data"); + let pass = m.pop().expect("no data"); assert!(m.is_empty()); assert_eq!(overflow["tags"]["action"], "overflow"); assert_eq!(overflow["fields"]["count"], 1); diff --git a/tremor-pipeline/src/op/trickle/select.rs b/tremor-pipeline/src/op/trickle/select.rs index d175f3ba9e..459c3011cc 100644 --- a/tremor-pipeline/src/op/trickle/select.rs +++ b/tremor-pipeline/src/op/trickle/select.rs @@ -36,7 +36,7 @@ use tremor_value::{utils::sorted_serialize, Value}; #[derive(Debug)] pub(crate) struct Select { - select: ast::SelectStmt<'static>, + stmt: ast::SelectStmt<'static>, windows: Vec, groups: HashMap, recursion_limit: u32, @@ -70,7 +70,7 @@ impl Select { .unwrap_or(0); Self { windows, - select: select.clone(), + stmt: select.clone(), groups: HashMap::new(), recursion_limit: tremor_script::recursion_limit(), dflt_group, @@ -177,7 +177,7 @@ impl Operator for Select { mut event: Event, ) -> Result { let Self { - select, + stmt: select, windows, groups, recursion_limit, @@ -318,7 +318,7 @@ impl Operator for Select { ) -> Result { // we only react on ticks and when we have windows let Self { - select, + stmt: select, windows, groups, recursion_limit, diff --git a/tremor-pipeline/src/op/trickle/select/test.rs b/tremor-pipeline/src/op/trickle/select/test.rs index 171365a211..c894dbc70d 100644 --- a/tremor-pipeline/src/op/trickle/select/test.rs +++ b/tremor-pipeline/src/op/trickle/select/test.rs @@ -223,7 +223,7 @@ fn select_stmt_from_query(query_str: &str) -> Result