diff --git a/.dockerignore b/.dockerignore index eb5a316..fb898f3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,5 @@ +.env +.git +.github +docker target diff --git a/.env b/.env new file mode 100644 index 0000000..43eda2a --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database diff --git a/.github/workflows/docker.yml b/.github/workflows/docker-capture.yml similarity index 95% rename from .github/workflows/docker.yml rename to .github/workflows/docker-capture.yml index a2180db..5bc5100 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker-capture.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: push: branches: - - 'main' + - "main" permissions: packages: write @@ -14,7 +14,6 @@ jobs: name: build and publish capture image runs-on: buildjet-8vcpu-ubuntu-2204-arm steps: - - name: Check Out Repo uses: actions/checkout@v3 @@ -63,7 +62,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: RUST_BACKTRACE=1 + build-args: RUST_BACKTRACE=1 BIN=capture-server - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml new file mode 100644 index 0000000..2cafd62 --- /dev/null +++ b/.github/workflows/docker-hook-api.yml @@ -0,0 +1,68 @@ +name: Build hook-api docker image + +on: + workflow_dispatch: + push: + branches: + - "main" + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-api image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/posthog/hook-api + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push api + id: docker_build_hook_api + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BIN=hook-api + + - name: Hook-api image digest + run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml new file mode 100644 index 0000000..fc662bd --- /dev/null +++ b/.github/workflows/docker-hook-janitor.yml @@ -0,0 +1,68 @@ +name: Build hook-janitor docker image + +on: + workflow_dispatch: + push: + branches: + - "main" + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-janitor image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/posthog/hook-janitor + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push janitor + id: docker_build_hook_janitor + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BIN=hook-janitor + + - name: Hook-janitor image digest + run: echo ${{ steps.docker_build_hook_janitor.outputs.digest }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml new file mode 100644 index 0000000..77db4a7 --- /dev/null +++ b/.github/workflows/docker-hook-worker.yml @@ -0,0 +1,68 @@ +name: Build hook-worker docker image + +on: + workflow_dispatch: + push: + branches: + - "main" + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-worker image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/posthog/hook-worker + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push worker + id: docker_build_hook_worker + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BIN=hook-worker + + - name: Hook-worker image digest + run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml new file mode 100644 index 0000000..c186aab --- /dev/null +++ b/.github/workflows/docker-migrator.yml @@ -0,0 +1,67 @@ +name: Build hook-migrator docker image + +on: + workflow_dispatch: + push: + branches: + - "main" + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-migrator image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/posthog/hook-migrator + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push migrator + id: docker_build_hook_migrator + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile.migrate + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Hook-migrator image digest + run: echo ${{ steps.docker_build_hook_migrator.outputs.digest }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index de8ec08..ea3be4e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -3,9 +3,9 @@ name: Rust on: workflow_dispatch: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] env: CARGO_TERM_COLOR: always @@ -15,95 +15,96 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Install rust - uses: dtolnay/rust-toolchain@master - with: + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: toolchain: stable - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} - - name: Run cargo build - run: cargo build --all --locked --release && strip target/release/capture-server + - name: Run cargo build + run: cargo build --all --locked --release && find target/release/ -maxdepth 1 -executable -type f | xargs strip test: runs-on: buildjet-4vcpu-ubuntu-2204 timeout-minutes: 10 steps: - - uses: actions/checkout@v3 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Setup end2end dependencies - run: | - docker compose up -d --wait - echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts - - - name: Install rust - uses: dtolnay/rust-toolchain@master - with: + - uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Setup dependencies + run: | + docker compose up kafka redis db echo_server -d --wait + docker compose up setup_test_db + echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts + + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: toolchain: stable - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${ runner.os }-cargo-debug-${{ hashFiles('**/Cargo.lock') }} + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${ runner.os }-cargo-debug-${{ hashFiles('**/Cargo.lock') }} - - name: Run cargo test - run: cargo test --all-features + - name: Run cargo test + run: cargo test --all-features - - name: Run cargo check - run: cargo check --all-features + - name: Run cargo check + run: cargo check --all-features clippy: runs-on: buildjet-4vcpu-ubuntu-2204 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Install latest rust - uses: dtolnay/rust-toolchain@master - with: + - name: Install latest rust + uses: dtolnay/rust-toolchain@master + with: toolchain: stable components: clippy - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }} - - - name: Run clippy - run: cargo clippy -- -D warnings - + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }} + + - name: Run clippy + run: cargo clippy -- -D warnings + format: runs-on: buildjet-4vcpu-ubuntu-2204 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Install latest rust - uses: dtolnay/rust-toolchain@master - with: + - name: Install latest rust + uses: dtolnay/rust-toolchain@master + with: toolchain: stable components: rustfmt - - name: Format - run: cargo fmt -- --check + - name: Format + run: cargo fmt -- --check diff --git a/Cargo.lock b/Cargo.lock index 6bf02df..5f611e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,13 +19,15 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -37,11 +39,32 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "assert-json-diff" @@ -55,13 +78,38 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + +[[package]] +name = "atomic-write-file" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" +dependencies = [ + "nix", + "rand", ] [[package]] @@ -77,13 +125,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.3.4", "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -102,13 +150,47 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +dependencies = [ + "async-trait", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "axum-client-ip" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ef117890a418b7832678d9ea1e1c08456dd7b2fd1dadb9676cd6f0fe7eb4b21" dependencies = [ - "axum", + "axum 0.6.20", "forwarded-header-value", "serde", ] @@ -122,25 +204,46 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", "tower-service", ] +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "axum-test-helper" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d349b3174ceac58442ea1f768233c817e59447c0343be2584fca9f0ed71d3a" dependencies = [ - "axum", + "axum 0.6.20", "bytes", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "reqwest", "serde", "tokio", @@ -165,9 +268,15 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bitflags" @@ -177,9 +286,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] [[package]] name = "bumpalo" @@ -187,6 +308,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.5.0" @@ -200,7 +327,7 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", - "axum", + "axum 0.6.20", "axum-client-ip", "axum-test-helper", "base64", @@ -235,7 +362,7 @@ version = "0.1.0" dependencies = [ "anyhow", "assert-json-diff", - "axum", + "axum 0.7.4", "capture", "envconfig", "futures", @@ -269,6 +396,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.0", +] + [[package]] name = "cmake" version = "0.1.50" @@ -292,11 +433,17 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -304,9 +451,33 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc16" @@ -325,34 +496,45 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "cfg-if", + "generic-array", + "typenum", ] [[package]] @@ -362,18 +544,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", ] +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] @@ -383,6 +577,24 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast" version = "0.11.0" @@ -394,6 +606,9 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -432,12 +647,39 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", ] [[package]] @@ -446,11 +688,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -465,6 +713,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -488,9 +747,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -513,9 +772,9 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -528,9 +787,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -538,49 +797,60 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -590,9 +860,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -606,11 +876,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -619,9 +899,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -649,17 +929,36 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", - "indexmap 1.9.3", + "http 0.2.11", + "indexmap 2.2.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.0.0", + "indexmap 2.2.2", "slab", "tokio", "tokio-util", @@ -683,21 +982,182 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.3", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "hook-api" +version = "0.1.0" +dependencies = [ + "axum 0.7.4", + "envconfig", + "eyre", + "hook-common", + "http-body-util", + "metrics", + "metrics-exporter-prometheus", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "tokio", + "tower", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "hook-common" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum 0.7.4", + "chrono", + "http 0.2.11", + "metrics", + "metrics-exporter-prometheus", + "regex", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "thiserror", + "time", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "hook-janitor" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum 0.7.4", + "envconfig", + "eyre", + "futures", + "hook-common", + "http-body-util", + "metrics", + "metrics-exporter-prometheus", + "rdkafka", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "thiserror", + "time", + "tokio", + "tower", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "hook-worker" +version = "0.1.0" +dependencies = [ + "axum 0.7.4", + "chrono", + "envconfig", + "futures", + "hook-common", + "http 0.2.11", + "metrics", + "reqwest", + "serde", + "serde_derive", + "sqlx", + "thiserror", + "time", + "tokio", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "http" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] [[package]] name = "http" -version = "0.2.9" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" dependencies = [ "bytes", "fnv", @@ -706,12 +1166,35 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -735,35 +1218,54 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.5", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.2", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -776,22 +1278,67 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "1.9.3" @@ -804,19 +1351,19 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.3", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itertools" @@ -827,17 +1374,26 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -847,18 +1403,38 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" -version = "0.2.149" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "libc", @@ -868,15 +1444,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -897,15 +1473,6 @@ dependencies = [ "libc", ] -[[package]] -name = "mach2" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" -dependencies = [ - "libc", -] - [[package]] name = "matchers" version = "0.1.0" @@ -922,72 +1489,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] -name = "memchr" -version = "2.6.4" +name = "md-5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] [[package]] -name = "memoffset" -version = "0.9.0" +name = "memchr" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "metrics" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +checksum = "77b9e10a211c839210fd7f99954bda26e5f8e26ec686ad68da6a32df7c80e782" dependencies = [ "ahash", - "metrics-macros", "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +checksum = "83a4c4718a371ddfb7806378f23617876eea8b82e5ff1324516bcd283249d9ea" dependencies = [ "base64", - "hyper", + "hyper 0.14.28", + "hyper-tls", "indexmap 1.9.3", "ipnet", "metrics", "metrics-util", - "quanta 0.11.1", + "quanta 0.12.2", "thiserror", "tokio", - "tracing", -] - -[[package]] -name = "metrics-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.38", + "tracing", ] [[package]] name = "metrics-util" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +checksum = "2670b8badcc285d486261e2e9f1615b506baff91427b61bd336a472b65bbf5ed" dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.13.1", "metrics", "num_cpus", - "quanta 0.11.1", + "quanta 0.12.2", "sketches-ddsketch", ] @@ -1007,24 +1564,30 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1072,12 +1635,33 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "libc", +] + [[package]] name = "no-std-compat" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonempty" version = "0.7.0" @@ -1106,6 +1690,50 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -1113,6 +1741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1148,26 +1777,26 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -1184,7 +1813,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -1195,9 +1824,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -1213,7 +1842,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.0.2", + "indexmap 2.2.2", "js-sys", "once_cell", "pin-project-lite", @@ -1229,7 +1858,7 @@ checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" dependencies = [ "async-trait", "futures-core", - "http", + "http 0.2.11", "opentelemetry", "opentelemetry-proto", "opentelemetry-semantic-conventions", @@ -1263,9 +1892,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5b3ce3f5705e2ae493be467a0b23be4bc563c193cdb7713e55372c89a906b34" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", @@ -1285,9 +1914,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.1.1" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", ] @@ -1310,41 +1939,56 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -1359,17 +2003,44 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" @@ -1385,7 +2056,7 @@ checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ "difflib", "float-cmp", - "itertools", + "itertools 0.10.5", "normalize-line-endings", "predicates-core", "regex", @@ -1419,9 +2090,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -1443,7 +2114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -1459,7 +2130,7 @@ dependencies = [ "libc", "mach", "once_cell", - "raw-cpuid", + "raw-cpuid 10.7.0", "wasi 0.10.2+wasi-snapshot-preview1", "web-sys", "winapi", @@ -1467,15 +2138,14 @@ dependencies = [ [[package]] name = "quanta" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" dependencies = [ "crossbeam-utils", "libc", - "mach2", "once_cell", - "raw-cpuid", + "raw-cpuid 11.0.1", "wasi 0.11.0+wasi-snapshot-preview1", "web-sys", "winapi", @@ -1483,9 +2153,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1529,11 +2199,20 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "raw-cpuid" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +dependencies = [ + "bitflags 2.4.2", +] + [[package]] name = "rdkafka" -version = "0.36.0" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54f02a5a40220f8a2dfa47ddb38ba9064475a5807a69504b6f91711df2eea63" +checksum = "1beea247b9a7600a81d4cc33f659ce1a77e1988323d7d2809c7ed1c21f4c316d" dependencies = [ "futures-channel", "futures-util", @@ -1545,6 +2224,7 @@ dependencies = [ "serde_json", "slab", "tokio", + "tracing", ] [[package]] @@ -1580,7 +2260,7 @@ dependencies = [ "rand", "ryu", "sha1_smol", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tokio-util", "url", @@ -1595,15 +2275,6 @@ dependencies = [ "redis", ] -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -1615,14 +2286,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.0" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.0", - "regex-syntax 0.8.0", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", ] [[package]] @@ -1636,13 +2307,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.0" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d58da636bd923eae52b7e9120271cbefb16f399069ee566ca5ebf9c30e32238" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.0", + "regex-syntax 0.8.2", ] [[package]] @@ -1653,25 +2324,25 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-tls", "ipnet", "js-sys", @@ -1682,9 +2353,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -1698,6 +2371,26 @@ dependencies = [ "winreg", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1706,15 +2399,24 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", ] [[package]] @@ -1725,165 +2427,456 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "serde_json" +version = "1.0.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ - "windows-sys", + "libc", + "windows-sys 0.48.0", ] [[package]] -name = "scopeguard" -version = "1.2.0" +name = "spin" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] -name = "security-framework" -version = "2.9.2" +name = "spin" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", + "lock_api", ] [[package]] -name = "security-framework-sys" -version = "2.9.1" +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ - "core-foundation-sys", - "libc", + "base64ct", + "der", ] [[package]] -name = "serde" -version = "1.0.188" +name = "sqlformat" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "serde_derive", + "itertools 0.12.1", + "nom", + "unicode_categories", ] [[package]] -name = "serde_derive" -version = "1.0.188" +name = "sqlx" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.38", + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", ] [[package]] -name = "serde_json" -version = "1.0.107" +name = "sqlx-core" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" dependencies = [ - "itoa", - "ryu", + "ahash", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.2.2", + "log", + "memchr", + "native-tls", + "once_cell", + "paste", + "percent-encoding", "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", ] [[package]] -name = "serde_path_to_error" -version = "0.1.14" +name = "sqlx-macros" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" dependencies = [ - "itoa", - "serde", + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", ] [[package]] -name = "serde_urlencoded" -version = "0.7.1" +name = "sqlx-macros-core" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" dependencies = [ - "form_urlencoded", - "itoa", - "ryu", + "atomic-write-file", + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", ] [[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - -[[package]] -name = "sharded-slab" -version = "0.1.7" +name = "sqlx-mysql" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" dependencies = [ - "lazy_static", + "atoi", + "base64", + "bitflags 2.4.2", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", ] [[package]] -name = "signal-hook-registry" -version = "1.4.1" +name = "sqlx-postgres" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" dependencies = [ - "libc", + "atoi", + "base64", + "bitflags 2.4.2", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", ] [[package]] -name = "sketches-ddsketch" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" - -[[package]] -name = "slab" -version = "0.4.9" +name = "sqlx-sqlite" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" dependencies = [ - "autocfg", + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid", ] [[package]] -name = "smallvec" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" - -[[package]] -name = "socket2" -version = "0.4.9" +name = "stringprep" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" dependencies = [ - "libc", - "winapi", + "finl_unicode", + "unicode-bidi", + "unicode-normalization", ] [[package]] -name = "socket2" -version = "0.5.4" +name = "subtle" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" -dependencies = [ - "libc", - "windows-sys", -] +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -1898,9 +2891,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -1936,15 +2929,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.4.1", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1955,22 +2947,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -1985,12 +2977,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -2004,10 +2998,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -2028,9 +3023,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -2040,9 +3035,9 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2057,13 +3052,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] @@ -2089,9 +3084,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2103,9 +3098,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" [[package]] name = "toml_edit" @@ -2113,7 +3108,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.2", "toml_datetime", "winnow", ] @@ -2125,15 +3120,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", - "axum", + "axum 0.6.20", "base64", "bytes", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -2172,12 +3167,12 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -2203,12 +3198,12 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6be418f6d18863291f0a7fa1da1de71495a19a54b5fb44969136f731a47e86" dependencies = [ - "axum", + "axum 0.6.20", "forwarded-header-value", "futures", "futures-core", "governor", - "http", + "http 0.2.11", "pin-project", "thiserror", "tokio", @@ -2219,11 +3214,10 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -2232,36 +3226,25 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", ] -[[package]] -name = "tracing-log" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" -dependencies = [ - "lazy_static", - "log", - "tracing-core", -] - [[package]] name = "tracing-log" version = "0.2.0" @@ -2286,16 +3269,16 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log 0.2.0", + "tracing-log", "tracing-subscriber", "web-time", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "matchers", "nu-ansi-term", @@ -2306,14 +3289,20 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log 0.1.3", + "tracing-log", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicase" @@ -2326,9 +3315,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -2345,11 +3334,23 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -2364,10 +3365,12 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "uuid" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ + "atomic", + "getrandom", "serde", ] @@ -2412,9 +3415,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2422,24 +3425,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -2449,9 +3452,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2459,28 +3462,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -2491,9 +3494,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -2501,14 +3504,20 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" + [[package]] name = "winapi" version = "0.3.9" @@ -2531,13 +3540,31 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -2546,13 +3573,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -2561,47 +3603,89 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" -version = "0.5.16" +version = "0.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +checksum = "a7cad8365489051ae9f054164e459304af2e7e9bb407c958076c8bf4aef52da5" dependencies = [ "memchr", ] @@ -2613,5 +3697,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", ] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/Cargo.toml b/Cargo.toml index faf9d2a..db01334 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,35 +3,63 @@ resolver = "2" members = [ "capture", - "capture-server" + "capture-server", + "hook-common", + "hook-api", + "hook-worker", + "hook-janitor", ] [profile.release] -debug = 2 # https://www.polarsignals.com/docs/rust +debug = 2 # https://www.polarsignals.com/docs/rust [workspace.dependencies] +anyhow = "1.0" assert-json-diff = "2.0.2" -axum = "0.6.15" +async-trait = "0.1.74" +axum = { version = "0.7.1", features = ["http2"] } axum-client-ip = "0.4.1" -tokio = { version = "1.0", features = ["full"] } -tracing = "0.1" -tracing-subscriber = "0.3" -serde = { version = "1.0.160", features = ["derive"] } -serde_json = "1.0.96" -governor = {version = "0.5.1", features=["dashmap"]} -tower_governor = "0.0.4" -time = { version = "0.3.20", features = ["formatting", "macros", "parsing", "serde"] } -tower-http = { version = "0.4.0", features = ["cors", "trace"] } +base64 = "0.21.1" bytes = "1" -anyhow = "1.0" +chrono = { version = "0.4" } +envconfig = "0.10.0" +eyre = "0.6.9" flate2 = "1.0" -base64 = "0.21.1" -uuid = { version = "1.3.3", features = ["serde"] } -async-trait = "0.1.68" -serde_urlencoded = "0.7.1" +futures = { version = "0.3.29" } +governor = { version = "0.5.1", features = ["dashmap"] } +http = { version = "0.2" } +http-body-util = "0.1.0" +metrics = "0.22.0" +metrics-exporter-prometheus = "0.13.0" rand = "0.8.5" -rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl"] } -metrics = "0.21.1" -metrics-exporter-prometheus = "0.12.1" -thiserror = "1.0.48" -envconfig = "0.10.0" +rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl", "tracing"] } +regex = "1.10.2" +reqwest = { version = "0.11" } +serde = { version = "1.0", features = ["derive"] } +serde_derive = { version = "1.0" } +serde_json = { version = "1.0" } +serde_urlencoded = "0.7.1" +sqlx = { version = "0.7", features = [ + "chrono", + "json", + "migrate", + "postgres", + "runtime-tokio", + "tls-native-tls", + "uuid", +] } +thiserror = { version = "1.0" } +time = { version = "0.3.20", features = [ + "formatting", + "macros", + "parsing", + "serde", +] } +tokio = { version = "1.34.0", features = ["full"] } +tower = "0.4.13" +tower_governor = "0.0.4" +tower-http = { version = "0.4.0", features = ["cors", "trace"] } +tracing = "0.1.40" +tracing-subscriber = "0.3.18" +url = { version = "2.5.0 " } +uuid = { version = "1.6.1", features = ["v7", "serde"] } diff --git a/Dockerfile b/Dockerfile index c9e3d8c..7bbfda6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,15 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.72.0-buster AS chef -WORKDIR app +FROM docker.io/lukemathwalker/cargo-chef:latest-rust-1.74.0-buster AS chef +ARG BIN +WORKDIR /app FROM chef AS planner +ARG BIN + COPY . . -RUN cargo chef prepare --recipe-path recipe.json +RUN cargo chef prepare --recipe-path recipe.json --bin $BIN FROM chef AS builder +ARG BIN # Ensure working C compile setup (not installed by default in arm64 images) RUN apt update && apt install build-essential cmake -y @@ -14,13 +18,21 @@ COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release --bin capture-server +RUN cargo build --release --bin $BIN FROM debian:bullseye-20230320-slim AS runtime -WORKDIR app +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + "ca-certificates" \ + && \ + rm -rf /var/lib/apt/lists/* + +ARG BIN +ENV ENTRYPOINT=/usr/local/bin/$BIN +WORKDIR /app USER nobody -COPY --from=builder /app/target/release/capture-server /usr/local/bin -ENTRYPOINT ["/usr/local/bin/capture-server"] +COPY --from=builder /app/target/release/$BIN /usr/local/bin +ENTRYPOINT [ $ENTRYPOINT ] diff --git a/Dockerfile.migrate b/Dockerfile.migrate new file mode 100644 index 0000000..4779077 --- /dev/null +++ b/Dockerfile.migrate @@ -0,0 +1,16 @@ +FROM docker.io/library/rust:1.74.0-buster as builder + +RUN apt update && apt install build-essential cmake -y +RUN cargo install sqlx-cli@0.7.3 --no-default-features --features native-tls,postgres --root /app/target/release/ + +FROM debian:bullseye-20230320-slim AS runtime +WORKDIR /sqlx + +ADD bin /sqlx/bin/ +ADD migrations /sqlx/migrations/ + +COPY --from=builder /app/target/release/bin/sqlx /usr/local/bin + +RUN chmod +x ./bin/migrate + +CMD ["./bin/migrate"] diff --git a/README.md b/README.md index cae8981..9e89577 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,40 @@ -# capture +# hog-rs + +PostHog Rust service monorepo. This is *not* the Rust client library for PostHog. + +## capture This is a rewrite of [capture.py](https://github.com/PostHog/posthog/blob/master/posthog/api/capture.py), in Rust. -## Why? +### Why? Capture is very simple. It takes some JSON, checks a key in Redis, and then pushes onto Kafka. It's mostly IO bound. We currently use far too much compute to run this service, and it could be more efficient. This effort should not take too long to complete, but should massively reduce our CPU usage - and therefore spend. -## How? +### How? I'm trying to ensure the rewrite at least vaguely resembles the Python version. This will both minimize accidental regressions, but also serve as a "rosetta stone" for engineers at PostHog who have not written Rust before. + +## rusty-hook +A reliable and performant webhook system for PostHog + +### Requirements + +1. [Rust](https://www.rust-lang.org/tools/install). +2. [Docker](https://docs.docker.com/engine/install/), or [podman](https://podman.io/docs/installation) and [podman-compose](https://github.com/containers/podman-compose#installation): To setup development stack. + +### Testing + +1. Start development stack: +```bash +docker compose -f docker-compose.yml up -d --wait +``` + +2. Test: +```bash +# Note that tests require a DATABASE_URL environment variable to be set, e.g.: +# export DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database +# But there is an .env file in the project root that should be used automatically. +cargo test +``` diff --git a/bin/migrate b/bin/migrate new file mode 100755 index 0000000..6e36fc4 --- /dev/null +++ b/bin/migrate @@ -0,0 +1,4 @@ +#!/bin/sh + +sqlx database create +sqlx migrate run diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 3b2b100..bd8f79f 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = { workspace = true } +axum = { version = "0.6.15" } # TODO: Bring up to date with the workspace. axum-client-ip = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } @@ -29,12 +29,16 @@ rdkafka = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } thiserror = { workspace = true } -redis = { version="0.23.3", features=["tokio-comp", "cluster", "cluster-async"] } +redis = { version = "0.23.3", features = [ + "tokio-comp", + "cluster", + "cluster-async", +] } envconfig = { workspace = true } dashmap = "5.5.3" [dev-dependencies] -assert-json-diff = { workspace = true } +assert-json-diff = { workspace = true } axum-test-helper = "0.2.0" mockall = "0.11.2" redis-test = "0.2.3" diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 6a90378..d7303fd 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -101,7 +101,7 @@ pub async fn event( tracing::Span::current().record("token", &token); - counter!("capture_events_received_total", events.len() as u64); + counter!("capture_events_received_total").increment(events.len() as u64); let sent_at = meta.sent_at.and_then(|value| { let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases diff --git a/capture/src/limiters/overflow.rs b/capture/src/limiters/overflow.rs index 0e91a99..65bf14a 100644 --- a/capture/src/limiters/overflow.rs +++ b/capture/src/limiters/overflow.rs @@ -48,7 +48,7 @@ impl OverflowLimiter { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(10)); loop { interval.tick().await; - gauge!("partition_limits_key_count", self.limiter.len() as f64); + gauge!("partition_limits_key_count").set(self.limiter.len() as f64); } } diff --git a/capture/src/prometheus.rs b/capture/src/prometheus.rs index f225520..6f5dc12 100644 --- a/capture/src/prometheus.rs +++ b/capture/src/prometheus.rs @@ -7,11 +7,11 @@ use metrics::counter; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; pub fn report_dropped_events(cause: &'static str, quantity: u64) { - counter!("capture_events_dropped_total", quantity, "cause" => cause); + counter!("capture_events_dropped_total", "cause" => cause).increment(quantity); } pub fn report_overflow_partition(quantity: u64) { - counter!("capture_partition_key_capacity_exceeded_total", quantity); + counter!("capture_partition_key_capacity_exceeded_total").increment(quantity); } pub fn setup_metrics_recorder() -> PrometheusHandle { @@ -62,8 +62,8 @@ pub async fn track_metrics(req: Request, next: Next) -> impl IntoRespon ("status", status), ]; - metrics::increment_counter!("http_requests_total", &labels); - metrics::histogram!("http_requests_duration_seconds", latency, &labels); + metrics::counter!("http_requests_total", &labels).increment(1); + metrics::histogram!("http_requests_duration_seconds", &labels).record(latency); response } diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index dc57c11..4a2bd94 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -1,7 +1,7 @@ use std::time::Duration; use async_trait::async_trait; -use metrics::{absolute_counter, counter, gauge, histogram}; +use metrics::{counter, gauge, histogram}; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord, Producer}; use rdkafka::util::Timeout; @@ -28,53 +28,52 @@ impl rdkafka::ClientContext for KafkaContext { self.liveness.report_healthy_blocking(); // Update exported metrics - gauge!("capture_kafka_callback_queue_depth", stats.replyq as f64); - gauge!("capture_kafka_producer_queue_depth", stats.msg_cnt as f64); - gauge!( - "capture_kafka_producer_queue_depth_limit", - stats.msg_max as f64 - ); - gauge!("capture_kafka_producer_queue_bytes", stats.msg_max as f64); - gauge!( - "capture_kafka_producer_queue_bytes_limit", - stats.msg_size_max as f64 - ); + gauge!("capture_kafka_callback_queue_depth",).set(stats.replyq as f64); + gauge!("capture_kafka_producer_queue_depth",).set(stats.msg_cnt as f64); + gauge!("capture_kafka_producer_queue_depth_limit",).set(stats.msg_max as f64); + gauge!("capture_kafka_producer_queue_bytes",).set(stats.msg_max as f64); + gauge!("capture_kafka_producer_queue_bytes_limit",).set(stats.msg_size_max as f64); for (topic, stats) in stats.topics { gauge!( "capture_kafka_produce_avg_batch_size_bytes", - stats.batchsize.avg as f64, - "topic" => topic.clone() - ); + "topic" => topic.clone() + ) + .set(stats.batchsize.avg as f64); gauge!( "capture_kafka_produce_avg_batch_size_events", - stats.batchcnt.avg as f64, + "topic" => topic - ); + ) + .set(stats.batchcnt.avg as f64); } for (_, stats) in stats.brokers { let id_string = format!("{}", stats.nodeid); gauge!( "capture_kafka_broker_requests_pending", - stats.outbuf_cnt as f64, + "broker" => id_string.clone() - ); + ) + .set(stats.outbuf_cnt as f64); gauge!( "capture_kafka_broker_responses_awaiting", - stats.waitresp_cnt as f64, + "broker" => id_string.clone() - ); - absolute_counter!( + ) + .set(stats.waitresp_cnt as f64); + counter!( "capture_kafka_broker_tx_errors_total", - stats.txerrs, + "broker" => id_string.clone() - ); - absolute_counter!( + ) + .absolute(stats.txerrs); + counter!( "capture_kafka_broker_rx_errors_total", - stats.rxerrs, + "broker" => id_string - ); + ) + .absolute(stats.rxerrs); } } } @@ -180,7 +179,7 @@ impl KafkaSink { match delivery.await { Err(_) => { // Cancelled due to timeout while retrying - counter!("capture_kafka_produce_errors_total", 1); + counter!("capture_kafka_produce_errors_total").increment(1); error!("failed to produce to Kafka before write timeout"); Err(CaptureError::RetryableSinkError) } @@ -191,12 +190,12 @@ impl KafkaSink { } Ok(Err((err, _))) => { // Unretriable produce error - counter!("capture_kafka_produce_errors_total", 1); + counter!("capture_kafka_produce_errors_total").increment(1); error!("failed to produce to Kafka: {}", err); Err(CaptureError::RetryableSinkError) } Ok(Ok(_)) => { - counter!("capture_events_ingested_total", 1); + counter!("capture_events_ingested_total").increment(1); Ok(()) } } @@ -210,7 +209,7 @@ impl Event for KafkaSink { let limited = self.partition.is_limited(&event.key()); let ack = Self::kafka_send(self.producer.clone(), self.topic.clone(), event, limited).await?; - histogram!("capture_event_batch_size", 1.0); + histogram!("capture_event_batch_size").record(1.0); Self::process_ack(ack) .instrument(info_span!("ack_wait_one")) .await @@ -253,7 +252,7 @@ impl Event for KafkaSink { .instrument(info_span!("ack_wait_many")) .await?; - histogram!("capture_event_batch_size", batch_size as f64); + histogram!("capture_event_batch_size").record(batch_size as f64); Ok(()) } } diff --git a/capture/src/sinks/print.rs b/capture/src/sinks/print.rs index 50bc1ad..5e71899 100644 --- a/capture/src/sinks/print.rs +++ b/capture/src/sinks/print.rs @@ -12,7 +12,7 @@ pub struct PrintSink {} impl Event for PrintSink { async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { info!("single event: {:?}", event); - counter!("capture_events_ingested_total", 1); + counter!("capture_events_ingested_total").increment(1); Ok(()) } @@ -20,8 +20,8 @@ impl Event for PrintSink { let span = tracing::span!(tracing::Level::INFO, "batch of events"); let _enter = span.enter(); - histogram!("capture_event_batch_size", events.len() as f64); - counter!("capture_events_ingested_total", events.len() as u64); + histogram!("capture_event_batch_size").record(events.len() as f64); + counter!("capture_events_ingested_total").increment(events.len() as u64); for event in events { info!("event: {:?}", event); } diff --git a/docker-compose.yml b/docker-compose.yml index 804ae78..4709d1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,51 +1,89 @@ version: "3" services: - zookeeper: - image: zookeeper:3.7.0 - restart: on-failure + zookeeper: + image: zookeeper:3.7.0 + restart: on-failure - kafka: - image: ghcr.io/posthog/kafka-container:v2.8.2 - restart: on-failure - depends_on: - - zookeeper - environment: - KAFKA_BROKER_ID: 1001 - KAFKA_CFG_RESERVED_BROKER_MAX_ID: 1001 - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER: 'true' - ports: - - '9092:9092' - healthcheck: - test: kafka-cluster.sh cluster-id --bootstrap-server localhost:9092 || exit 1 - interval: 3s - timeout: 10s - retries: 10 + kafka: + image: ghcr.io/posthog/kafka-container:v2.8.2 + restart: on-failure + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1001 + KAFKA_CFG_RESERVED_BROKER_MAX_ID: 1001 + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: "true" + ports: + - "9092:9092" + healthcheck: + test: kafka-cluster.sh cluster-id --bootstrap-server localhost:9092 || exit 1 + interval: 3s + timeout: 10s + retries: 10 - redis: - image: redis:6.2.7-alpine - restart: on-failure - command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb - ports: - - '6379:6379' - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 3s - timeout: 10s - retries: 10 + redis: + image: redis:6.2.7-alpine + restart: on-failure + command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 10s + retries: 10 - kafka-ui: - image: provectuslabs/kafka-ui:latest - profiles: ["ui"] - ports: - - '8080:8080' - depends_on: - - zookeeper - - kafka - environment: - KAFKA_CLUSTERS_0_NAME: local - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 - KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181 + kafka-ui: + image: provectuslabs/kafka-ui:latest + profiles: ["ui"] + ports: + - "8080:8080" + depends_on: + - zookeeper + - kafka + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 + KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181 + + db: + container_name: db + image: docker.io/library/postgres:16-alpine + restart: on-failure + environment: + POSTGRES_USER: posthog + POSTGRES_DB: posthog + POSTGRES_PASSWORD: posthog + healthcheck: + test: ["CMD-SHELL", "pg_isready -U posthog"] + interval: 5s + timeout: 5s + ports: + - "15432:5432" + command: postgres -c max_connections=1000 -c idle_in_transaction_session_timeout=300000 + + setup_test_db: + container_name: setup-test-db + build: + context: . + dockerfile: Dockerfile.migrate + restart: on-failure + depends_on: + db: + condition: service_healthy + restart: true + environment: + DATABASE_URL: postgres://posthog:posthog@db:5432/test_database + + echo_server: + image: docker.io/library/caddy:2 + container_name: echo-server + restart: on-failure + ports: + - "18081:8081" + volumes: + - ./docker/echo-server/Caddyfile:/etc/caddy/Caddyfile diff --git a/docker/echo-server/Caddyfile b/docker/echo-server/Caddyfile new file mode 100644 index 0000000..a13ac68 --- /dev/null +++ b/docker/echo-server/Caddyfile @@ -0,0 +1,17 @@ +{ + auto_https off +} + +:8081 + +route /echo { + respond `{http.request.body}` 200 { + close + } +} + +route /fail { + respond `{http.request.body}` 400 { + close + } +} diff --git a/hook-api/Cargo.toml b/hook-api/Cargo.toml new file mode 100644 index 0000000..96c897c --- /dev/null +++ b/hook-api/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "hook-api" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = { workspace = true } +envconfig = { workspace = true } +eyre = { workspace = true } +hook-common = { path = "../hook-common" } +http-body-util = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sqlx = { workspace = true } +tokio = { workspace = true } +tower = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } diff --git a/hook-api/src/config.rs b/hook-api/src/config.rs new file mode 100644 index 0000000..55fa404 --- /dev/null +++ b/hook-api/src/config.rs @@ -0,0 +1,25 @@ +use envconfig::Envconfig; + +#[derive(Envconfig)] +pub struct Config { + #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] + pub host: String, + + #[envconfig(from = "BIND_PORT", default = "3300")] + pub port: u16, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub database_url: String, + + #[envconfig(default = "default")] + pub queue_name: String, + + #[envconfig(default = "100")] + pub max_pg_connections: u32, +} + +impl Config { + pub fn bind(&self) -> String { + format!("{}:{}", self.host, self.port) + } +} diff --git a/hook-api/src/handlers/app.rs b/hook-api/src/handlers/app.rs new file mode 100644 index 0000000..7b1e840 --- /dev/null +++ b/hook-api/src/handlers/app.rs @@ -0,0 +1,49 @@ +use axum::{routing, Router}; + +use hook_common::pgqueue::PgQueue; + +use super::webhook; + +pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { + router + .route("/", routing::get(index)) + .route("/_readiness", routing::get(index)) + .route("/_liveness", routing::get(index)) // No async loop for now, just check axum health + .route("/webhook", routing::post(webhook::post).with_state(pg_pool)) +} + +pub async fn index() -> &'static str { + "rusty-hook api" +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + body::Body, + http::{Request, StatusCode}, + }; + use hook_common::pgqueue::PgQueue; + use http_body_util::BodyExt; // for `collect` + use sqlx::PgPool; + use tower::ServiceExt; // for `call`, `oneshot`, and `ready` + + #[sqlx::test(migrations = "../migrations")] + async fn index(db: PgPool) { + let pg_queue = PgQueue::new_from_pool("test_index", db) + .await + .expect("failed to construct pg_queue"); + + let app = add_routes(Router::new(), pg_queue); + + let response = app + .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"rusty-hook api"); + } +} diff --git a/hook-api/src/handlers/mod.rs b/hook-api/src/handlers/mod.rs new file mode 100644 index 0000000..e392f8a --- /dev/null +++ b/hook-api/src/handlers/mod.rs @@ -0,0 +1,4 @@ +mod app; +mod webhook; + +pub use app::add_routes; diff --git a/hook-api/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs new file mode 100644 index 0000000..16ebc6d --- /dev/null +++ b/hook-api/src/handlers/webhook.rs @@ -0,0 +1,298 @@ +use std::time::Instant; + +use axum::{extract::State, http::StatusCode, Json}; +use hook_common::webhook::{WebhookJobMetadata, WebhookJobParameters}; +use serde_derive::Deserialize; +use url::Url; + +use hook_common::pgqueue::{NewJob, PgQueue}; +use serde::Serialize; +use tracing::{debug, error}; + +const MAX_BODY_SIZE: usize = 1_000_000; + +#[derive(Serialize, Deserialize)] +pub struct WebhookPostResponse { + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +/// The body of a request made to create a webhook Job. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookPostRequestBody { + parameters: WebhookJobParameters, + metadata: WebhookJobMetadata, + + #[serde(default = "default_max_attempts")] + max_attempts: u32, +} + +fn default_max_attempts() -> u32 { + 3 +} + +pub async fn post( + State(pg_queue): State, + Json(payload): Json, +) -> Result, (StatusCode, Json)> { + debug!("received payload: {:?}", payload); + + if payload.parameters.body.len() > MAX_BODY_SIZE { + return Err(( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("body too large".to_owned()), + }), + )); + } + + let url_hostname = get_hostname(&payload.parameters.url)?; + // We could cast to i32, but this ensures we are not wrapping. + let max_attempts = i32::try_from(payload.max_attempts).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("invalid number of max attempts".to_owned()), + }), + ) + })?; + let job = NewJob::new( + max_attempts, + payload.metadata, + payload.parameters, + url_hostname.as_str(), + ); + + let start_time = Instant::now(); + + pg_queue.enqueue(job).await.map_err(internal_error)?; + + let elapsed_time = start_time.elapsed().as_secs_f64(); + metrics::histogram!("webhook_api_enqueue").record(elapsed_time); + + Ok(Json(WebhookPostResponse { error: None })) +} + +fn internal_error(err: E) -> (StatusCode, Json) +where + E: std::error::Error, +{ + error!("internal error: {}", err); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(WebhookPostResponse { + error: Some(err.to_string()), + }), + ) +} + +fn get_hostname(url_str: &str) -> Result)> { + let url = Url::parse(url_str).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("could not parse url".to_owned()), + }), + ) + })?; + + match url.host_str() { + Some(hostname) => Ok(hostname.to_owned()), + None => Err(( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("couldn't extract hostname from url".to_owned()), + }), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::{ + body::Body, + http::{self, Request, StatusCode}, + Router, + }; + use hook_common::pgqueue::PgQueue; + use hook_common::webhook::{HttpMethod, WebhookJobParameters}; + use http_body_util::BodyExt; + use sqlx::PgPool; // for `collect` + use std::collections; + use tower::ServiceExt; // for `call`, `oneshot`, and `ready` + + use crate::handlers::app::add_routes; + + #[sqlx::test(migrations = "../migrations")] + async fn webhook_success(db: PgPool) { + let pg_queue = PgQueue::new_from_pool("test_index", db) + .await + .expect("failed to construct pg_queue"); + + let app = add_routes(Router::new(), pg_queue); + + let mut headers = collections::HashMap::new(); + headers.insert("Content-Type".to_owned(), "application/json".to_owned()); + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&WebhookPostRequestBody { + parameters: WebhookJobParameters { + headers, + method: HttpMethod::POST, + url: "http://example.com/".to_owned(), + body: r#"{"a": "b"}"#.to_owned(), + }, + metadata: WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }, + max_attempts: 1, + }) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"{}"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn webhook_bad_url(db: PgPool) { + let pg_queue = PgQueue::new_from_pool("test_index", db) + .await + .expect("failed to construct pg_queue"); + + let app = add_routes(Router::new(), pg_queue); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&WebhookPostRequestBody { + parameters: WebhookJobParameters { + headers: collections::HashMap::new(), + method: HttpMethod::POST, + url: "invalid".to_owned(), + body: r#"{"a": "b"}"#.to_owned(), + }, + metadata: WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }, + max_attempts: 1, + }) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[sqlx::test(migrations = "../migrations")] + async fn webhook_payload_missing_fields(db: PgPool) { + let pg_queue = PgQueue::new_from_pool("test_index", db) + .await + .expect("failed to construct pg_queue"); + + let app = add_routes(Router::new(), pg_queue); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body("{}".to_owned()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + } + + #[sqlx::test(migrations = "../migrations")] + async fn webhook_payload_not_json(db: PgPool) { + let pg_queue = PgQueue::new_from_pool("test_index", db) + .await + .expect("failed to construct pg_queue"); + + let app = add_routes(Router::new(), pg_queue); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body("x".to_owned()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[sqlx::test(migrations = "../migrations")] + async fn webhook_payload_body_too_large(db: PgPool) { + let pg_queue = PgQueue::new_from_pool("test_index", db) + .await + .expect("failed to construct pg_queue"); + + let app = add_routes(Router::new(), pg_queue); + + let bytes: Vec = vec![b'a'; 1_000_000 * 2]; + let long_string = String::from_utf8_lossy(&bytes); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&WebhookPostRequestBody { + parameters: WebhookJobParameters { + headers: collections::HashMap::new(), + method: HttpMethod::POST, + url: "http://example.com".to_owned(), + body: long_string.to_string(), + }, + metadata: WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }, + max_attempts: 1, + }) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } +} diff --git a/hook-api/src/main.rs b/hook-api/src/main.rs new file mode 100644 index 0000000..9a9a9fd --- /dev/null +++ b/hook-api/src/main.rs @@ -0,0 +1,44 @@ +use axum::Router; +use config::Config; +use envconfig::Envconfig; +use eyre::Result; + +use hook_common::metrics::setup_metrics_routes; +use hook_common::pgqueue::PgQueue; + +mod config; +mod handlers; + +async fn listen(app: Router, bind: String) -> Result<()> { + let listener = tokio::net::TcpListener::bind(bind).await?; + + axum::serve(listener, app).await?; + + Ok(()) +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let config = Config::init_from_env().expect("failed to load configuration from env"); + + let pg_queue = PgQueue::new( + // TODO: Coupling the queue name to the PgQueue object doesn't seem ideal from the api + // side, but we don't need more than one queue for now. + &config.queue_name, + &config.database_url, + config.max_pg_connections, + "hook-api", + ) + .await + .expect("failed to initialize queue"); + + let app = handlers::add_routes(Router::new(), pg_queue); + let app = setup_metrics_routes(app); + + match listen(app, config.bind()).await { + Ok(_) => {} + Err(e) => tracing::error!("failed to start hook-api http server, {}", e), + } +} diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml new file mode 100644 index 0000000..ea7ce2f --- /dev/null +++ b/hook-common/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "hook-common" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = { workspace = true } +axum = { workspace = true, features = ["http2"] } +chrono = { workspace = true } +http = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } +reqwest = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sqlx = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } # We need a runtime for async tests diff --git a/hook-common/README.md b/hook-common/README.md new file mode 100644 index 0000000..d277a6c --- /dev/null +++ b/hook-common/README.md @@ -0,0 +1,2 @@ +# hook-common +Library of common utilities used by rusty-hook. diff --git a/hook-common/src/health.rs b/hook-common/src/health.rs new file mode 100644 index 0000000..c5c79c9 --- /dev/null +++ b/hook-common/src/health.rs @@ -0,0 +1,346 @@ +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use std::collections::HashMap; +use std::ops::Add; +use std::sync::{Arc, RwLock}; + +use time::Duration; +use tokio::sync::mpsc; +use tracing::{info, warn}; + +/// Health reporting for components of the service. +/// +/// FIXME: copied over from capture, make sure to keep in sync until we share the crate +/// +/// The capture server contains several asynchronous loops, and +/// the process can only be trusted with user data if all the +/// loops are properly running and reporting. +/// +/// HealthRegistry allows an arbitrary number of components to +/// be registered and report their health. The process' health +/// status is the combination of these individual health status: +/// - if any component is unhealthy, the process is unhealthy +/// - if all components recently reported healthy, the process is healthy +/// - if a component failed to report healthy for its defined deadline, +/// it is considered unhealthy, and the check fails. +/// +/// Trying to merge the k8s concepts of liveness and readiness in +/// a single state is full of foot-guns, so HealthRegistry does not +/// try to do it. Each probe should have its separate instance of +/// the registry to avoid confusions. + +#[derive(Default, Debug)] +pub struct HealthStatus { + /// The overall status: true of all components are healthy + pub healthy: bool, + /// Current status of each registered component, for display + pub components: HashMap, +} +impl IntoResponse for HealthStatus { + /// Computes the axum status code based on the overall health status, + /// and prints each component status in the body for debugging. + fn into_response(self) -> Response { + let body = format!("{:?}", self); + match self.healthy { + true => (StatusCode::OK, body), + false => (StatusCode::INTERNAL_SERVER_ERROR, body), + } + .into_response() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum ComponentStatus { + /// Automatically set when a component is newly registered + Starting, + /// Recently reported healthy, will need to report again before the date + HealthyUntil(time::OffsetDateTime), + /// Reported unhealthy + Unhealthy, + /// Automatically set when the HealthyUntil deadline is reached + Stalled, +} +struct HealthMessage { + component: String, + status: ComponentStatus, +} + +pub struct HealthHandle { + component: String, + deadline: Duration, + sender: mpsc::Sender, +} + +impl HealthHandle { + /// Asynchronously report healthy, returns when the message is queued. + /// Must be called more frequently than the configured deadline. + pub async fn report_healthy(&self) { + self.report_status(ComponentStatus::HealthyUntil( + time::OffsetDateTime::now_utc().add(self.deadline), + )) + .await + } + + /// Asynchronously report component status, returns when the message is queued. + pub async fn report_status(&self, status: ComponentStatus) { + let message = HealthMessage { + component: self.component.clone(), + status, + }; + if let Err(err) = self.sender.send(message).await { + warn!("failed to report heath status: {}", err) + } + } + + /// Synchronously report as healthy, returns when the message is queued. + /// Must be called more frequently than the configured deadline. + pub fn report_healthy_blocking(&self) { + self.report_status_blocking(ComponentStatus::HealthyUntil( + time::OffsetDateTime::now_utc().add(self.deadline), + )) + } + + /// Asynchronously report component status, returns when the message is queued. + pub fn report_status_blocking(&self, status: ComponentStatus) { + let message = HealthMessage { + component: self.component.clone(), + status, + }; + if let Err(err) = self.sender.blocking_send(message) { + warn!("failed to report heath status: {}", err) + } + } +} + +#[derive(Clone)] +pub struct HealthRegistry { + name: String, + components: Arc>>, + sender: mpsc::Sender, +} + +impl HealthRegistry { + pub fn new(name: &str) -> Self { + let (tx, mut rx) = mpsc::channel::(16); + let registry = Self { + name: name.to_owned(), + components: Default::default(), + sender: tx, + }; + + let components = registry.components.clone(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Ok(mut map) = components.write() { + _ = map.insert(message.component, message.status); + } else { + // Poisoned mutex: Just warn, the probes will fail and the process restart + warn!("poisoned HeathRegistry mutex") + } + } + }); + + registry + } + + /// Registers a new component in the registry. The returned handle should be passed + /// to the component, to allow it to frequently report its health status. + pub async fn register(&self, component: String, deadline: time::Duration) -> HealthHandle { + let handle = HealthHandle { + component, + deadline, + sender: self.sender.clone(), + }; + handle.report_status(ComponentStatus::Starting).await; + handle + } + + /// Returns the overall process status, computed from the status of all the components + /// currently registered. Can be used as an axum handler. + pub fn get_status(&self) -> HealthStatus { + let components = self + .components + .read() + .expect("poisoned HeathRegistry mutex"); + + let result = HealthStatus { + healthy: !components.is_empty(), // unhealthy if no component has registered yet + components: Default::default(), + }; + let now = time::OffsetDateTime::now_utc(); + + let result = components + .iter() + .fold(result, |mut result, (name, status)| { + match status { + ComponentStatus::HealthyUntil(until) => { + if until.gt(&now) { + _ = result.components.insert(name.clone(), status.clone()) + } else { + result.healthy = false; + _ = result + .components + .insert(name.clone(), ComponentStatus::Stalled) + } + } + _ => { + result.healthy = false; + _ = result.components.insert(name.clone(), status.clone()) + } + } + result + }); + match result.healthy { + true => info!("{} health check ok", self.name), + false => warn!("{} health check failed: {:?}", self.name, result.components), + } + result + } +} + +#[cfg(test)] +mod tests { + use crate::health::{ComponentStatus, HealthRegistry, HealthStatus}; + use axum::http::StatusCode; + use axum::response::IntoResponse; + use std::ops::{Add, Sub}; + use time::{Duration, OffsetDateTime}; + + async fn assert_or_retry(check: F) + where + F: Fn() -> bool, + { + assert_or_retry_for_duration(check, Duration::seconds(5)).await + } + + async fn assert_or_retry_for_duration(check: F, timeout: Duration) + where + F: Fn() -> bool, + { + let deadline = OffsetDateTime::now_utc().add(timeout); + while !check() && OffsetDateTime::now_utc().lt(&deadline) { + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert!(check()) + } + #[tokio::test] + async fn defaults_to_unhealthy() { + let registry = HealthRegistry::new("liveness"); + assert!(!registry.get_status().healthy); + } + + #[tokio::test] + async fn one_component() { + let registry = HealthRegistry::new("liveness"); + + // New components are registered in Starting + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + assert_or_retry(|| registry.get_status().components.len() == 1).await; + let mut status = registry.get_status(); + assert!(!status.healthy); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Starting) + ); + + // Status goes healthy once the component reports + handle.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + + // Status goes unhealthy if the components says so + handle.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Unhealthy) + ); + } + + #[tokio::test] + async fn staleness_check() { + let registry = HealthRegistry::new("liveness"); + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + + // Status goes healthy once the component reports + handle.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + let mut status = registry.get_status(); + assert_eq!(status.components.len(), 1); + + // If the component's ping is too old, it is considered stalled and the healthcheck fails + // FIXME: we should mock the time instead + handle + .report_status(ComponentStatus::HealthyUntil( + OffsetDateTime::now_utc().sub(Duration::seconds(1)), + )) + .await; + assert_or_retry(|| !registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Stalled) + ); + } + + #[tokio::test] + async fn several_components() { + let registry = HealthRegistry::new("liveness"); + let handle1 = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + let handle2 = registry + .register("two".to_string(), Duration::seconds(30)) + .await; + assert_or_retry(|| registry.get_status().components.len() == 2).await; + + // First component going healthy is not enough + handle1.report_healthy().await; + assert_or_retry(|| { + registry.get_status().components.get("one").unwrap() != &ComponentStatus::Starting + }) + .await; + assert!(!registry.get_status().healthy); + + // Second component going healthy brings the health to green + handle2.report_healthy().await; + assert_or_retry(|| { + registry.get_status().components.get("two").unwrap() != &ComponentStatus::Starting + }) + .await; + assert!(registry.get_status().healthy); + + // First component going unhealthy takes down the health to red + handle1.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + + // First component recovering returns the health to green + handle1.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + + // Second component going unhealthy takes down the health to red + handle2.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + } + + #[tokio::test] + async fn into_response() { + let nok = HealthStatus::default().into_response(); + assert_eq!(nok.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let ok = HealthStatus { + healthy: true, + components: Default::default(), + } + .into_response(); + assert_eq!(ok.status(), StatusCode::OK); + } +} diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs new file mode 100644 index 0000000..f941f58 --- /dev/null +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -0,0 +1,208 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use uuid::Uuid; + +use super::{deserialize_datetime, serialize_datetime}; + +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub enum AppMetricCategory { + ProcessEvent, + OnEvent, + ScheduledTask, + Webhook, + ComposeWebhook, +} + +// NOTE: These are stored in Postgres and deserialized by the cleanup/janitor process, so these +// names need to remain stable, or new variants need to be deployed to the cleanup/janitor +// process before they are used. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub enum ErrorType { + TimeoutError, + ConnectionError, + BadHttpStatus(u16), + ParseError, +} + +// NOTE: This is stored in Postgres and deserialized by the cleanup/janitor process, so this +// shouldn't change. It is intended to replicate the shape of `error_details` used in the +// plugin-server and by the frontend. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct ErrorDetails { + pub error: Error, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct Error { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, + // This field will only be useful if we start running plugins in Rust (via a WASM runtime or + // something) and want to provide the user with stack traces like we do for TypeScript plugins. + #[serde(skip_serializing_if = "Option::is_none")] + pub stack: Option, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct AppMetric { + #[serde( + serialize_with = "serialize_datetime", + deserialize_with = "deserialize_datetime" + )] + pub timestamp: DateTime, + pub team_id: u32, + pub plugin_config_id: i32, + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + #[serde( + serialize_with = "serialize_category", + deserialize_with = "deserialize_category" + )] + pub category: AppMetricCategory, + pub successes: u32, + pub successes_on_retry: u32, + pub failures: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_uuid: Option, + #[serde( + serialize_with = "serialize_error_type", + deserialize_with = "deserialize_error_type", + default, + skip_serializing_if = "Option::is_none" + )] + pub error_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_details: Option, +} + +fn serialize_category(category: &AppMetricCategory, serializer: S) -> Result +where + S: Serializer, +{ + let category_str = match category { + AppMetricCategory::ProcessEvent => "processEvent", + AppMetricCategory::OnEvent => "onEvent", + AppMetricCategory::ScheduledTask => "scheduledTask", + AppMetricCategory::Webhook => "webhook", + AppMetricCategory::ComposeWebhook => "composeWebhook", + }; + serializer.serialize_str(category_str) +} + +fn deserialize_category<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + + let category = match &s[..] { + "processEvent" => AppMetricCategory::ProcessEvent, + "onEvent" => AppMetricCategory::OnEvent, + "scheduledTask" => AppMetricCategory::ScheduledTask, + "webhook" => AppMetricCategory::Webhook, + "composeWebhook" => AppMetricCategory::ComposeWebhook, + _ => { + return Err(serde::de::Error::unknown_variant( + &s, + &[ + "processEvent", + "onEvent", + "scheduledTask", + "webhook", + "composeWebhook", + ], + )) + } + }; + + Ok(category) +} + +fn serialize_error_type(error_type: &Option, serializer: S) -> Result +where + S: Serializer, +{ + let error_type = match error_type { + Some(error_type) => error_type, + None => return serializer.serialize_none(), + }; + + let error_type = match error_type { + ErrorType::ConnectionError => "Connection Error".to_owned(), + ErrorType::TimeoutError => "Timeout Error".to_owned(), + ErrorType::BadHttpStatus(s) => format!("Bad HTTP Status: {}", s), + ErrorType::ParseError => "Parse Error".to_owned(), + }; + serializer.serialize_str(&error_type) +} + +fn deserialize_error_type<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let opt = Option::::deserialize(deserializer)?; + let error_type = match opt { + Some(s) => { + let error_type = match &s[..] { + "Connection Error" => ErrorType::ConnectionError, + "Timeout Error" => ErrorType::TimeoutError, + _ if s.starts_with("Bad HTTP Status:") => { + let status = &s["Bad HTTP Status:".len()..]; + ErrorType::BadHttpStatus(status.parse().map_err(serde::de::Error::custom)?) + } + "Parse Error" => ErrorType::ParseError, + _ => { + return Err(serde::de::Error::unknown_variant( + &s, + &[ + "Connection Error", + "Timeout Error", + "Bad HTTP Status: ", + "Parse Error", + ], + )) + } + }; + Some(error_type) + } + None => None, + }; + + Ok(error_type) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_app_metric_serialization() { + use chrono::prelude::*; + + let app_metric = AppMetric { + timestamp: Utc.with_ymd_and_hms(2023, 12, 14, 12, 2, 0).unwrap(), + team_id: 123, + plugin_config_id: 456, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 10, + successes_on_retry: 0, + failures: 2, + error_uuid: Some(Uuid::parse_str("550e8400-e29b-41d4-a716-446655447777").unwrap()), + error_type: Some(ErrorType::ConnectionError), + error_details: Some(ErrorDetails { + error: Error { + name: "FooError".to_owned(), + message: Some("Error Message".to_owned()), + stack: None, + }, + }), + }; + + let serialized_json = serde_json::to_string(&app_metric).unwrap(); + + let expected_json = r#"{"timestamp":"2023-12-14 12:02:00","team_id":123,"plugin_config_id":456,"category":"webhook","successes":10,"successes_on_retry":0,"failures":2,"error_uuid":"550e8400-e29b-41d4-a716-446655447777","error_type":"Connection Error","error_details":{"error":{"name":"FooError","message":"Error Message"}}}"#; + + assert_eq!(serialized_json, expected_json); + } +} diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs new file mode 100644 index 0000000..f548563 --- /dev/null +++ b/hook-common/src/kafka_messages/mod.rs @@ -0,0 +1,25 @@ +pub mod app_metrics; +pub mod plugin_logs; + +use chrono::{DateTime, NaiveDateTime, Utc}; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&datetime.format("%Y-%m-%d %H:%M:%S").to_string()) +} + +pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let formatted: String = Deserialize::deserialize(deserializer)?; + let datetime = match NaiveDateTime::parse_from_str(&formatted, "%Y-%m-%d %H:%M:%S") { + Ok(d) => d.and_utc(), + Err(_) => return Err(serde::de::Error::custom("Invalid datetime format")), + }; + + Ok(datetime) +} diff --git a/hook-common/src/kafka_messages/plugin_logs.rs b/hook-common/src/kafka_messages/plugin_logs.rs new file mode 100644 index 0000000..fb83580 --- /dev/null +++ b/hook-common/src/kafka_messages/plugin_logs.rs @@ -0,0 +1,128 @@ +use chrono::{DateTime, Utc}; +use serde::{Serialize, Serializer}; +use uuid::Uuid; + +use super::serialize_datetime; + +#[allow(dead_code)] +#[derive(Serialize)] +pub enum PluginLogEntrySource { + System, + Plugin, + Console, +} + +#[allow(dead_code)] +#[derive(Serialize)] +pub enum PluginLogEntryType { + Debug, + Log, + Info, + Warn, + Error, +} + +#[derive(Serialize)] +pub struct PluginLogEntry { + #[serde(serialize_with = "serialize_source")] + pub source: PluginLogEntrySource, + #[serde(rename = "type", serialize_with = "serialize_type")] + pub type_: PluginLogEntryType, + pub id: Uuid, + pub team_id: u32, + pub plugin_id: i32, + pub plugin_config_id: i32, + #[serde(serialize_with = "serialize_datetime")] + pub timestamp: DateTime, + #[serde(serialize_with = "serialize_message")] + pub message: String, + pub instance_id: Uuid, +} + +fn serialize_source(source: &PluginLogEntrySource, serializer: S) -> Result +where + S: Serializer, +{ + let source_str = match source { + PluginLogEntrySource::System => "SYSTEM", + PluginLogEntrySource::Plugin => "PLUGIN", + PluginLogEntrySource::Console => "CONSOLE", + }; + serializer.serialize_str(source_str) +} + +fn serialize_type(type_: &PluginLogEntryType, serializer: S) -> Result +where + S: Serializer, +{ + let type_str = match type_ { + PluginLogEntryType::Debug => "DEBUG", + PluginLogEntryType::Log => "LOG", + PluginLogEntryType::Info => "INFO", + PluginLogEntryType::Warn => "WARN", + PluginLogEntryType::Error => "ERROR", + }; + serializer.serialize_str(type_str) +} + +fn serialize_message(msg: &String, serializer: S) -> Result +where + S: Serializer, +{ + if msg.len() > 50_000 { + return Err(serde::ser::Error::custom( + "Message is too long for ClickHouse", + )); + } + + serializer.serialize_str(msg) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_plugin_log_entry_serialization() { + use chrono::prelude::*; + + let log_entry = PluginLogEntry { + source: PluginLogEntrySource::Plugin, + type_: PluginLogEntryType::Warn, + id: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(), + team_id: 4, + plugin_id: 5, + plugin_config_id: 6, + timestamp: Utc.with_ymd_and_hms(2023, 12, 14, 12, 2, 0).unwrap(), + message: "My message!".to_string(), + instance_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), + }; + + let serialized_json = serde_json::to_string(&log_entry).unwrap(); + + assert_eq!( + serialized_json, + r#"{"source":"PLUGIN","type":"WARN","id":"550e8400-e29b-41d4-a716-446655440000","team_id":4,"plugin_id":5,"plugin_config_id":6,"timestamp":"2023-12-14 12:02:00","message":"My message!","instance_id":"00000000-0000-0000-0000-000000000000"}"# + ); + } + + #[test] + fn test_plugin_log_entry_message_too_long() { + use chrono::prelude::*; + + let log_entry = PluginLogEntry { + source: PluginLogEntrySource::Plugin, + type_: PluginLogEntryType::Warn, + id: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(), + team_id: 4, + plugin_id: 5, + plugin_config_id: 6, + timestamp: Utc.with_ymd_and_hms(2023, 12, 14, 12, 2, 0).unwrap(), + message: "My message!".repeat(10_000).to_string(), + instance_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), + }; + + let err = serde_json::to_string(&log_entry).unwrap_err(); + assert_eq!(err.to_string(), "Message is too long for ClickHouse"); + } +} diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs new file mode 100644 index 0000000..7f49049 --- /dev/null +++ b/hook-common/src/lib.rs @@ -0,0 +1,6 @@ +pub mod health; +pub mod kafka_messages; +pub mod metrics; +pub mod pgqueue; +pub mod retry; +pub mod webhook; diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs new file mode 100644 index 0000000..66bcfc9 --- /dev/null +++ b/hook-common/src/metrics.rs @@ -0,0 +1,82 @@ +use std::time::{Instant, SystemTime}; + +use axum::{ + body::Body, extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse, + routing::get, Router, +}; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; + +/// Bind a `TcpListener` on the provided bind address to serve a `Router` on it. +/// This function is intended to take a Router as returned by `setup_metrics_router`, potentially with more routes added by the caller. +pub async fn serve(router: Router, bind: &str) -> Result<(), std::io::Error> { + let listener = tokio::net::TcpListener::bind(bind).await?; + + axum::serve(listener, router).await?; + + Ok(()) +} + +/// Add the prometheus endpoint and middleware to a router, should be called last. +pub fn setup_metrics_routes(router: Router) -> Router { + let recorder_handle = setup_metrics_recorder(); + + router + .route( + "/metrics", + get(move || std::future::ready(recorder_handle.render())), + ) + .layer(axum::middleware::from_fn(track_metrics)) +} + +pub fn setup_metrics_recorder() -> PrometheusHandle { + const EXPONENTIAL_SECONDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + PrometheusBuilder::new() + .set_buckets(EXPONENTIAL_SECONDS) + .unwrap() + .install_recorder() + .unwrap() +} + +/// Middleware to record some common HTTP metrics +/// Someday tower-http might provide a metrics middleware: https://github.com/tower-rs/tower-http/issues/57 +pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { + let start = Instant::now(); + + let path = if let Some(matched_path) = req.extensions().get::() { + matched_path.as_str().to_owned() + } else { + req.uri().path().to_owned() + }; + + let method = req.method().clone(); + + // Run the rest of the request handling first, so we can measure it and get response + // codes. + let response = next.run(req).await; + + let latency = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + + let labels = [ + ("method", method.to_string()), + ("path", path), + ("status", status), + ]; + + metrics::counter!("http_requests_total", &labels).increment(1); + metrics::histogram!("http_requests_duration_seconds", &labels).record(latency); + + response +} + +/// Returns the number of seconds since the Unix epoch, to use in prom gauges. +/// Saturates to zero if the system time is set before epoch. +pub fn get_current_timestamp_seconds() -> f64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64 +} diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs new file mode 100644 index 0000000..4a8b489 --- /dev/null +++ b/hook-common/src/pgqueue.rs @@ -0,0 +1,957 @@ +//! # PgQueue +//! +//! A job queue implementation backed by a PostgreSQL table. +use std::time; +use std::{str::FromStr, sync::Arc}; + +use async_trait::async_trait; +use chrono; +use serde; +use sqlx::postgres::any::AnyConnectionBackend; +use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions}; +use thiserror::Error; +use tokio::sync::Mutex; +use tracing::error; + +/// Enumeration of errors for operations with PgQueue. +/// Errors that can originate from sqlx and are wrapped by us to provide additional context. +#[derive(Error, Debug)] +pub enum PgQueueError { + #[error("pool creation failed with: {error}")] + PoolCreationError { error: sqlx::Error }, + #[error("connection failed with: {error}")] + ConnectionError { error: sqlx::Error }, + #[error("{command} query failed with: {error}")] + QueryError { command: String, error: sqlx::Error }, + #[error("{0} is not a valid JobStatus")] + ParseJobStatusError(String), + #[error("{0} is not a valid HttpMethod")] + ParseHttpMethodError(String), + #[error("transaction was already closed")] + TransactionAlreadyClosedError, +} + +#[derive(Error, Debug)] +pub enum PgJobError { + #[error("retry is an invalid state for this PgJob: {error}")] + RetryInvalidError { job: T, error: String }, + #[error("connection failed with: {error}")] + ConnectionError { error: sqlx::Error }, + #[error("{command} query failed with: {error}")] + QueryError { command: String, error: sqlx::Error }, + #[error("transaction {command} failed with: {error}")] + TransactionError { command: String, error: sqlx::Error }, + #[error("transaction was already closed")] + TransactionAlreadyClosedError, +} + +/// Enumeration of possible statuses for a Job. +#[derive(Debug, PartialEq, sqlx::Type)] +#[sqlx(type_name = "job_status")] +#[sqlx(rename_all = "lowercase")] +pub enum JobStatus { + /// A job that is waiting in the queue to be picked up by a worker. + Available, + /// A job that was cancelled by a worker. + Cancelled, + /// A job that was successfully completed by a worker. + Completed, + /// A job that has + Discarded, + /// A job that was unsuccessfully completed by a worker. + Failed, +} + +/// Allow casting JobStatus from strings. +impl FromStr for JobStatus { + type Err = PgQueueError; + + fn from_str(s: &str) -> Result { + match s { + "available" => Ok(JobStatus::Available), + "completed" => Ok(JobStatus::Completed), + "failed" => Ok(JobStatus::Failed), + invalid => Err(PgQueueError::ParseJobStatusError(invalid.to_owned())), + } + } +} + +/// JobParameters are stored and read to and from a JSONB field, so we accept anything that fits `sqlx::types::Json`. +pub type JobParameters = sqlx::types::Json; + +/// JobMetadata are stored and read to and from a JSONB field, so we accept anything that fits `sqlx::types::Json`. +pub type JobMetadata = sqlx::types::Json; + +/// A Job to be executed by a worker dequeueing a PgQueue. +#[derive(sqlx::FromRow, Debug)] +pub struct Job { + /// A unique id identifying a job. + pub id: i64, + /// A number corresponding to the current job attempt. + pub attempt: i32, + /// A datetime corresponding to when the job was attempted. + pub attempted_at: chrono::DateTime, + /// A vector of identifiers that have attempted this job. E.g. thread ids, pod names, etc... + pub attempted_by: Vec, + /// A datetime corresponding to when the job was created. + pub created_at: chrono::DateTime, + /// The current job's number of max attempts. + pub max_attempts: i32, + /// Arbitrary job metadata stored as JSON. + pub metadata: JobMetadata, + /// Arbitrary job parameters stored as JSON. + pub parameters: JobParameters, + /// The queue this job belongs to. + pub queue: String, + /// The current status of the job. + pub status: JobStatus, + /// The target of the job. E.g. an endpoint or service we are trying to reach. + pub target: String, +} + +impl Job { + /// Return true if this job attempt is greater or equal to the maximum number of possible attempts. + pub fn is_gte_max_attempts(&self) -> bool { + self.attempt >= self.max_attempts + } + + /// Consume `Job` to transition it to a `RetryableJob`, i.e. a `Job` that may be retried. + fn retryable(self) -> RetryableJob { + RetryableJob { + id: self.id, + attempt: self.attempt, + queue: self.queue, + retry_queue: None, + } + } + + /// Consume `Job` to complete it. + /// A `CompletedJob` is finalized and cannot be used further; it is returned for reporting or inspection. + /// + /// # Arguments + /// + /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as completed. + async fn complete<'c, E>(self, executor: E) -> Result + where + E: sqlx::Executor<'c, Database = sqlx::Postgres>, + { + let base_query = r#" +UPDATE + job_queue +SET + last_attempt_finished_at = NOW(), + status = 'completed'::job_status +WHERE + queue = $1 + AND id = $2 +RETURNING + job_queue.* + "#; + + sqlx::query(base_query) + .bind(&self.queue) + .bind(self.id) + .execute(executor) + .await?; + + Ok(CompletedJob { + id: self.id, + queue: self.queue, + }) + } + + /// Consume `Job` to fail it. + /// A `FailedJob` is finalized and cannot be used further; it is returned for reporting or inspection. + /// + /// # Arguments + /// + /// * `error`: Any JSON-serializable value to be stored as an error. + /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as failed. + async fn fail<'c, E, S>(self, error: S, executor: E) -> Result, sqlx::Error> + where + S: serde::Serialize + std::marker::Sync + std::marker::Send, + E: sqlx::Executor<'c, Database = sqlx::Postgres>, + { + let json_error = sqlx::types::Json(error); + let base_query = r#" +UPDATE + job_queue +SET + last_attempt_finished_at = NOW(), + status = 'failed'::job_status, + errors = array_append(errors, $3) +WHERE + queue = $1 + AND id = $2 +RETURNING + job_queue.* + "#; + + sqlx::query(base_query) + .bind(&self.queue) + .bind(self.id) + .bind(&json_error) + .execute(executor) + .await?; + + Ok(FailedJob { + id: self.id, + error: json_error, + queue: self.queue, + }) + } +} + +#[async_trait] +pub trait PgQueueJob { + async fn complete(mut self) -> Result>>; + + async fn fail( + mut self, + error: E, + ) -> Result, PgJobError>>; + + async fn retry( + mut self, + error: E, + retry_interval: time::Duration, + queue: &str, + ) -> Result>>; +} + +/// A Job within an open PostgreSQL transaction. +/// This implementation allows 'hiding' the job from any other workers running SKIP LOCKED queries. +#[derive(Debug)] +pub struct PgTransactionJob<'c, J, M> { + pub job: Job, + + /// The open transaction this job came from. If multiple jobs were queried at once, then this + /// transaction will be shared between them (across async tasks and threads as necessary). See + /// below for more information. + shared_txn: Arc>>>, +} + +// Container struct for a batch of PgTransactionJob. Includes a reference to the shared transaction +// for committing the work when all of the jobs are finished. +pub struct PgTransactionBatch<'c, J, M> { + pub jobs: Vec>, + + /// The open transaction the jobs in the Vec came from. This should be used to commit or + /// rollback when all of the work is finished. + shared_txn: Arc>>>, +} + +impl<'c, J, M> PgTransactionBatch<'_, J, M> { + pub async fn commit(self) -> PgQueueResult<()> { + let mut txn_guard = self.shared_txn.lock().await; + + txn_guard + .as_deref_mut() + .ok_or(PgQueueError::TransactionAlreadyClosedError)? + .commit() + .await + .map_err(|e| PgQueueError::QueryError { + command: "COMMIT".to_owned(), + error: e, + })?; + + Ok(()) + } +} + +#[async_trait] +impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactionJob<'c, J, M> { + async fn complete( + mut self, + ) -> Result>>> { + let mut txn_guard = self.shared_txn.lock().await; + + let txn_ref = txn_guard + .as_deref_mut() + .ok_or(PgJobError::TransactionAlreadyClosedError)?; + + let completed_job = + self.job + .complete(txn_ref) + .await + .map_err(|error| PgJobError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(completed_job) + } + + async fn fail( + mut self, + error: S, + ) -> Result, PgJobError>>> { + let mut txn_guard = self.shared_txn.lock().await; + + let txn_ref = txn_guard + .as_deref_mut() + .ok_or(PgJobError::TransactionAlreadyClosedError)?; + + let failed_job = + self.job + .fail(error, txn_ref) + .await + .map_err(|error| PgJobError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(failed_job) + } + + async fn retry( + mut self, + error: E, + retry_interval: time::Duration, + queue: &str, + ) -> Result>>> { + // Ideally, the transition to RetryableJob should be fallible. + // But taking ownership of self when we return this error makes things difficult. + if self.job.is_gte_max_attempts() { + return Err(PgJobError::RetryInvalidError { + job: Box::new(self), + error: "Maximum attempts reached".to_owned(), + }); + } + + let mut txn_guard = self.shared_txn.lock().await; + + let txn_ref = txn_guard + .as_deref_mut() + .ok_or(PgJobError::TransactionAlreadyClosedError)?; + + let retried_job = self + .job + .retryable() + .queue(queue) + .retry(error, retry_interval, txn_ref) + .await + .map_err(|error| PgJobError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(retried_job) + } +} + +/// A Job that has failed but can still be enqueued into a PgQueue to be retried at a later point. +/// The time until retry will depend on the PgQueue's RetryPolicy. +pub struct RetryableJob { + /// A unique id identifying a job. + pub id: i64, + /// A number corresponding to the current job attempt. + pub attempt: i32, + /// A unique id identifying a job queue. + queue: String, + /// An optional separate queue where to enqueue this job when retrying. + retry_queue: Option, +} + +impl RetryableJob { + /// Set the queue for a `RetryableJob`. + /// If not set, `Job` will be retried to its original queue on calling `retry`. + fn queue(mut self, queue: &str) -> Self { + self.retry_queue = Some(queue.to_owned()); + self + } + + /// Return the queue that a `Job` is to be retried into. + fn retry_queue(&self) -> &str { + self.retry_queue.as_ref().unwrap_or(&self.queue) + } + + /// Consume `Job` to retry it. + /// A `RetriedJob` cannot be used further; it is returned for reporting or inspection. + /// + /// # Arguments + /// + /// * `error`: Any JSON-serializable value to be stored as an error. + /// * `retry_interval`: The duration until the `Job` is to be retried again. Used to set `scheduled_at`. + /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as completed. + async fn retry<'c, S, E>( + self, + error: S, + retry_interval: time::Duration, + executor: E, + ) -> Result + where + S: serde::Serialize + std::marker::Sync + std::marker::Send, + E: sqlx::Executor<'c, Database = sqlx::Postgres>, + { + let json_error = sqlx::types::Json(error); + let base_query = r#" +UPDATE + job_queue +SET + last_attempt_finished_at = NOW(), + status = 'available'::job_status, + scheduled_at = NOW() + $3, + errors = array_append(errors, $4), + queue = $5 +WHERE + queue = $1 + AND id = $2 +RETURNING + job_queue.* + "#; + + sqlx::query(base_query) + .bind(&self.queue) + .bind(self.id) + .bind(retry_interval) + .bind(&json_error) + .bind(self.retry_queue()) + .execute(executor) + .await?; + + Ok(RetriedJob { + id: self.id, + queue: self.queue, + retry_queue: self.retry_queue.to_owned(), + }) + } +} + +/// State a `Job` is transitioned to after successfully completing. +#[derive(Debug)] +pub struct CompletedJob { + /// A unique id identifying a job. + pub id: i64, + /// A unique id identifying a job queue. + pub queue: String, +} + +/// State a `Job` is transitioned to after it has been enqueued for retrying. +#[derive(Debug)] +pub struct RetriedJob { + /// A unique id identifying a job. + pub id: i64, + /// A unique id identifying a job queue. + pub queue: String, + pub retry_queue: Option, +} + +/// State a `Job` is transitioned to after exhausting all of their attempts. +#[derive(Debug)] +pub struct FailedJob { + /// A unique id identifying a job. + pub id: i64, + /// Any JSON-serializable value to be stored as an error. + pub error: sqlx::types::Json, + /// A unique id identifying a job queue. + pub queue: String, +} + +/// This struct represents a new job being created to be enqueued into a `PgQueue`. +#[derive(Debug)] +pub struct NewJob { + /// The maximum amount of attempts this NewJob has to complete. + pub max_attempts: i32, + /// The JSON-deserializable parameters for this NewJob. + pub metadata: JobMetadata, + /// The JSON-deserializable parameters for this NewJob. + pub parameters: JobParameters, + /// The target of the NewJob. E.g. an endpoint or service we are trying to reach. + pub target: String, +} + +impl NewJob { + pub fn new(max_attempts: i32, metadata: M, parameters: J, target: &str) -> Self { + Self { + max_attempts, + metadata: sqlx::types::Json(metadata), + parameters: sqlx::types::Json(parameters), + target: target.to_owned(), + } + } +} + +/// A queue implemented on top of a PostgreSQL table. +#[derive(Clone)] +pub struct PgQueue { + /// A name to identify this PgQueue as multiple may share a table. + name: String, + /// A connection pool used to connect to the PostgreSQL database. + pool: PgPool, +} + +pub type PgQueueResult = std::result::Result; + +impl PgQueue { + /// Initialize a new PgQueue backed by table in PostgreSQL by intializing a connection pool to the database in `url`. + /// + /// # Arguments + /// + /// * `queue_name`: A name for the queue we are going to initialize. + /// * `url`: A URL pointing to where the PostgreSQL database is hosted. + pub async fn new( + queue_name: &str, + url: &str, + max_connections: u32, + app_name: &'static str, + ) -> PgQueueResult { + let name = queue_name.to_owned(); + let options = PgConnectOptions::from_str(url) + .map_err(|error| PgQueueError::PoolCreationError { error })? + .application_name(app_name); + let pool = PgPoolOptions::new() + .max_connections(max_connections) + .connect_lazy_with(options); + + Ok(Self { name, pool }) + } + + /// Initialize a new PgQueue backed by table in PostgreSQL from a provided connection pool. + /// + /// # Arguments + /// + /// * `queue_name`: A name for the queue we are going to initialize. + /// * `pool`: A database connection pool to be used by this queue. + pub async fn new_from_pool(queue_name: &str, pool: PgPool) -> PgQueueResult { + let name = queue_name.to_owned(); + + Ok(Self { name, pool }) + } + + /// Dequeue up to `limit` `Job`s from this `PgQueue` and hold the transaction. Any other + /// `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one + /// worker can dequeue a job. Holding a transaction open can have performance implications, but + /// it means no `'running'` state is required. + pub async fn dequeue_tx< + 'a, + J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, + M: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, + >( + &self, + attempted_by: &str, + limit: u32, + ) -> PgQueueResult>> { + let mut tx = self + .pool + .begin() + .await + .map_err(|error| PgQueueError::ConnectionError { error })?; + + // The query that follows uses a FOR UPDATE SKIP LOCKED clause. + // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. + let base_query = r#" +WITH available_in_queue AS ( + SELECT + id + FROM + job_queue + WHERE + status = 'available' + AND scheduled_at <= NOW() + AND queue = $1 + ORDER BY + attempt, + scheduled_at + LIMIT $2 + FOR UPDATE SKIP LOCKED +) +UPDATE + job_queue +SET + attempted_at = NOW(), + attempt = attempt + 1, + attempted_by = array_append(attempted_by, $3::text) +FROM + available_in_queue +WHERE + job_queue.id = available_in_queue.id +RETURNING + job_queue.* + "#; + + let query_result: Result>, sqlx::Error> = sqlx::query_as(base_query) + .bind(&self.name) + .bind(limit as i64) + .bind(attempted_by) + .fetch_all(&mut *tx) + .await; + + match query_result { + Ok(jobs) => { + if jobs.is_empty() { + return Ok(None); + } + + let shared_txn = Arc::new(Mutex::new(Some(tx))); + + let pg_jobs: Vec> = jobs + .into_iter() + .map(|job| PgTransactionJob { + job, + shared_txn: shared_txn.clone(), + }) + .collect(); + + Ok(Some(PgTransactionBatch { + jobs: pg_jobs, + shared_txn: shared_txn.clone(), + })) + } + + // Transaction is rolled back on drop. + Err(sqlx::Error::RowNotFound) => Ok(None), + + Err(e) => Err(PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error: e, + }), + } + } + + /// Enqueue a `NewJob` into this PgQueue. + /// We take ownership of `NewJob` to enforce a specific `NewJob` is only enqueued once. + pub async fn enqueue< + J: serde::Serialize + std::marker::Sync, + M: serde::Serialize + std::marker::Sync, + >( + &self, + job: NewJob, + ) -> PgQueueResult<()> { + // TODO: Escaping. I think sqlx doesn't support identifiers. + let base_query = r#" +INSERT INTO job_queue + (attempt, created_at, scheduled_at, max_attempts, metadata, parameters, queue, status, target) +VALUES + (0, NOW(), NOW(), $1, $2, $3, $4, 'available'::job_status, $5) + "#; + + sqlx::query(base_query) + .bind(job.max_attempts) + .bind(&job.metadata) + .bind(&job.parameters) + .bind(&self.name) + .bind(&job.target) + .execute(&self.pool) + .await + .map_err(|error| PgQueueError::QueryError { + command: "INSERT".to_owned(), + error, + })?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::retry::RetryPolicy; + + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug, Clone)] + struct JobMetadata { + team_id: u32, + plugin_config_id: i32, + plugin_id: i32, + } + + impl Default for JobMetadata { + fn default() -> Self { + Self { + team_id: 0, + plugin_config_id: 1, + plugin_id: 2, + } + } + } + + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug, Clone)] + struct JobParameters { + method: String, + body: String, + url: String, + } + + impl Default for JobParameters { + fn default() -> Self { + Self { + method: "POST".to_string(), + body: "{\"event\":\"event-name\"}".to_string(), + url: "https://localhost".to_string(), + } + } + } + + /// Use process id as a worker id for tests. + fn worker_id() -> String { + std::process::id().to_string() + } + + /// Hardcoded test value for job target. + fn job_target() -> String { + "https://myhost/endpoint".to_owned() + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_can_dequeue_tx_job(db: PgPool) { + let job_target = job_target(); + let job_metadata = JobMetadata::default(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + + let queue = PgQueue::new_from_pool("test_can_dequeue_tx_job", db) + .await + .expect("failed to connect to local test postgresql database"); + + let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); + queue.enqueue(new_job).await.expect("failed to enqueue job"); + + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue jobs") + .expect("didn't find any jobs to dequeue"); + + let tx_job = batch.jobs.pop().unwrap(); + + assert_eq!(tx_job.job.attempt, 1); + assert!(tx_job.job.attempted_by.contains(&worker_id)); + assert_eq!(tx_job.job.attempted_by.len(), 1); + assert_eq!(tx_job.job.max_attempts, 1); + assert_eq!(*tx_job.job.metadata.as_ref(), JobMetadata::default()); + assert_eq!(*tx_job.job.parameters.as_ref(), JobParameters::default()); + assert_eq!(tx_job.job.target, job_target); + + // Transactional jobs must be completed, failed or retried before being dropped. This is + // to prevent logic bugs when using the shared txn. + tx_job.complete().await.expect("failed to complete job"); + + batch.commit().await.expect("failed to commit transaction"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_can_dequeue_multiple_tx_jobs(db: PgPool) { + let job_target = job_target(); + let job_metadata = JobMetadata::default(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + + let queue = PgQueue::new_from_pool("test_can_dequeue_multiple_tx_jobs", db) + .await + .expect("failed to connect to local test postgresql database"); + + for _ in 0..5 { + queue + .enqueue(NewJob::new( + 1, + job_metadata.clone(), + job_parameters.clone(), + &job_target, + )) + .await + .expect("failed to enqueue job"); + } + + // Only get 4 jobs, leaving one in the queue. + let limit = 4; + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, limit) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + + assert_eq!(batch.jobs.len(), limit as usize); + + // Complete those 4 and commit. + for job in std::mem::take(&mut batch.jobs) { + job.complete().await.expect("failed to complete job"); + } + batch.commit().await.expect("failed to commit transaction"); + + // Try to get up to 4 jobs, but only 1 remains. + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, limit) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + assert_eq!(batch.jobs.len(), 1); // Only one job should have been left in the queue. + + for job in std::mem::take(&mut batch.jobs) { + job.complete().await.expect("failed to complete job"); + } + batch.commit().await.expect("failed to commit transaction"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_dequeue_tx_returns_none_on_no_jobs(db: PgPool) { + let worker_id = worker_id(); + let queue = PgQueue::new_from_pool("test_dequeue_tx_returns_none_on_no_jobs", db) + .await + .expect("failed to connect to local test postgresql database"); + + let batch: Option> = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job"); + + assert!(batch.is_none()); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_can_retry_job_with_remaining_attempts(db: PgPool) { + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); + let queue_name = "test_can_retry_job_with_remaining_attempts".to_owned(); + + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) + .queue(&queue_name) + .provide(); + + let queue = PgQueue::new_from_pool(&queue_name, db) + .await + .expect("failed to connect to local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + let job = batch.jobs.pop().unwrap(); + + let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); + let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); + let _ = job + .retry( + "a very reasonable failure reason", + retry_interval, + &retry_queue, + ) + .await + .expect("failed to retry job"); + batch.commit().await.expect("failed to commit transaction"); + + let retried_job: PgTransactionJob = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job") + .expect("didn't find retried job to dequeue") + .jobs + .pop() + .unwrap(); + + assert_eq!(retried_job.job.attempt, 2); + assert!(retried_job.job.attempted_by.contains(&worker_id)); + assert_eq!(retried_job.job.attempted_by.len(), 2); + assert_eq!(retried_job.job.max_attempts, 2); + assert_eq!( + *retried_job.job.parameters.as_ref(), + JobParameters::default() + ); + assert_eq!(retried_job.job.target, job_target); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_can_retry_job_to_different_queue(db: PgPool) { + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); + let queue_name = "test_can_retry_job_to_different_queue".to_owned(); + let retry_queue_name = "test_can_retry_job_to_different_queue_retry".to_owned(); + + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) + .queue(&retry_queue_name) + .provide(); + + let queue = PgQueue::new_from_pool(&queue_name, db.clone()) + .await + .expect("failed to connect to queue in local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let mut batch: PgTransactionBatch = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + let job = batch.jobs.pop().unwrap(); + + let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); + let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); + let _ = job + .retry( + "a very reasonable failure reason", + retry_interval, + &retry_queue, + ) + .await + .expect("failed to retry job"); + batch.commit().await.expect("failed to commit transaction"); + + let retried_job_not_found: Option> = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job"); + + assert!(retried_job_not_found.is_none()); + + let queue = PgQueue::new_from_pool(&retry_queue_name, db) + .await + .expect("failed to connect to retry queue in local test postgresql database"); + + let retried_job: PgTransactionJob = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job") + .expect("job not found in retry queue") + .jobs + .pop() + .unwrap(); + + assert_eq!(retried_job.job.attempt, 2); + assert!(retried_job.job.attempted_by.contains(&worker_id)); + assert_eq!(retried_job.job.attempted_by.len(), 2); + assert_eq!(retried_job.job.max_attempts, 2); + assert_eq!( + *retried_job.job.parameters.as_ref(), + JobParameters::default() + ); + assert_eq!(retried_job.job.target, job_target); + } + + #[sqlx::test(migrations = "../migrations")] + #[should_panic(expected = "failed to retry job")] + async fn test_cannot_retry_job_without_remaining_attempts(db: PgPool) { + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)).provide(); + + let queue = PgQueue::new_from_pool("test_cannot_retry_job_without_remaining_attempts", db) + .await + .expect("failed to connect to local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + + let job: PgTransactionJob = queue + .dequeue_tx(&worker_id, 1) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue") + .jobs + .pop() + .unwrap(); + + let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); + + job.retry("a very reasonable failure reason", retry_interval, "any") + .await + .expect("failed to retry job"); + } +} diff --git a/hook-common/src/retry.rs b/hook-common/src/retry.rs new file mode 100644 index 0000000..b00f967 --- /dev/null +++ b/hook-common/src/retry.rs @@ -0,0 +1,225 @@ +//! # Retry +//! +//! Module providing a `RetryPolicy` struct to configure job retrying. +use std::time; + +#[derive(Clone, Debug)] +/// A retry policy to determine retry parameters for a job. +pub struct RetryPolicy { + /// Coefficient to multiply initial_interval with for every past attempt. + pub backoff_coefficient: u32, + /// The backoff interval for the first retry. + pub initial_interval: time::Duration, + /// The maximum possible backoff between retries. + pub maximum_interval: Option, + /// An optional queue to send WebhookJob retries to. + pub queue: Option, +} + +impl RetryPolicy { + /// Initialize a `RetryPolicyBuilder`. + pub fn build(backoff_coefficient: u32, initial_interval: time::Duration) -> RetryPolicyBuilder { + RetryPolicyBuilder::new(backoff_coefficient, initial_interval) + } + + /// Determine interval for retrying at a given attempt number. + /// If not `None`, this method will respect `preferred_retry_interval` as long as it falls within `candidate_interval <= preferred_retry_interval <= maximum_interval`. + pub fn retry_interval( + &self, + attempt: u32, + preferred_retry_interval: Option, + ) -> time::Duration { + let candidate_interval = + self.initial_interval * self.backoff_coefficient.pow(attempt.saturating_sub(1)); + + match (preferred_retry_interval, self.maximum_interval) { + (Some(duration), Some(max_interval)) => { + let min_interval_allowed = std::cmp::min(candidate_interval, max_interval); + + if min_interval_allowed <= duration && duration <= max_interval { + duration + } else { + min_interval_allowed + } + } + (Some(duration), None) => std::cmp::max(candidate_interval, duration), + (None, Some(max_interval)) => std::cmp::min(candidate_interval, max_interval), + (None, None) => candidate_interval, + } + } + + /// Determine the queue to be used for retrying. + /// Only whether a queue is configured in this RetryPolicy is used to determine which queue to use for retrying. + /// This may be extended in the future to support more decision parameters. + pub fn retry_queue<'s>(&'s self, current_queue: &'s str) -> &'s str { + if let Some(new_queue) = &self.queue { + new_queue + } else { + current_queue + } + } +} + +impl Default for RetryPolicy { + fn default() -> Self { + RetryPolicyBuilder::default().provide() + } +} + +/// Builder pattern struct to provide a `RetryPolicy`. +pub struct RetryPolicyBuilder { + /// Coefficient to multiply initial_interval with for every past attempt. + pub backoff_coefficient: u32, + /// The backoff interval for the first retry. + pub initial_interval: time::Duration, + /// The maximum possible backoff between retries. + pub maximum_interval: Option, + /// An optional queue to send WebhookJob retries to. + pub queue: Option, +} + +impl Default for RetryPolicyBuilder { + fn default() -> Self { + Self { + backoff_coefficient: 2, + initial_interval: time::Duration::from_secs(1), + maximum_interval: None, + queue: None, + } + } +} + +impl RetryPolicyBuilder { + pub fn new(backoff_coefficient: u32, initial_interval: time::Duration) -> Self { + Self { + backoff_coefficient, + initial_interval, + ..RetryPolicyBuilder::default() + } + } + + pub fn maximum_interval(mut self, interval: time::Duration) -> RetryPolicyBuilder { + self.maximum_interval = Some(interval); + self + } + + pub fn queue(mut self, queue: &str) -> RetryPolicyBuilder { + self.queue = Some(queue.to_owned()); + self + } + + /// Provide a `RetryPolicy` according to build parameters provided thus far. + pub fn provide(&self) -> RetryPolicy { + RetryPolicy { + backoff_coefficient: self.backoff_coefficient, + initial_interval: self.initial_interval, + maximum_interval: self.maximum_interval, + queue: self.queue.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constant_retry_interval() { + let retry_policy = RetryPolicy::build(1, time::Duration::from_secs(2)).provide(); + let first_interval = retry_policy.retry_interval(1, None); + let second_interval = retry_policy.retry_interval(2, None); + let third_interval = retry_policy.retry_interval(3, None); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(2)); + assert_eq!(third_interval, time::Duration::from_secs(2)); + } + + #[test] + fn test_retry_interval_never_exceeds_maximum() { + let retry_policy = RetryPolicy::build(2, time::Duration::from_secs(2)) + .maximum_interval(time::Duration::from_secs(4)) + .provide(); + let first_interval = retry_policy.retry_interval(1, None); + let second_interval = retry_policy.retry_interval(2, None); + let third_interval = retry_policy.retry_interval(3, None); + let fourth_interval = retry_policy.retry_interval(4, None); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(4)); + assert_eq!(third_interval, time::Duration::from_secs(4)); + assert_eq!(fourth_interval, time::Duration::from_secs(4)); + } + + #[test] + fn test_retry_interval_increases_with_coefficient() { + let retry_policy = RetryPolicy::build(2, time::Duration::from_secs(2)).provide(); + let first_interval = retry_policy.retry_interval(1, None); + let second_interval = retry_policy.retry_interval(2, None); + let third_interval = retry_policy.retry_interval(3, None); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(4)); + assert_eq!(third_interval, time::Duration::from_secs(8)); + } + + #[test] + fn test_retry_interval_respects_preferred() { + let retry_policy = RetryPolicy::build(1, time::Duration::from_secs(2)).provide(); + let preferred = time::Duration::from_secs(999); + let first_interval = retry_policy.retry_interval(1, Some(preferred)); + let second_interval = retry_policy.retry_interval(2, Some(preferred)); + let third_interval = retry_policy.retry_interval(3, Some(preferred)); + + assert_eq!(first_interval, preferred); + assert_eq!(second_interval, preferred); + assert_eq!(third_interval, preferred); + } + + #[test] + fn test_retry_interval_ignores_small_preferred() { + let retry_policy = RetryPolicy::build(1, time::Duration::from_secs(5)).provide(); + let preferred = time::Duration::from_secs(2); + let first_interval = retry_policy.retry_interval(1, Some(preferred)); + let second_interval = retry_policy.retry_interval(2, Some(preferred)); + let third_interval = retry_policy.retry_interval(3, Some(preferred)); + + assert_eq!(first_interval, time::Duration::from_secs(5)); + assert_eq!(second_interval, time::Duration::from_secs(5)); + assert_eq!(third_interval, time::Duration::from_secs(5)); + } + + #[test] + fn test_retry_interval_ignores_large_preferred() { + let retry_policy = RetryPolicy::build(2, time::Duration::from_secs(2)) + .maximum_interval(time::Duration::from_secs(4)) + .provide(); + let preferred = time::Duration::from_secs(10); + let first_interval = retry_policy.retry_interval(1, Some(preferred)); + let second_interval = retry_policy.retry_interval(2, Some(preferred)); + let third_interval = retry_policy.retry_interval(3, Some(preferred)); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(4)); + assert_eq!(third_interval, time::Duration::from_secs(4)); + } + + #[test] + fn test_returns_retry_queue_if_set() { + let retry_queue_name = "retry_queue".to_owned(); + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) + .queue(&retry_queue_name) + .provide(); + let current_queue = "queue".to_owned(); + + assert_eq!(retry_policy.retry_queue(¤t_queue), retry_queue_name); + } + + #[test] + fn test_returns_queue_if_retry_queue_not_set() { + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)).provide(); + let current_queue = "queue".to_owned(); + + assert_eq!(retry_policy.retry_queue(¤t_queue), current_queue); + } +} diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs new file mode 100644 index 0000000..11e0285 --- /dev/null +++ b/hook-common/src/webhook.rs @@ -0,0 +1,225 @@ +use std::collections; +use std::convert::From; +use std::fmt; +use std::str::FromStr; + +use serde::{de::Visitor, Deserialize, Serialize}; + +use crate::kafka_messages::app_metrics; +use crate::pgqueue::PgQueueError; + +/// Supported HTTP methods for webhooks. +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum HttpMethod { + DELETE, + GET, + PATCH, + POST, + PUT, +} + +/// Allow casting `HttpMethod` from strings. +impl FromStr for HttpMethod { + type Err = PgQueueError; + + fn from_str(s: &str) -> Result { + match s.to_ascii_uppercase().as_ref() { + "DELETE" => Ok(HttpMethod::DELETE), + "GET" => Ok(HttpMethod::GET), + "PATCH" => Ok(HttpMethod::PATCH), + "POST" => Ok(HttpMethod::POST), + "PUT" => Ok(HttpMethod::PUT), + invalid => Err(PgQueueError::ParseHttpMethodError(invalid.to_owned())), + } + } +} + +/// Implement `std::fmt::Display` to convert HttpMethod to string. +impl fmt::Display for HttpMethod { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + HttpMethod::DELETE => write!(f, "DELETE"), + HttpMethod::GET => write!(f, "GET"), + HttpMethod::PATCH => write!(f, "PATCH"), + HttpMethod::POST => write!(f, "POST"), + HttpMethod::PUT => write!(f, "PUT"), + } + } +} + +struct HttpMethodVisitor; + +impl<'de> Visitor<'de> for HttpMethodVisitor { + type Value = HttpMethod; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "the string representation of HttpMethod") + } + + fn visit_str(self, s: &str) -> Result + where + E: serde::de::Error, + { + match HttpMethod::from_str(s) { + Ok(method) => Ok(method), + Err(_) => Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Str(s), + &self, + )), + } + } +} + +/// Deserialize required to read `HttpMethod` from database. +impl<'de> Deserialize<'de> for HttpMethod { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(HttpMethodVisitor) + } +} + +/// Serialize required to write `HttpMethod` to database. +impl Serialize for HttpMethod { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +/// Convenience to cast `HttpMethod` to `http::Method`. +/// Not all `http::Method` variants are valid `HttpMethod` variants, hence why we +/// can't just use the former or implement `From`. +impl From for http::Method { + fn from(val: HttpMethod) -> Self { + match val { + HttpMethod::DELETE => http::Method::DELETE, + HttpMethod::GET => http::Method::GET, + HttpMethod::PATCH => http::Method::PATCH, + HttpMethod::POST => http::Method::POST, + HttpMethod::PUT => http::Method::PUT, + } + } +} + +impl From<&HttpMethod> for http::Method { + fn from(val: &HttpMethod) -> Self { + match val { + HttpMethod::DELETE => http::Method::DELETE, + HttpMethod::GET => http::Method::GET, + HttpMethod::PATCH => http::Method::PATCH, + HttpMethod::POST => http::Method::POST, + HttpMethod::PUT => http::Method::PUT, + } + } +} + +/// `JobParameters` required for the `WebhookWorker` to execute a webhook. +/// These parameters should match the exported Webhook interface that PostHog plugins. +/// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookJobParameters { + pub body: String, + pub headers: collections::HashMap, + pub method: HttpMethod, + pub url: String, +} + +/// `JobMetadata` required for the `WebhookWorker` to execute a webhook. +/// These should be set if the Webhook is associated with a plugin `composeWebhook` invocation. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookJobMetadata { + pub team_id: u32, + pub plugin_id: i32, + pub plugin_config_id: i32, +} + +/// An error originating during a Webhook Job invocation. +/// This is to be serialized to be stored as an error whenever retrying or failing a webhook job. +#[derive(Deserialize, Serialize, Debug)] +pub struct WebhookJobError { + pub r#type: app_metrics::ErrorType, + pub details: app_metrics::ErrorDetails, +} + +/// Webhook jobs boil down to an HTTP request, so it's useful to have a way to convert from &reqwest::Error. +/// For the convertion we check all possible error types with the associated is_* methods provided by reqwest. +/// Some precision may be lost as our app_metrics::ErrorType does not support the same number of variants. +impl From<&reqwest::Error> for WebhookJobError { + fn from(error: &reqwest::Error) -> Self { + if error.is_timeout() { + WebhookJobError::new_timeout(&error.to_string()) + } else if error.is_status() { + WebhookJobError::new_http_status( + error.status().expect("status code is defined").into(), + &error.to_string(), + ) + } else { + // Catch all other errors as `app_metrics::ErrorType::Connection` errors. + // Not all of `reqwest::Error` may strictly be connection errors, so our supported error types may need an extension + // depending on how strict error reporting has to be. + WebhookJobError::new_connection(&error.to_string()) + } + } +} + +impl WebhookJobError { + pub fn new_timeout(message: &str) -> Self { + let error_details = app_metrics::Error { + name: "Timeout Error".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::TimeoutError, + details: app_metrics::ErrorDetails { + error: error_details, + }, + } + } + + pub fn new_connection(message: &str) -> Self { + let error_details = app_metrics::Error { + name: "Connection Error".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::ConnectionError, + details: app_metrics::ErrorDetails { + error: error_details, + }, + } + } + + pub fn new_http_status(status_code: u16, message: &str) -> Self { + let error_details = app_metrics::Error { + name: "Bad Http Status".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::BadHttpStatus(status_code), + details: app_metrics::ErrorDetails { + error: error_details, + }, + } + } + + pub fn new_parse(message: &str) -> Self { + let error_details = app_metrics::Error { + name: "Parse Error".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::ParseError, + details: app_metrics::ErrorDetails { + error: error_details, + }, + } + } +} diff --git a/hook-janitor/Cargo.toml b/hook-janitor/Cargo.toml new file mode 100644 index 0000000..96a80eb --- /dev/null +++ b/hook-janitor/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "hook-janitor" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = { workspace = true } +axum = { workspace = true } +envconfig = { workspace = true } +eyre = { workspace = true } +futures = { workspace = true } +hook-common = { path = "../hook-common" } +http-body-util = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } +rdkafka = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sqlx = { workspace = true } +time = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tower = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } diff --git a/hook-janitor/src/cleanup.rs b/hook-janitor/src/cleanup.rs new file mode 100644 index 0000000..82b9130 --- /dev/null +++ b/hook-janitor/src/cleanup.rs @@ -0,0 +1,34 @@ +use async_trait::async_trait; +use std::result::Result; +use std::str::FromStr; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CleanerError { + #[error("invalid cleaner mode")] + InvalidCleanerMode, +} + +// Mode names, used by config/environment parsing to verify the mode is supported. +#[derive(Debug)] +pub enum CleanerModeName { + Webhooks, +} + +impl FromStr for CleanerModeName { + type Err = CleanerError; + + fn from_str(s: &str) -> Result { + match s { + "webhooks" => Ok(CleanerModeName::Webhooks), + _ => Err(CleanerError::InvalidCleanerMode), + } + } +} + +// Right now, all this trait does is allow us to call `cleanup` in a loop in `main.rs`. There may +// be other benefits as we build this out, or we could remove it if it doesn't end up being useful. +#[async_trait] +pub trait Cleaner { + async fn cleanup(&self); +} diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs new file mode 100644 index 0000000..389de03 --- /dev/null +++ b/hook-janitor/src/config.rs @@ -0,0 +1,57 @@ +use envconfig::Envconfig; + +#[derive(Envconfig)] +pub struct Config { + #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] + pub host: String, + + #[envconfig(from = "BIND_PORT", default = "3302")] + pub port: u16, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub database_url: String, + + #[envconfig(default = "30")] + pub cleanup_interval_secs: u64, + + // The cleanup task needs to have special knowledge of the queue it's cleaning up. This is so it + // can do things like flush the proper app_metrics or plugin_log_entries, and so it knows what + // to expect in the job's payload JSONB column. + #[envconfig(default = "webhooks")] + pub mode: String, + + #[envconfig(nested = true)] + pub kafka: KafkaConfig, +} + +#[derive(Envconfig, Clone)] +pub struct KafkaConfig { + #[envconfig(default = "20")] + pub kafka_producer_linger_ms: u32, // Maximum time between producer batches during low traffic + + #[envconfig(default = "400")] + pub kafka_producer_queue_mib: u32, // Size of the in-memory producer queue in mebibytes + + #[envconfig(default = "20000")] + pub kafka_message_timeout_ms: u32, // Time before we stop retrying producing a message: 20 seconds + + #[envconfig(default = "none")] + pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd + + #[envconfig(default = "false")] + pub kafka_tls: bool, + + #[envconfig(default = "clickhouse_app_metrics")] + pub app_metrics_topic: String, + + #[envconfig(default = "plugin_log_entries")] + pub plugin_log_entries_topic: String, + + pub kafka_hosts: String, +} + +impl Config { + pub fn bind(&self) -> String { + format!("{}:{}", self.host, self.port) + } +} diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql new file mode 100644 index 0000000..e0b9a7a --- /dev/null +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -0,0 +1,166 @@ +INSERT INTO + job_queue ( + errors, + metadata, + attempted_at, + last_attempt_finished_at, + parameters, + queue, + status, + target + ) +VALUES + -- team:1, plugin_config:2, completed in hour 20 + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, completed in hour 20 (purposeful duplicate) + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, completed in hour 21 (different hour) + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 21:01:18.799371+00', + '2023-12-19 21:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:3, completed in hour 20 (different plugin_config) + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', + '2023-12-19 20:01:18.80335+00', + '2023-12-19 20:01:18.80335+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, completed but in a different queue + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'not-webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:2, plugin_config:4, completed in hour 20 (different team) + ( + NULL, + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 20 + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 20 (purposeful duplicate) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 20 (different error) + ( + ARRAY ['{"type":"ConnectionError","details":{"error":{"name":"Connection Error"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 21 (different hour) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 21:01:18.799371+00', + '2023-12-19 21:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:3, failed in hour 20 (different plugin_config) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed but in a different queue + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'not-webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:2, plugin_config:4, failed in hour 20 (purposeful duplicate) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, available + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', + '{"body": "hello world", "headers": {}, "method": "POST", "url": "https://myhost/endpoint"}', + 'webhooks', + 'available', + 'https://myhost/endpoint' + ); \ No newline at end of file diff --git a/hook-janitor/src/handlers/app.rs b/hook-janitor/src/handlers/app.rs new file mode 100644 index 0000000..507a1cb --- /dev/null +++ b/hook-janitor/src/handlers/app.rs @@ -0,0 +1,14 @@ +use axum::{routing::get, Router}; +use hook_common::health::HealthRegistry; +use std::future::ready; + +pub fn app(liveness: HealthRegistry) -> Router { + Router::new() + .route("/", get(index)) + .route("/_readiness", get(index)) + .route("/_liveness", get(move || ready(liveness.get_status()))) +} + +pub async fn index() -> &'static str { + "rusty-hook janitor" +} diff --git a/hook-janitor/src/handlers/mod.rs b/hook-janitor/src/handlers/mod.rs new file mode 100644 index 0000000..a884c04 --- /dev/null +++ b/hook-janitor/src/handlers/mod.rs @@ -0,0 +1,3 @@ +mod app; + +pub use app::app; diff --git a/hook-janitor/src/kafka_producer.rs b/hook-janitor/src/kafka_producer.rs new file mode 100644 index 0000000..ba36866 --- /dev/null +++ b/hook-janitor/src/kafka_producer.rs @@ -0,0 +1,57 @@ +use crate::config::KafkaConfig; + +use hook_common::health::HealthHandle; +use rdkafka::error::KafkaError; +use rdkafka::producer::FutureProducer; +use rdkafka::ClientConfig; +use tracing::debug; + +pub struct KafkaContext { + liveness: HealthHandle, +} + +impl rdkafka::ClientContext for KafkaContext { + fn stats(&self, _: rdkafka::Statistics) { + // Signal liveness, as the main rdkafka loop is running and calling us + self.liveness.report_healthy_blocking(); + + // TODO: Take stats recording pieces that we want from `capture-rs`. + } +} + +pub async fn create_kafka_producer( + config: &KafkaConfig, + liveness: HealthHandle, +) -> Result, KafkaError> { + let mut client_config = ClientConfig::new(); + client_config + .set("bootstrap.servers", &config.kafka_hosts) + .set("statistics.interval.ms", "10000") + .set("linger.ms", config.kafka_producer_linger_ms.to_string()) + .set( + "message.timeout.ms", + config.kafka_message_timeout_ms.to_string(), + ) + .set( + "compression.codec", + config.kafka_compression_codec.to_owned(), + ) + .set( + "queue.buffering.max.kbytes", + (config.kafka_producer_queue_mib * 1024).to_string(), + ); + + if config.kafka_tls { + client_config + .set("security.protocol", "ssl") + .set("enable.ssl.certificate.verification", "false"); + }; + + debug!("rdkafka configuration: {:?}", client_config); + let api: FutureProducer = + client_config.create_with_context(KafkaContext { liveness })?; + + // TODO: ping the kafka brokers to confirm configuration is OK (copy capture) + + Ok(api) +} diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs new file mode 100644 index 0000000..46ee375 --- /dev/null +++ b/hook-janitor/src/main.rs @@ -0,0 +1,97 @@ +use axum::Router; +use cleanup::{Cleaner, CleanerModeName}; +use config::Config; +use envconfig::Envconfig; +use eyre::Result; +use futures::future::{select, Either}; +use hook_common::health::{HealthHandle, HealthRegistry}; +use kafka_producer::create_kafka_producer; +use std::{str::FromStr, time::Duration}; +use tokio::sync::Semaphore; +use webhooks::WebhookCleaner; + +use hook_common::metrics::setup_metrics_routes; + +mod cleanup; +mod config; +mod handlers; +mod kafka_producer; +mod webhooks; + +async fn listen(app: Router, bind: String) -> Result<()> { + let listener = tokio::net::TcpListener::bind(bind).await?; + + axum::serve(listener, app).await?; + + Ok(()) +} + +async fn cleanup_loop(cleaner: Box, interval_secs: u64, liveness: HealthHandle) { + let semaphore = Semaphore::new(1); + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + + loop { + let _permit = semaphore.acquire().await; + interval.tick().await; + liveness.report_healthy().await; + cleaner.cleanup().await; + drop(_permit); + } +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let config = Config::init_from_env().expect("failed to load configuration from env"); + + let mode_name = CleanerModeName::from_str(&config.mode) + .unwrap_or_else(|_| panic!("invalid cleaner mode: {}", config.mode)); + + let liveness = HealthRegistry::new("liveness"); + + let cleaner = match mode_name { + CleanerModeName::Webhooks => { + let kafka_liveness = liveness + .register("rdkafka".to_string(), time::Duration::seconds(30)) + .await; + let kafka_producer = create_kafka_producer(&config.kafka, kafka_liveness) + .await + .expect("failed to create kafka producer"); + + Box::new( + WebhookCleaner::new( + &config.database_url, + kafka_producer, + config.kafka.app_metrics_topic.to_owned(), + ) + .expect("unable to create webhook cleaner"), + ) + } + }; + + let cleanup_liveness = liveness + .register( + "cleanup_loop".to_string(), + time::Duration::seconds(config.cleanup_interval_secs as i64 * 2), + ) + .await; + let cleanup_loop = Box::pin(cleanup_loop( + cleaner, + config.cleanup_interval_secs, + cleanup_liveness, + )); + + let app = setup_metrics_routes(handlers::app(liveness)); + let http_server = Box::pin(listen(app, config.bind())); + + match select(http_server, cleanup_loop).await { + Either::Left((listen_result, _)) => match listen_result { + Ok(_) => {} + Err(e) => tracing::error!("failed to start hook-janitor http server, {}", e), + }, + Either::Right((_, _)) => { + tracing::error!("hook-janitor cleanup task exited") + } + }; +} diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs new file mode 100644 index 0000000..7f7fadd --- /dev/null +++ b/hook-janitor/src/webhooks.rs @@ -0,0 +1,901 @@ +use std::str::FromStr; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use futures::future::join_all; +use hook_common::webhook::WebhookJobError; +use rdkafka::error::KafkaError; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use serde_json::error::Error as SerdeError; +use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions, Postgres}; +use sqlx::types::{chrono, Uuid}; +use sqlx::{Row, Transaction}; +use thiserror::Error; +use tracing::{debug, error, info}; + +use crate::cleanup::Cleaner; +use crate::kafka_producer::KafkaContext; + +use hook_common::kafka_messages::app_metrics::{AppMetric, AppMetricCategory}; +use hook_common::metrics::get_current_timestamp_seconds; + +#[derive(Error, Debug)] +pub enum WebhookCleanerError { + #[error("failed to create postgres pool: {error}")] + PoolCreationError { error: sqlx::Error }, + #[error("failed to acquire conn: {error}")] + AcquireConnError { error: sqlx::Error }, + #[error("failed to acquire conn and start txn: {error}")] + StartTxnError { error: sqlx::Error }, + #[error("failed to get queue depth: {error}")] + GetQueueDepthError { error: sqlx::Error }, + #[error("failed to get row count: {error}")] + GetRowCountError { error: sqlx::Error }, + #[error("failed to get completed rows: {error}")] + GetCompletedRowsError { error: sqlx::Error }, + #[error("failed to get failed rows: {error}")] + GetFailedRowsError { error: sqlx::Error }, + #[error("failed to serialize rows: {error}")] + SerializeRowsError { error: SerdeError }, + #[error("failed to produce to kafka: {error}")] + KafkaProduceError { error: KafkaError }, + #[error("failed to produce to kafka (timeout)")] + KafkaProduceCanceled, + #[error("failed to delete rows: {error}")] + DeleteRowsError { error: sqlx::Error }, + #[error("attempted to delete a different number of rows than expected")] + DeleteConsistencyError, + #[error("failed to rollback txn: {error}")] + RollbackTxnError { error: sqlx::Error }, + #[error("failed to commit txn: {error}")] + CommitTxnError { error: sqlx::Error }, +} + +type Result = std::result::Result; + +pub struct WebhookCleaner { + pg_pool: PgPool, + kafka_producer: FutureProducer, + app_metrics_topic: String, +} + +#[derive(sqlx::FromRow, Debug)] +struct CompletedRow { + // App Metrics truncates/aggregates rows on the hour, so we take advantage of that to GROUP BY + // and aggregate to select fewer rows. + hour: DateTime, + // A note about the `try_from`s: Postgres returns all of those types as `bigint` (i64), but + // we know their true sizes, and so we can convert them to the correct types here. If this + // ever fails then something has gone wrong. + #[sqlx(try_from = "i64")] + team_id: u32, + #[sqlx(try_from = "i64")] + plugin_config_id: i32, + #[sqlx(try_from = "i64")] + successes: u32, +} + +impl From for AppMetric { + fn from(row: CompletedRow) -> Self { + AppMetric { + timestamp: row.hour, + team_id: row.team_id, + plugin_config_id: row.plugin_config_id, + job_id: None, + category: AppMetricCategory::Webhook, + successes: row.successes, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + } + } +} + +#[derive(sqlx::FromRow, Debug)] +struct FailedRow { + // App Metrics truncates/aggregates rows on the hour, so we take advantage of that to GROUP BY + // and aggregate to select fewer rows. + hour: DateTime, + // A note about the `try_from`s: Postgres returns all of those types as `bigint` (i64), but + // we know their true sizes, and so we can convert them to the correct types here. If this + // ever fails then something has gone wrong. + #[sqlx(try_from = "i64")] + team_id: u32, + #[sqlx(try_from = "i64")] + plugin_config_id: i32, + #[sqlx(json)] + last_error: WebhookJobError, + #[sqlx(try_from = "i64")] + failures: u32, +} + +#[derive(sqlx::FromRow, Debug)] +struct QueueDepth { + oldest_scheduled_at_untried: DateTime, + count_untried: i64, + oldest_scheduled_at_retries: DateTime, + count_retries: i64, +} + +impl From for AppMetric { + fn from(row: FailedRow) -> Self { + AppMetric { + timestamp: row.hour, + team_id: row.team_id, + plugin_config_id: row.plugin_config_id, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: row.failures, + error_uuid: Some(Uuid::now_v7()), + error_type: Some(row.last_error.r#type), + error_details: Some(row.last_error.details), + } + } +} + +// A simple wrapper type that ensures we don't use any old Transaction object when we need one +// that has set the isolation level to serializable. +struct SerializableTxn<'a>(Transaction<'a, Postgres>); + +struct CleanupStats { + rows_processed: u64, + completed_row_count: u64, + completed_agg_row_count: u64, + failed_row_count: u64, + failed_agg_row_count: u64, +} + +impl WebhookCleaner { + pub fn new( + database_url: &str, + kafka_producer: FutureProducer, + app_metrics_topic: String, + ) -> Result { + let options = PgConnectOptions::from_str(database_url) + .map_err(|error| WebhookCleanerError::PoolCreationError { error })? + .application_name("hook-janitor"); + let pg_pool = PgPoolOptions::new() + .acquire_timeout(Duration::from_secs(10)) + .connect_lazy_with(options); + + Ok(Self { + pg_pool, + kafka_producer, + app_metrics_topic, + }) + } + + #[allow(dead_code)] // This is used in tests. + pub fn new_from_pool( + pg_pool: PgPool, + kafka_producer: FutureProducer, + app_metrics_topic: String, + ) -> Result { + Ok(Self { + pg_pool, + kafka_producer, + app_metrics_topic, + }) + } + + async fn get_queue_depth(&self) -> Result { + let mut conn = self + .pg_pool + .acquire() + .await + .map_err(|e| WebhookCleanerError::AcquireConnError { error: e })?; + + let base_query = r#" + SELECT + COALESCE(MIN(CASE WHEN attempt = 0 THEN scheduled_at END), now()) AS oldest_scheduled_at_untried, + COALESCE(SUM(CASE WHEN attempt = 0 THEN 1 ELSE 0 END), 0) AS count_untried, + COALESCE(MIN(CASE WHEN attempt > 0 THEN scheduled_at END), now()) AS oldest_scheduled_at_retries, + COALESCE(SUM(CASE WHEN attempt > 0 THEN 1 ELSE 0 END), 0) AS count_retries + FROM job_queue + WHERE status = 'available'; + "#; + + let row = sqlx::query_as::<_, QueueDepth>(base_query) + .fetch_one(&mut *conn) + .await + .map_err(|e| WebhookCleanerError::GetQueueDepthError { error: e })?; + + Ok(row) + } + + async fn start_serializable_txn(&self) -> Result { + let mut tx = self + .pg_pool + .begin() + .await + .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; + + // We use serializable isolation so that we observe a snapshot of the DB at the time we + // start the cleanup process. This prevents us from accidentally deleting rows that are + // added (or become 'completed' or 'failed') after we start the cleanup process. + // + // If we find that this has a significant performance impact, we could instead move + // rows to a temporary table for processing and then deletion. + sqlx::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + .execute(&mut *tx) + .await + .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; + + Ok(SerializableTxn(tx)) + } + + async fn get_row_count_for_status( + &self, + tx: &mut SerializableTxn<'_>, + status: &str, + ) -> Result { + let base_query = r#" + SELECT count(*) FROM job_queue + WHERE status = $1::job_status; + "#; + + let count: i64 = sqlx::query(base_query) + .bind(status) + .fetch_one(&mut *tx.0) + .await + .map_err(|e| WebhookCleanerError::GetRowCountError { error: e })? + .get(0); + + Ok(count as u64) + } + + async fn get_completed_agg_rows( + &self, + tx: &mut SerializableTxn<'_>, + ) -> Result> { + let base_query = r#" + SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, + (metadata->>'team_id')::bigint AS team_id, + (metadata->>'plugin_config_id')::bigint AS plugin_config_id, + count(*) as successes + FROM job_queue + WHERE status = 'completed' + GROUP BY hour, team_id, plugin_config_id + ORDER BY hour, team_id, plugin_config_id; + "#; + + let rows = sqlx::query_as::<_, CompletedRow>(base_query) + .fetch_all(&mut *tx.0) + .await + .map_err(|e| WebhookCleanerError::GetCompletedRowsError { error: e })?; + + Ok(rows) + } + + async fn get_failed_agg_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { + let base_query = r#" + SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, + (metadata->>'team_id')::bigint AS team_id, + (metadata->>'plugin_config_id')::bigint AS plugin_config_id, + errors[array_upper(errors, 1)] AS last_error, + count(*) as failures + FROM job_queue + WHERE status = 'failed' + GROUP BY hour, team_id, plugin_config_id, last_error + ORDER BY hour, team_id, plugin_config_id, last_error; + "#; + + let rows = sqlx::query_as::<_, FailedRow>(base_query) + .fetch_all(&mut *tx.0) + .await + .map_err(|e| WebhookCleanerError::GetFailedRowsError { error: e })?; + + Ok(rows) + } + + async fn send_metrics_to_kafka(&self, metrics: Vec) -> Result<()> { + if metrics.is_empty() { + return Ok(()); + } + + let payloads: Vec = metrics + .into_iter() + .map(|metric| serde_json::to_string(&metric)) + .collect::, SerdeError>>() + .map_err(|e| WebhookCleanerError::SerializeRowsError { error: e })?; + + let mut delivery_futures = Vec::new(); + + for payload in payloads { + match self.kafka_producer.send_result(FutureRecord { + topic: self.app_metrics_topic.as_str(), + payload: Some(&payload), + partition: None, + key: None::<&str>, + timestamp: None, + headers: None, + }) { + Ok(future) => delivery_futures.push(future), + Err((error, _)) => return Err(WebhookCleanerError::KafkaProduceError { error }), + } + } + + for result in join_all(delivery_futures).await { + match result { + Ok(Ok(_)) => {} + Ok(Err((error, _))) => { + return Err(WebhookCleanerError::KafkaProduceError { error }) + } + Err(_) => { + // Cancelled due to timeout while retrying + return Err(WebhookCleanerError::KafkaProduceCanceled); + } + } + } + + Ok(()) + } + + async fn delete_observed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result { + // This DELETE is only safe because we are in serializable isolation mode, see the note + // in `start_serializable_txn`. + let base_query = r#" + DELETE FROM job_queue + WHERE status IN ('failed', 'completed') + "#; + + let result = sqlx::query(base_query) + .execute(&mut *tx.0) + .await + .map_err(|e| WebhookCleanerError::DeleteRowsError { error: e })?; + + Ok(result.rows_affected()) + } + + async fn rollback_txn(&self, tx: SerializableTxn<'_>) -> Result<()> { + tx.0.rollback() + .await + .map_err(|e| WebhookCleanerError::RollbackTxnError { error: e })?; + + Ok(()) + } + + async fn commit_txn(&self, tx: SerializableTxn<'_>) -> Result<()> { + tx.0.commit() + .await + .map_err(|e| WebhookCleanerError::CommitTxnError { error: e })?; + + Ok(()) + } + + async fn cleanup_impl(&self) -> Result { + debug!("WebhookCleaner starting cleanup"); + + // Note that we select all completed and failed rows without any pagination at the moment. + // We aggregrate as much as possible with GROUP BY, truncating the timestamp down to the + // hour just like App Metrics does. A completed row is 24 bytes (and aggregates an entire + // hour per `plugin_config_id`), and a failed row is 104 bytes + the error message length + // (and aggregates an entire hour per `plugin_config_id` per `error`), so we can fit a lot + // of rows in memory. It seems unlikely we'll need to paginate, but that can be added in the + // future if necessary. + + let untried_status = [("status", "untried")]; + let retries_status = [("status", "retries")]; + + let queue_depth = self.get_queue_depth().await?; + metrics::gauge!("queue_depth_oldest_scheduled", &untried_status) + .set(queue_depth.oldest_scheduled_at_untried.timestamp() as f64); + metrics::gauge!("queue_depth", &untried_status).set(queue_depth.count_untried as f64); + metrics::gauge!("queue_depth_oldest_scheduled", &retries_status) + .set(queue_depth.oldest_scheduled_at_retries.timestamp() as f64); + metrics::gauge!("queue_depth", &retries_status).set(queue_depth.count_retries as f64); + + let mut tx = self.start_serializable_txn().await?; + + let (completed_row_count, completed_agg_row_count) = { + let completed_row_count = self.get_row_count_for_status(&mut tx, "completed").await?; + let completed_agg_rows = self.get_completed_agg_rows(&mut tx).await?; + let agg_row_count = completed_agg_rows.len() as u64; + let completed_app_metrics: Vec = + completed_agg_rows.into_iter().map(Into::into).collect(); + self.send_metrics_to_kafka(completed_app_metrics).await?; + (completed_row_count, agg_row_count) + }; + + let (failed_row_count, failed_agg_row_count) = { + let failed_row_count = self.get_row_count_for_status(&mut tx, "failed").await?; + let failed_agg_rows = self.get_failed_agg_rows(&mut tx).await?; + let agg_row_count = failed_agg_rows.len() as u64; + let failed_app_metrics: Vec = + failed_agg_rows.into_iter().map(Into::into).collect(); + self.send_metrics_to_kafka(failed_app_metrics).await?; + (failed_row_count, agg_row_count) + }; + + let mut rows_deleted = 0; + if completed_agg_row_count + failed_agg_row_count != 0 { + rows_deleted = self.delete_observed_rows(&mut tx).await?; + + if rows_deleted != completed_row_count + failed_row_count { + // This should never happen, but if it does, we want to know about it (and abort the + // txn). + error!( + attempted_rows_deleted = rows_deleted, + completed_row_count = completed_row_count, + failed_row_count = failed_row_count, + "WebhookCleaner::cleanup attempted to delete a different number of rows than expected" + ); + + self.rollback_txn(tx).await?; + + return Err(WebhookCleanerError::DeleteConsistencyError); + } + + self.commit_txn(tx).await?; + } + + Ok(CleanupStats { + rows_processed: rows_deleted, + completed_row_count, + completed_agg_row_count, + failed_row_count, + failed_agg_row_count, + }) + } +} + +#[async_trait] +impl Cleaner for WebhookCleaner { + async fn cleanup(&self) { + let start_time = Instant::now(); + metrics::counter!("webhook_cleanup_attempts",).increment(1); + + match self.cleanup_impl().await { + Ok(stats) => { + metrics::counter!("webhook_cleanup_success",).increment(1); + metrics::gauge!("webhook_cleanup_last_success_timestamp",) + .set(get_current_timestamp_seconds()); + + if stats.rows_processed > 0 { + let elapsed_time = start_time.elapsed().as_secs_f64(); + metrics::histogram!("webhook_cleanup_duration").record(elapsed_time); + metrics::counter!("webhook_cleanup_rows_processed",) + .increment(stats.rows_processed); + metrics::counter!("webhook_cleanup_completed_row_count",) + .increment(stats.completed_row_count); + metrics::counter!("webhook_cleanup_completed_agg_row_count",) + .increment(stats.completed_agg_row_count); + metrics::counter!("webhook_cleanup_failed_row_count",) + .increment(stats.failed_row_count); + metrics::counter!("webhook_cleanup_failed_agg_row_count",) + .increment(stats.failed_agg_row_count); + + info!( + rows_processed = stats.rows_processed, + completed_row_count = stats.completed_row_count, + completed_agg_row_count = stats.completed_agg_row_count, + failed_row_count = stats.failed_row_count, + failed_agg_row_count = stats.failed_agg_row_count, + "WebhookCleaner::cleanup finished" + ); + } else { + debug!("WebhookCleaner finished cleanup, there were no rows to process"); + } + } + Err(error) => { + metrics::counter!("webhook_cleanup_failures",).increment(1); + error!(error = ?error, "WebhookCleaner::cleanup failed"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config; + use crate::kafka_producer::{create_kafka_producer, KafkaContext}; + use hook_common::health::HealthRegistry; + use hook_common::kafka_messages::app_metrics::{ + Error as WebhookError, ErrorDetails, ErrorType, + }; + use hook_common::pgqueue::PgQueueJob; + use hook_common::pgqueue::{NewJob, PgQueue, PgTransactionBatch}; + use hook_common::webhook::{HttpMethod, WebhookJobMetadata, WebhookJobParameters}; + use rdkafka::consumer::{Consumer, StreamConsumer}; + use rdkafka::mocking::MockCluster; + use rdkafka::producer::{DefaultProducerContext, FutureProducer}; + use rdkafka::types::{RDKafkaApiKey, RDKafkaRespErr}; + use rdkafka::{ClientConfig, Message}; + use sqlx::{PgPool, Row}; + use std::collections::HashMap; + use std::str::FromStr; + + const APP_METRICS_TOPIC: &str = "app_metrics"; + + async fn create_mock_kafka() -> ( + MockCluster<'static, DefaultProducerContext>, + FutureProducer, + ) { + let registry = HealthRegistry::new("liveness"); + let handle = registry + .register("one".to_string(), time::Duration::seconds(30)) + .await; + let cluster = MockCluster::new(1).expect("failed to create mock brokers"); + + let config = config::KafkaConfig { + kafka_producer_linger_ms: 0, + kafka_producer_queue_mib: 50, + kafka_message_timeout_ms: 5000, + kafka_compression_codec: "none".to_string(), + kafka_hosts: cluster.bootstrap_servers(), + app_metrics_topic: APP_METRICS_TOPIC.to_string(), + plugin_log_entries_topic: "plugin_log_entries".to_string(), + kafka_tls: false, + }; + + ( + cluster, + create_kafka_producer(&config, handle) + .await + .expect("failed to create mocked kafka producer"), + ) + } + + fn check_app_metric_vector_equality(v1: &[AppMetric], v2: &[AppMetric]) { + // Ignores `error_uuid`s. + assert_eq!(v1.len(), v2.len()); + for (item1, item2) in v1.iter().zip(v2) { + let mut item1 = item1.clone(); + item1.error_uuid = None; + let mut item2 = item2.clone(); + item2.error_uuid = None; + assert_eq!(item1, item2); + } + } + + #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] + async fn test_cleanup_impl(db: PgPool) { + let (mock_cluster, mock_producer) = create_mock_kafka().await; + mock_cluster + .create_topic(APP_METRICS_TOPIC, 1, 1) + .expect("failed to create mock app_metrics topic"); + + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", mock_cluster.bootstrap_servers()) + .set("group.id", "mock") + .set("auto.offset.reset", "earliest") + .create() + .expect("failed to create mock consumer"); + consumer.subscribe(&[APP_METRICS_TOPIC]).unwrap(); + + let webhook_cleaner = + WebhookCleaner::new_from_pool(db, mock_producer, APP_METRICS_TOPIC.to_owned()) + .expect("unable to create webhook cleaner"); + + let cleanup_stats = webhook_cleaner + .cleanup_impl() + .await + .expect("webbook cleanup_impl failed"); + + // Rows that are not 'completed' or 'failed' should not be processed. + assert_eq!(cleanup_stats.rows_processed, 13); + + let mut received_app_metrics = Vec::new(); + for _ in 0..(cleanup_stats.completed_agg_row_count + cleanup_stats.failed_agg_row_count) { + let kafka_msg = consumer.recv().await.unwrap(); + let payload_str = String::from_utf8(kafka_msg.payload().unwrap().to_vec()).unwrap(); + let app_metric: AppMetric = serde_json::from_str(&payload_str).unwrap(); + received_app_metrics.push(app_metric); + } + + let expected_app_metrics = vec![ + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 3, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 3, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 1, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 2, + plugin_config_id: 4, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 1, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T21:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 1, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::ConnectionError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Connection Error".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 3, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 3, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 2, + plugin_config_id: 4, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T21:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + ]; + + check_app_metric_vector_equality(&expected_app_metrics, &received_app_metrics); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_cleanup_impl_empty_queue(db: PgPool) { + let (mock_cluster, mock_producer) = create_mock_kafka().await; + mock_cluster + .create_topic(APP_METRICS_TOPIC, 1, 1) + .expect("failed to create mock app_metrics topic"); + + // No payload should be produced to kafka as the queue is empty. + // Set a non-retriable produce error that would bubble-up when cleanup_impl is called. + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE; 1]; + mock_cluster.request_errors(RDKafkaApiKey::Produce, &err); + + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", mock_cluster.bootstrap_servers()) + .set("group.id", "mock") + .set("auto.offset.reset", "earliest") + .create() + .expect("failed to create mock consumer"); + consumer.subscribe(&[APP_METRICS_TOPIC]).unwrap(); + + let webhook_cleaner = + WebhookCleaner::new_from_pool(db, mock_producer, APP_METRICS_TOPIC.to_owned()) + .expect("unable to create webhook cleaner"); + + let cleanup_stats = webhook_cleaner + .cleanup_impl() + .await + .expect("webbook cleanup_impl failed"); + + // Reported metrics are all zeroes + assert_eq!(cleanup_stats.rows_processed, 0); + assert_eq!(cleanup_stats.completed_row_count, 0); + assert_eq!(cleanup_stats.completed_agg_row_count, 0); + assert_eq!(cleanup_stats.failed_row_count, 0); + assert_eq!(cleanup_stats.failed_agg_row_count, 0); + } + + #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] + async fn test_serializable_isolation(db: PgPool) { + let (_, mock_producer) = create_mock_kafka().await; + let webhook_cleaner = + WebhookCleaner::new_from_pool(db.clone(), mock_producer, APP_METRICS_TOPIC.to_owned()) + .expect("unable to create webhook cleaner"); + + let queue = PgQueue::new_from_pool("webhooks", db.clone()) + .await + .expect("failed to connect to local test postgresql database"); + + async fn get_count_from_new_conn(db: &PgPool, status: &str) -> i64 { + let mut conn = db.acquire().await.unwrap(); + let count: i64 = + sqlx::query("SELECT count(*) FROM job_queue WHERE status = $1::job_status") + .bind(&status) + .fetch_one(&mut *conn) + .await + .unwrap() + .get(0); + count + } + + // Important! Serializable txn is started here. + let mut tx = webhook_cleaner.start_serializable_txn().await.unwrap(); + webhook_cleaner + .get_completed_agg_rows(&mut tx) + .await + .unwrap(); + webhook_cleaner.get_failed_agg_rows(&mut tx).await.unwrap(); + + // All 15 rows in the DB are visible from outside the txn. + // The 13 the cleaner will process, plus 1 available and 1 running. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 6); + assert_eq!(get_count_from_new_conn(&db, "failed").await, 7); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + + { + // The fixtures include an available job, so let's complete it while the txn is open. + let mut batch: PgTransactionBatch<'_, WebhookJobParameters, WebhookJobMetadata> = queue + .dequeue_tx(&"worker_id", 1) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + let webhook_job = batch.jobs.pop().unwrap(); + webhook_job + .complete() + .await + .expect("failed to complete job"); + batch.commit().await.expect("failed to commit batch"); + } + + { + // Enqueue and complete another job while the txn is open. + let job_parameters = WebhookJobParameters { + body: "foo".to_owned(), + headers: HashMap::new(), + method: HttpMethod::POST, + url: "http://example.com".to_owned(), + }; + let job_metadata = WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }; + let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let mut batch: PgTransactionBatch<'_, WebhookJobParameters, WebhookJobMetadata> = queue + .dequeue_tx(&"worker_id", 1) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + let webhook_job = batch.jobs.pop().unwrap(); + webhook_job + .complete() + .await + .expect("failed to complete job"); + batch.commit().await.expect("failed to commit batch"); + } + + { + // Enqueue another available job while the txn is open. + let job_parameters = WebhookJobParameters { + body: "foo".to_owned(), + headers: HashMap::new(), + method: HttpMethod::POST, + url: "http://example.com".to_owned(), + }; + let job_metadata = WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }; + let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); + queue.enqueue(new_job).await.expect("failed to enqueue job"); + } + + // There are now 2 more completed rows (jobs added above) than before, visible from outside the txn. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 8); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + + let rows_processed = webhook_cleaner.delete_observed_rows(&mut tx).await.unwrap(); + // The 13 rows in the DB when the txn started should be deleted. + assert_eq!(rows_processed, 13); + + // We haven't committed, so the rows are still visible from outside the txn. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 8); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + + webhook_cleaner.commit_txn(tx).await.unwrap(); + + // We have committed, what remains are: + // * The 1 available job we completed while the txn was open. + // * The 2 brand new jobs we added while the txn was open. + // * The 1 running job that didn't change. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 2); + assert_eq!(get_count_from_new_conn(&db, "failed").await, 0); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + } +} diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml new file mode 100644 index 0000000..6ed5796 --- /dev/null +++ b/hook-worker/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "hook-worker" +version = "0.1.0" +edition = "2021" + +[dependencies] +axum = { workspace = true } +chrono = { workspace = true } +envconfig = { workspace = true } +futures = "0.3" +hook-common = { path = "../hook-common" } +http = { version = "0.2" } +metrics = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +sqlx = { workspace = true } +time = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tokio = { workspace = true } +url = { version = "2.2" } diff --git a/hook-worker/README.md b/hook-worker/README.md new file mode 100644 index 0000000..9b1884a --- /dev/null +++ b/hook-worker/README.md @@ -0,0 +1,2 @@ +# hook-worker +Consume and process webhook jobs diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs new file mode 100644 index 0000000..ceb690f --- /dev/null +++ b/hook-worker/src/config.rs @@ -0,0 +1,101 @@ +use std::str::FromStr; +use std::time; + +use envconfig::Envconfig; + +#[derive(Envconfig, Clone)] +pub struct Config { + #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] + pub host: String, + + #[envconfig(from = "BIND_PORT", default = "3301")] + pub port: u16, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub database_url: String, + + #[envconfig(default = "worker")] + pub worker_name: String, + + #[envconfig(default = "default")] + pub queue_name: NonEmptyString, + + #[envconfig(default = "100")] + pub poll_interval: EnvMsDuration, + + #[envconfig(default = "5000")] + pub request_timeout: EnvMsDuration, + + #[envconfig(default = "1024")] + pub max_concurrent_jobs: usize, + + #[envconfig(default = "100")] + pub max_pg_connections: u32, + + #[envconfig(nested = true)] + pub retry_policy: RetryPolicyConfig, + + #[envconfig(default = "1")] + pub dequeue_batch_size: u32, +} + +impl Config { + /// Produce a host:port address for binding a TcpListener. + pub fn bind(&self) -> String { + format!("{}:{}", self.host, self.port) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct EnvMsDuration(pub time::Duration); + +#[derive(Debug, PartialEq, Eq)] +pub struct ParseEnvMsDurationError; + +impl FromStr for EnvMsDuration { + type Err = ParseEnvMsDurationError; + + fn from_str(s: &str) -> Result { + let ms = s.parse::().map_err(|_| ParseEnvMsDurationError)?; + + Ok(EnvMsDuration(time::Duration::from_millis(ms))) + } +} + +#[derive(Envconfig, Clone)] +pub struct RetryPolicyConfig { + #[envconfig(default = "2")] + pub backoff_coefficient: u32, + + #[envconfig(default = "1000")] + pub initial_interval: EnvMsDuration, + + #[envconfig(default = "100000")] + pub maximum_interval: EnvMsDuration, + + pub retry_queue_name: Option, +} + +#[derive(Debug, Clone)] +pub struct NonEmptyString(pub String); + +impl NonEmptyString { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct StringIsEmptyError; + +impl FromStr for NonEmptyString { + type Err = StringIsEmptyError; + + fn from_str(s: &str) -> Result { + if s.is_empty() { + Err(StringIsEmptyError) + } else { + Ok(NonEmptyString(s.to_owned())) + } + } +} diff --git a/hook-worker/src/error.rs b/hook-worker/src/error.rs new file mode 100644 index 0000000..614fe72 --- /dev/null +++ b/hook-worker/src/error.rs @@ -0,0 +1,33 @@ +use std::time; + +use hook_common::pgqueue; +use thiserror::Error; + +/// Enumeration of errors related to webhook job processing in the WebhookWorker. +#[derive(Error, Debug)] +pub enum WebhookError { + #[error("{0} is not a valid HttpMethod")] + ParseHttpMethodError(String), + #[error("error parsing webhook headers")] + ParseHeadersError(http::Error), + #[error("error parsing webhook url")] + ParseUrlError(url::ParseError), + #[error("a webhook could not be delivered but it could be retried later: {error}")] + RetryableRequestError { + error: reqwest::Error, + retry_after: Option, + }, + #[error("a webhook could not be delivered and it cannot be retried further: {0}")] + NonRetryableRetryableRequestError(reqwest::Error), +} + +/// Enumeration of errors related to initialization and consumption of webhook jobs. +#[derive(Error, Debug)] +pub enum WorkerError { + #[error("timed out while waiting for jobs to be available")] + TimeoutError, + #[error("an error occurred in the underlying queue")] + QueueError(#[from] pgqueue::PgQueueError), + #[error("an error occurred in the underlying job: {0}")] + PgJobError(String), +} diff --git a/hook-worker/src/lib.rs b/hook-worker/src/lib.rs new file mode 100644 index 0000000..22823c9 --- /dev/null +++ b/hook-worker/src/lib.rs @@ -0,0 +1,3 @@ +pub mod config; +pub mod error; +pub mod worker; diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs new file mode 100644 index 0000000..2997dfc --- /dev/null +++ b/hook-worker/src/main.rs @@ -0,0 +1,77 @@ +//! Consume `PgQueue` jobs to run webhook calls. +use axum::routing::get; +use axum::Router; +use envconfig::Envconfig; +use std::future::ready; + +use hook_common::health::HealthRegistry; +use hook_common::{ + metrics::serve, metrics::setup_metrics_routes, pgqueue::PgQueue, retry::RetryPolicy, +}; +use hook_worker::config::Config; +use hook_worker::error::WorkerError; +use hook_worker::worker::WebhookWorker; + +#[tokio::main] +async fn main() -> Result<(), WorkerError> { + tracing_subscriber::fmt::init(); + + let config = Config::init_from_env().expect("Invalid configuration:"); + + let liveness = HealthRegistry::new("liveness"); + let worker_liveness = liveness + .register("worker".to_string(), time::Duration::seconds(60)) // TODO: compute the value from worker params + .await; + + let mut retry_policy_builder = RetryPolicy::build( + config.retry_policy.backoff_coefficient, + config.retry_policy.initial_interval.0, + ) + .maximum_interval(config.retry_policy.maximum_interval.0); + + retry_policy_builder = if let Some(retry_queue_name) = &config.retry_policy.retry_queue_name { + retry_policy_builder.queue(retry_queue_name.as_str()) + } else { + retry_policy_builder + }; + + let queue = PgQueue::new( + config.queue_name.as_str(), + &config.database_url, + config.max_pg_connections, + "hook-worker", + ) + .await + .expect("failed to initialize queue"); + + let worker = WebhookWorker::new( + &config.worker_name, + &queue, + config.dequeue_batch_size, + config.poll_interval.0, + config.request_timeout.0, + config.max_concurrent_jobs, + retry_policy_builder.provide(), + worker_liveness, + ); + + let router = Router::new() + .route("/", get(index)) + .route("/_readiness", get(index)) + .route("/_liveness", get(move || ready(liveness.get_status()))); + let router = setup_metrics_routes(router); + let bind = config.bind(); + tokio::task::spawn(async move { + serve(router, &bind) + .await + .expect("failed to start serving metrics"); + }); + + worker.run().await; + + Ok(()) +} + +pub async fn index() -> &'static str { + "rusty-hook worker" +} diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs new file mode 100644 index 0000000..b83c909 --- /dev/null +++ b/hook-worker/src/worker.rs @@ -0,0 +1,562 @@ +use std::collections; +use std::sync::Arc; +use std::time; + +use futures::future::join_all; +use hook_common::health::HealthHandle; +use hook_common::pgqueue::PgTransactionBatch; +use hook_common::{ + pgqueue::{Job, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, + retry::RetryPolicy, + webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, +}; +use http::StatusCode; +use reqwest::header; +use tokio::sync; +use tracing::error; + +use crate::error::{WebhookError, WorkerError}; + +/// A WebhookJob is any `PgQueueJob` with `WebhookJobParameters` and `WebhookJobMetadata`. +trait WebhookJob: PgQueueJob + std::marker::Send { + fn parameters(&self) -> &WebhookJobParameters; + fn metadata(&self) -> &WebhookJobMetadata; + fn job(&self) -> &Job; + + fn attempt(&self) -> i32 { + self.job().attempt + } + + fn queue(&self) -> String { + self.job().queue.to_owned() + } + + fn target(&self) -> String { + self.job().target.to_owned() + } +} + +impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { + fn parameters(&self) -> &WebhookJobParameters { + &self.job.parameters + } + + fn metadata(&self) -> &WebhookJobMetadata { + &self.job.metadata + } + + fn job(&self) -> &Job { + &self.job + } +} + +/// A worker to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. +pub struct WebhookWorker<'p> { + /// An identifier for this worker. Used to mark jobs we have consumed. + name: String, + /// The queue we will be dequeuing jobs from. + queue: &'p PgQueue, + /// The maximum number of jobs to dequeue in one query. + dequeue_batch_size: u32, + /// The interval for polling the queue. + poll_interval: time::Duration, + /// The client used for HTTP requests. + client: reqwest::Client, + /// Maximum number of concurrent jobs being processed. + max_concurrent_jobs: usize, + /// The retry policy used to calculate retry intervals when a job fails with a retryable error. + retry_policy: RetryPolicy, + /// The liveness check handle, to call on a schedule to report healthy + liveness: HealthHandle, +} + +impl<'p> WebhookWorker<'p> { + #[allow(clippy::too_many_arguments)] + pub fn new( + name: &str, + queue: &'p PgQueue, + dequeue_batch_size: u32, + poll_interval: time::Duration, + request_timeout: time::Duration, + max_concurrent_jobs: usize, + retry_policy: RetryPolicy, + liveness: HealthHandle, + ) -> Self { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + + let client = reqwest::Client::builder() + .default_headers(headers) + .user_agent("PostHog Webhook Worker") + .timeout(request_timeout) + .build() + .expect("failed to construct reqwest client for webhook worker"); + + Self { + name: name.to_owned(), + queue, + dequeue_batch_size, + poll_interval, + client, + max_concurrent_jobs, + retry_policy, + liveness, + } + } + + /// Wait until at least one job becomes available in our queue in transactional mode. + async fn wait_for_jobs_tx<'a>( + &self, + ) -> PgTransactionBatch<'a, WebhookJobParameters, WebhookJobMetadata> { + let mut interval = tokio::time::interval(self.poll_interval); + + loop { + interval.tick().await; + self.liveness.report_healthy().await; + + match self + .queue + .dequeue_tx(&self.name, self.dequeue_batch_size) + .await + { + Ok(Some(batch)) => return batch, + Ok(None) => continue, + Err(error) => { + error!("error while trying to dequeue_tx job: {}", error); + continue; + } + } + } + } + + /// Run this worker to continuously process any jobs that become available. + pub async fn run(&self) { + let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); + let report_semaphore_utilization = || { + metrics::gauge!("webhook_worker_saturation_percent") + .set(1f64 - semaphore.available_permits() as f64 / self.max_concurrent_jobs as f64); + }; + + let dequeue_batch_size_histogram = metrics::histogram!("webhook_dequeue_batch_size"); + + loop { + report_semaphore_utilization(); + // TODO: We could grab semaphore permits here using something like: + // `min(semaphore.available_permits(), dequeue_batch_size)` + // And then dequeue only up to that many jobs. We'd then need to hand back the + // difference in permits based on how many jobs were dequeued. + let mut batch = self.wait_for_jobs_tx().await; + dequeue_batch_size_histogram.record(batch.jobs.len() as f64); + + // Get enough permits for the jobs before spawning a task. + let permits = semaphore + .clone() + .acquire_many_owned(batch.jobs.len() as u32) + .await + .expect("semaphore has been closed"); + + let client = self.client.clone(); + let retry_policy = self.retry_policy.clone(); + + tokio::spawn(async move { + let mut futures = Vec::new(); + + // We have to `take` the Vec of jobs from the batch to avoid a borrow checker + // error below when we commit. + for job in std::mem::take(&mut batch.jobs) { + let client = client.clone(); + let retry_policy = retry_policy.clone(); + + let future = + async move { process_webhook_job(client, job, &retry_policy).await }; + + futures.push(future); + } + + let results = join_all(futures).await; + for result in results { + if let Err(e) = result { + error!("error processing webhook job: {}", e); + } + } + + let _ = batch.commit().await.map_err(|e| { + error!("error committing transactional batch: {}", e); + }); + + drop(permits); + }); + } + } +} + +/// Process a webhook job by transitioning it to its appropriate state after its request is sent. +/// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request +/// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries +/// may be attempted). +/// +/// A webhook job is considered retryable after a failing request if: +/// 1. The job has attempts remaining (i.e. hasn't reached `max_attempts`), and... +/// 2. The status code indicates retrying at a later point could resolve the issue. This means: 429 and any 5XX. +/// +/// # Arguments +/// +/// * `client`: An HTTP client to execute the webhook job request. +/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. +/// * `retry_policy`: The retry policy used to set retry parameters if a job fails and has remaining attempts. +async fn process_webhook_job( + client: reqwest::Client, + webhook_job: W, + retry_policy: &RetryPolicy, +) -> Result<(), WorkerError> { + let parameters = webhook_job.parameters(); + + let labels = [("queue", webhook_job.queue())]; + metrics::counter!("webhook_jobs_total", &labels).increment(1); + + let now = tokio::time::Instant::now(); + + let send_result = send_webhook( + client, + ¶meters.method, + ¶meters.url, + ¶meters.headers, + parameters.body.clone(), + ) + .await; + + let elapsed = now.elapsed().as_secs_f64(); + + match send_result { + Ok(_) => { + webhook_job + .complete() + .await + .map_err(|error| WorkerError::PgJobError(error.to_string()))?; + + metrics::counter!("webhook_jobs_completed", &labels).increment(1); + metrics::histogram!("webhook_jobs_processing_duration_seconds", &labels) + .record(elapsed); + + Ok(()) + } + Err(WebhookError::ParseHeadersError(e)) => { + webhook_job + .fail(WebhookJobError::new_parse(&e.to_string())) + .await + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + + metrics::counter!("webhook_jobs_failed", &labels).increment(1); + + Ok(()) + } + Err(WebhookError::ParseHttpMethodError(e)) => { + webhook_job + .fail(WebhookJobError::new_parse(&e)) + .await + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + + metrics::counter!("webhook_jobs_failed", &labels).increment(1); + + Ok(()) + } + Err(WebhookError::ParseUrlError(e)) => { + webhook_job + .fail(WebhookJobError::new_parse(&e.to_string())) + .await + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + + metrics::counter!("webhook_jobs_failed", &labels).increment(1); + + Ok(()) + } + Err(WebhookError::RetryableRequestError { error, retry_after }) => { + let retry_interval = + retry_policy.retry_interval(webhook_job.attempt() as u32, retry_after); + let current_queue = webhook_job.queue(); + let retry_queue = retry_policy.retry_queue(¤t_queue); + + match webhook_job + .retry(WebhookJobError::from(&error), retry_interval, retry_queue) + .await + { + Ok(_) => { + metrics::counter!("webhook_jobs_retried", &labels).increment(1); + + Ok(()) + } + Err(PgJobError::RetryInvalidError { + job: webhook_job, .. + }) => { + webhook_job + .fail(WebhookJobError::from(&error)) + .await + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + + metrics::counter!("webhook_jobs_failed", &labels).increment(1); + + Ok(()) + } + Err(job_error) => Err(WorkerError::PgJobError(job_error.to_string())), + } + } + Err(WebhookError::NonRetryableRetryableRequestError(error)) => { + webhook_job + .fail(WebhookJobError::from(&error)) + .await + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + + metrics::counter!("webhook_jobs_failed", &labels).increment(1); + + Ok(()) + } + } +} + +/// Make an HTTP request to a webhook endpoint. +/// +/// # Arguments +/// +/// * `client`: An HTTP client to execute the HTTP request. +/// * `method`: The HTTP method to use in the HTTP request. +/// * `url`: The URL we are targetting with our request. Parsing this URL fail. +/// * `headers`: Key, value pairs of HTTP headers in a `std::collections::HashMap`. Can fail if headers are not valid. +/// * `body`: The body of the request. Ownership is required. +async fn send_webhook( + client: reqwest::Client, + method: &HttpMethod, + url: &str, + headers: &collections::HashMap, + body: String, +) -> Result { + let method: http::Method = method.into(); + let url: reqwest::Url = (url).parse().map_err(WebhookError::ParseUrlError)?; + let headers: reqwest::header::HeaderMap = (headers) + .try_into() + .map_err(WebhookError::ParseHeadersError)?; + let body = reqwest::Body::from(body); + + let response = client + .request(method, url) + .headers(headers) + .body(body) + .send() + .await + .map_err(|e| WebhookError::RetryableRequestError { + error: e, + retry_after: None, + })?; + + let retry_after = parse_retry_after_header(response.headers()); + + match response.error_for_status() { + Ok(response) => Ok(response), + Err(err) => { + if is_retryable_status( + err.status() + .expect("status code is set as error is generated from a response"), + ) { + Err(WebhookError::RetryableRequestError { + error: err, + retry_after, + }) + } else { + Err(WebhookError::NonRetryableRetryableRequestError(err)) + } + } + } +} + +fn is_retryable_status(status: StatusCode) -> bool { + status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() +} + +/// Attempt to parse a chrono::Duration from a Retry-After header, returning None if not possible. +/// Retry-After header can specify a date in RFC2822 or a number of seconds; we try to parse both. +/// If a Retry-After header is not present in the provided `header_map`, `None` is returned. +/// +/// # Arguments +/// +/// * `header_map`: A `&reqwest::HeaderMap` of response headers that could contain Retry-After. +fn parse_retry_after_header(header_map: &reqwest::header::HeaderMap) -> Option { + let retry_after_header = header_map.get(reqwest::header::RETRY_AFTER); + + let retry_after = match retry_after_header { + Some(header_value) => match header_value.to_str() { + Ok(s) => s, + Err(_) => { + return None; + } + }, + None => { + return None; + } + }; + + if let Ok(u) = retry_after.parse::() { + let duration = time::Duration::from_secs(u); + return Some(duration); + } + + if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(retry_after) { + let duration = + chrono::DateTime::::from(dt) - chrono::offset::Utc::now(); + + // This can only fail when negative, in which case we return None. + return duration.to_std().ok(); + } + + None +} + +mod tests { + use super::*; + // Note we are ignoring some warnings in this module. + // This is due to a long-standing cargo bug that reports imports and helper functions as unused. + // See: https://github.com/rust-lang/rust/issues/46379. + #[allow(unused_imports)] + use hook_common::health::HealthRegistry; + #[allow(unused_imports)] + use hook_common::pgqueue::{JobStatus, NewJob}; + #[allow(unused_imports)] + use sqlx::PgPool; + + /// Use process id as a worker id for tests. + #[allow(dead_code)] + fn worker_id() -> String { + std::process::id().to_string() + } + + #[allow(dead_code)] + async fn enqueue_job( + queue: &PgQueue, + max_attempts: i32, + job_parameters: WebhookJobParameters, + job_metadata: WebhookJobMetadata, + ) -> Result<(), PgQueueError> { + let job_target = job_parameters.url.to_owned(); + let new_job = NewJob::new(max_attempts, job_metadata, job_parameters, &job_target); + queue.enqueue(new_job).await?; + Ok(()) + } + + #[test] + fn test_is_retryable_status() { + assert!(!is_retryable_status(http::StatusCode::FORBIDDEN)); + assert!(!is_retryable_status(http::StatusCode::BAD_REQUEST)); + assert!(is_retryable_status(http::StatusCode::TOO_MANY_REQUESTS)); + assert!(is_retryable_status(http::StatusCode::INTERNAL_SERVER_ERROR)); + } + + #[test] + fn test_parse_retry_after_header() { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert(reqwest::header::RETRY_AFTER, "120".parse().unwrap()); + + let duration = parse_retry_after_header(&headers).unwrap(); + assert_eq!(duration, time::Duration::from_secs(120)); + + headers.remove(reqwest::header::RETRY_AFTER); + + let duration = parse_retry_after_header(&headers); + assert_eq!(duration, None); + + headers.insert( + reqwest::header::RETRY_AFTER, + "Wed, 21 Oct 2015 07:28:00 GMT".parse().unwrap(), + ); + + let duration = parse_retry_after_header(&headers); + assert_eq!(duration, None); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_wait_for_job(db: PgPool) { + let worker_id = worker_id(); + let queue_name = "test_wait_for_job".to_string(); + let queue = PgQueue::new_from_pool(&queue_name, db) + .await + .expect("failed to connect to PG"); + + let webhook_job_parameters = WebhookJobParameters { + body: "a webhook job body. much wow.".to_owned(), + headers: collections::HashMap::new(), + method: HttpMethod::POST, + url: "localhost".to_owned(), + }; + let webhook_job_metadata = WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }; + let registry = HealthRegistry::new("liveness"); + let liveness = registry + .register("worker".to_string(), ::time::Duration::seconds(30)) + .await; + // enqueue takes ownership of the job enqueued to avoid bugs that can cause duplicate jobs. + // Normally, a separate application would be enqueueing jobs for us to consume, so no ownership + // conflicts would arise. However, in this test we need to do the enqueueing ourselves. + // So, we clone the job to keep it around and assert the values returned by wait_for_job. + enqueue_job( + &queue, + 1, + webhook_job_parameters.clone(), + webhook_job_metadata, + ) + .await + .expect("failed to enqueue job"); + let worker = WebhookWorker::new( + &worker_id, + &queue, + 1, + time::Duration::from_millis(100), + time::Duration::from_millis(5000), + 10, + RetryPolicy::default(), + liveness, + ); + + let mut batch = worker.wait_for_jobs_tx().await; + let consumed_job = batch.jobs.pop().unwrap(); + + assert_eq!(consumed_job.job.attempt, 1); + assert!(consumed_job.job.attempted_by.contains(&worker_id)); + assert_eq!(consumed_job.job.attempted_by.len(), 1); + assert_eq!(consumed_job.job.max_attempts, 1); + assert_eq!( + *consumed_job.job.parameters.as_ref(), + webhook_job_parameters + ); + assert_eq!(consumed_job.job.target, webhook_job_parameters.url); + + consumed_job + .complete() + .await + .expect("job not successfully completed"); + batch.commit().await.expect("failed to commit batch"); + + assert!(registry.get_status().healthy) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_send_webhook(_: PgPool) { + let method = HttpMethod::POST; + let url = "http://localhost:18081/echo"; + let headers = collections::HashMap::new(); + let body = "a very relevant request body"; + let client = reqwest::Client::new(); + + let response = send_webhook(client, &method, url, &headers, body.to_owned()) + .await + .expect("send_webhook failed"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.text().await.expect("failed to read response body"), + body.to_owned(), + ); + } +} diff --git a/migrations/20231129172339_job_queue_table.sql b/migrations/20231129172339_job_queue_table.sql new file mode 100644 index 0000000..bf8c3df --- /dev/null +++ b/migrations/20231129172339_job_queue_table.sql @@ -0,0 +1,29 @@ +CREATE TYPE job_status AS ENUM( + 'available', + 'completed', + 'failed', + 'running' +); + +CREATE TABLE job_queue( + id BIGSERIAL PRIMARY KEY, + attempt INT NOT NULL DEFAULT 0, + attempted_at TIMESTAMPTZ DEFAULT NULL, + attempted_by TEXT [] DEFAULT ARRAY [] :: TEXT [], + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + errors JSONB [], + max_attempts INT NOT NULL DEFAULT 1, + metadata JSONB, + last_attempt_finished_at TIMESTAMPTZ DEFAULT NULL, + parameters JSONB, + queue TEXT NOT NULL DEFAULT 'default' :: text, + scheduled_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + status job_status NOT NULL DEFAULT 'available' :: job_status, + target TEXT NOT NULL +); + +-- Needed for `dequeue` queries +CREATE INDEX idx_queue_scheduled_at ON job_queue(queue, status, scheduled_at, attempt); + +-- Needed for UPDATE-ing incomplete jobs with a specific target (i.e. slow destinations) +CREATE INDEX idx_queue_target ON job_queue(queue, status, target); diff --git a/migrations/20240202003133_better_dequeue_index.sql b/migrations/20240202003133_better_dequeue_index.sql new file mode 100644 index 0000000..a619fb1 --- /dev/null +++ b/migrations/20240202003133_better_dequeue_index.sql @@ -0,0 +1,10 @@ +-- Dequeue is not hitting this index, so dropping is safe this time. +DROP INDEX idx_queue_scheduled_at; + +/* +Partial index used for dequeuing from job_queue. + +Dequeue only looks at available jobs so a partial index serves us well. +Moreover, dequeue sorts jobs by attempt and scheduled_at, which matches this index. +*/ +CREATE INDEX idx_queue_dequeue_partial ON job_queue(queue, attempt, scheduled_at) WHERE status = 'available' :: job_status;