From 83aec033428e80700f9177f81512e6c7c62b45f6 Mon Sep 17 00:00:00 2001
From: Jason Rhinelander <jason@imaginary.ca>
Date: Fri, 19 Jul 2024 11:49:16 -0300
Subject: [PATCH 1/5] Refactor onion proxy requests to use curl multi mode

The current cpr::PostCallback approach used for proxy requests (e.g.
from the last hop to a SOGS or to the big file server) runs into
considerable limitations when there are more than a handful of
simultaneous requests, and can encounter deadlocks with bugs in cpr's
threadpool implementation.  This replaces it using libcurl's
single-threaded, "multi handle" approach, using our libevent event loop
(from libquic) to manage the requests.

This should allow more efficient requests and considerably more capacity
for parallel in-progress proxied requests in each storage server.
---
 CMakeLists.txt                 |   2 +-
 oxenss/CMakeLists.txt          |   1 +
 oxenss/daemon/oxen-storage.cpp |   7 +-
 oxenss/http/CMakeLists.txt     |  12 ++
 oxenss/http/http_client.cpp    | 206 +++++++++++++++++++++++++++++++++
 oxenss/http/http_client.h      |  67 +++++++++++
 oxenss/rpc/CMakeLists.txt      |   1 +
 oxenss/rpc/request_handler.cpp |  75 +++++-------
 oxenss/rpc/request_handler.h   |  13 ++-
 oxenss/server/quic.cpp         |   3 +-
 oxenss/server/quic.h           |   4 +-
 oxenss/snode/service_node.cpp  |  45 +++----
 oxenss/snode/service_node.h    |  10 +-
 13 files changed, 357 insertions(+), 89 deletions(-)
 create mode 100644 oxenss/http/CMakeLists.txt
 create mode 100644 oxenss/http/http_client.cpp
 create mode 100644 oxenss/http/http_client.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 051f1c38..19dd0ea2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,7 +13,7 @@ endif()
 cmake_minimum_required(VERSION 3.13)
 
 project(oxenss
-    VERSION 2.7.0
+    VERSION 2.7.1
     LANGUAGES CXX C)
 
 set(CMAKE_CXX_STANDARD 20)
diff --git a/oxenss/CMakeLists.txt b/oxenss/CMakeLists.txt
index 5c241580..78c48d52 100644
--- a/oxenss/CMakeLists.txt
+++ b/oxenss/CMakeLists.txt
@@ -15,6 +15,7 @@ endif()
 add_subdirectory(common)
 add_subdirectory(crypto)
 add_subdirectory(daemon)
+add_subdirectory(http)
 add_subdirectory(logging)
 add_subdirectory(rpc)
 add_subdirectory(server)
diff --git a/oxenss/daemon/oxen-storage.cpp b/oxenss/daemon/oxen-storage.cpp
index 8b4f75b3..09301ca3 100644
--- a/oxenss/daemon/oxen-storage.cpp
+++ b/oxenss/daemon/oxen-storage.cpp
@@ -166,7 +166,7 @@ int main(int argc, char* argv[]) {
                 ssl_dh,
                 {me.pubkey_legacy, private_key}};
 
-        auto quic = std::make_shared<server::QUIC>(
+        auto quic = std::make_unique<server::QUIC>(
                 service_node,
                 request_handler,
                 rate_limiter,
@@ -174,6 +174,10 @@ int main(int argc, char* argv[]) {
                 private_key_ed25519);
         service_node.register_mq_server(quic.get());
 
+        auto http_client = std::make_shared<http::Client>(&quic->net());
+        service_node.set_http_client(http_client);
+        request_handler.set_http_client(http_client);
+
         oxenmq_server.init(
                 &service_node,
                 &request_handler,
@@ -202,6 +206,7 @@ int main(int argc, char* argv[]) {
             std::this_thread::sleep_for(100ms);
 
         log::warning(logcat, "Received signal {}; shutting down...", signalled.load());
+        http_client.reset();  // Kills outgoing requests and prevents new ones
         service_node.shutdown();
         log::info(logcat, "Stopping https server");
         https_server.shutdown(true);
diff --git a/oxenss/http/CMakeLists.txt b/oxenss/http/CMakeLists.txt
new file mode 100644
index 00000000..052d0e80
--- /dev/null
+++ b/oxenss/http/CMakeLists.txt
@@ -0,0 +1,12 @@
+
+add_library(http STATIC
+    http_client.cpp)
+
+target_link_libraries(http
+    PRIVATE
+    version
+    oxen::logging
+    PUBLIC
+    cpr::cpr
+    quic::quic
+    )
diff --git a/oxenss/http/http_client.cpp b/oxenss/http/http_client.cpp
new file mode 100644
index 00000000..51dada09
--- /dev/null
+++ b/oxenss/http/http_client.cpp
@@ -0,0 +1,206 @@
+#include "http_client.h"
+#include <curl/curl.h>
+#include <chrono>
+#include <oxen/log.hpp>
+#include <cpr/ssl_options.h>
+#include <oxenss/version.h>
+
+namespace oxenss::http {
+
+namespace log = oxen::log;
+
+auto logcat = log::Cat("http");
+
+static void curl_perform(int fd, short event, void* void_ctx);
+
+struct curl_context {
+    Client& client;
+    curl_socket_t sockfd;
+    event* evt;
+
+    curl_context(Client& client, curl_socket_t fd) :
+            client{client},
+            sockfd{fd},
+            evt{event_new(client.loop->loop().get(), sockfd, 0, Client::curl_perform_c, this)} {}
+    ~curl_context() {
+        event_del(evt);
+        event_free(evt);
+    }
+};
+
+void Client::curl_perform_c(int /*fd*/, short event, void* cctx) {
+    int running_handles;
+    int flags = 0;
+    auto* ctx = static_cast<curl_context*>(cctx);
+    auto& client = ctx->client;
+
+    if (event & EV_READ)
+        flags |= CURL_CSELECT_IN;
+    if (event & EV_WRITE)
+        flags |= CURL_CSELECT_OUT;
+
+    curl_multi_socket_action(client.curl_multi, ctx->sockfd, flags, &running_handles);
+    // Can't use `ctx` anymore because it might have been destroyed during the above call (typically
+    // because the socket is no longer being polled).
+
+    client.check_multi_info();
+}
+
+void Client::on_timeout_c(evutil_socket_t /*fd*/, short /*events*/, void* arg) {
+    auto& client = *static_cast<Client*>(arg);
+    int running_handles;
+    curl_multi_socket_action(client.curl_multi, CURL_SOCKET_TIMEOUT, 0, &running_handles);
+    client.check_multi_info();
+}
+
+int Client::start_timeout_c(CURLM* /*multi*/, long timeout_ms, void* userp) {
+    auto& client = *static_cast<Client*>(userp);
+    evtimer_del(client.ev_timeout);
+    if (timeout_ms >= 0) {
+        timeval tv;
+        tv.tv_sec = timeout_ms / 1000;
+        tv.tv_usec = (timeout_ms % 1000) * 1000;
+        if (timeout_ms == 0)
+            tv.tv_usec = 1; /* 0 means call socket_action asap */
+        evtimer_add(client.ev_timeout, &tv);
+    }
+    return 0;
+}
+
+int Client::handle_socket_c(
+        CURL* /*easy*/, curl_socket_t s, int action, void* userp, void* socketp) {
+    auto& client = *static_cast<Client*>(userp);
+    auto* curl_ctx = static_cast<curl_context*>(socketp);
+    int events = 0;
+
+    switch (action) {
+        case CURL_POLL_IN:
+        case CURL_POLL_OUT:
+        case CURL_POLL_INOUT:
+            if (!curl_ctx) {
+                curl_ctx = new curl_context{client, s};
+                curl_multi_assign(client.curl_multi, s, curl_ctx);
+            }
+
+            if (action != CURL_POLL_IN)
+                events |= EV_WRITE;
+            if (action != CURL_POLL_OUT)
+                events |= EV_READ;
+
+            events |= EV_PERSIST;
+
+            event_del(curl_ctx->evt);
+            event_assign(
+                    curl_ctx->evt,
+                    client.loop->loop().get(),
+                    curl_ctx->sockfd,
+                    events,
+                    Client::curl_perform_c,
+                    curl_ctx);
+            event_add(curl_ctx->evt, NULL);
+
+            break;
+        case CURL_POLL_REMOVE:
+            if (curl_ctx) {
+                curl_multi_assign(client.curl_multi, s, nullptr);
+                delete curl_ctx;
+            }
+            break;
+        default: log::error(logcat, "Unexpected socket action {} from libcurl", action);
+    }
+
+    return 0;
+}
+
+void Client::check_multi_info() {
+    int pending;
+    while (CURLMsg* message = curl_multi_info_read(curl_multi, &pending)) {
+        if (message->msg == CURLMSG_DONE) {
+            cpr::Session* raw_sess;
+            curl_easy_getinfo(message->easy_handle, CURLINFO_PRIVATE, &raw_sess);
+            assert(raw_sess);
+            auto session = raw_sess->shared_from_this();
+            assert(session);
+            auto resp = session->Complete(message->data.result);
+            auto it = active_reqs.find(session);
+            assert(it != active_reqs.end());
+            if (it->second) {
+                try {
+                    it->second(std::move(resp));
+                } catch (const std::exception& e) {
+                    log::error(logcat, "HTTP response handler raised exception: {}", e.what());
+                }
+            }
+            active_reqs.erase(it);
+            curl_multi_remove_handle(curl_multi, message->easy_handle);
+            break;
+        } else {
+            log::warning(
+                    logcat,
+                    "Unexpected/unhandled curl-multi message type: {}",
+                    static_cast<int>(message->msg));
+        }
+    }
+}
+
+Client::Client(oxen::quic::Network* loop_) :
+        loop{std::move(loop_)},
+        ev_timeout{evtimer_new(loop->loop().get(), Client::on_timeout_c, this)} {
+    assert(loop);
+    curl_multi = curl_multi_init();
+    curl_multi_setopt(curl_multi, CURLMOPT_SOCKETDATA, this);
+    curl_multi_setopt(curl_multi, CURLMOPT_SOCKETFUNCTION, Client::handle_socket_c);
+    curl_multi_setopt(curl_multi, CURLMOPT_TIMERDATA, this);
+    curl_multi_setopt(curl_multi, CURLMOPT_TIMERFUNCTION, Client::start_timeout_c);
+}
+
+Client::~Client() {
+    loop->call_get([this] {
+        alive.reset();
+        for (auto& [session, cb] : active_reqs)
+            curl_multi_remove_handle(curl_multi, session->GetCurlHolder()->handle);
+        active_reqs.clear();
+        curl_multi_cleanup(curl_multi);
+        event_free(ev_timeout);
+    });
+}
+
+void Client::post(
+        response_callback cb,
+        std::string url,
+        std::string payload,
+        std::chrono::milliseconds timeout,
+        std::optional<std::string> host_override,
+        bool https_disable_validation) {
+    auto sess = std::make_shared<cpr::Session>();
+    sess->SetUrl(std::move(url));
+    cpr::Header header{
+            {"User-Agent", fmt::format("Oxen Storage Server/{}", STORAGE_SERVER_VERSION_STRING)},
+            {"Content-Type", "application/octet-stream"}};
+    if (host_override)
+        header["Host"] = *host_override;
+    sess->SetHeader(std::move(header));
+    sess->SetTimeout(timeout);
+    auto ssl_opts = cpr::Ssl(cpr::ssl::TLSv1_2{});  // TLSv1_2 means "1.2 or later"
+    if (https_disable_validation) {
+        ssl_opts.SetOption(cpr::ssl::VerifyHost{false});
+        ssl_opts.SetOption(cpr::ssl::VerifyPeer{false});
+        ssl_opts.SetOption(cpr::ssl::VerifyStatus{false});
+    }
+    sess->SetSslOptions(std::move(ssl_opts));
+    sess->SetRedirect(cpr::Redirect{0L});
+    sess->SetBody(std::move(payload));
+    curl_easy_setopt(sess->GetCurlHolder()->handle, CURLOPT_PRIVATE, sess.get());
+    sess->PreparePost();
+    loop->call([this,
+                alive = std::weak_ptr{alive},
+                sess = std::move(sess),
+                cb = std::move(cb)]() mutable {
+        if (alive.expired())
+            return;  // this got destroyed before we got into the call
+        curl_multi_add_handle(curl_multi, sess->GetCurlHolder()->handle);
+        active_reqs.emplace(std::move(sess), std::move(cb));
+    });
+}
+
+}  // namespace oxenss::http
diff --git a/oxenss/http/http_client.h b/oxenss/http/http_client.h
new file mode 100644
index 00000000..ee20b6ed
--- /dev/null
+++ b/oxenss/http/http_client.h
@@ -0,0 +1,67 @@
+#pragma once
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include <cpr/response.h>
+#include <cpr/session.h>
+#include <curl/curl.h>
+#include <oxen/quic/network.hpp>
+
+namespace oxenss::http {
+
+/// Async client for making outbound storage server HTTP post requests.
+class Client {
+  public:
+    using response_callback = std::function<void(cpr::Response r)>;
+
+    // Starts a new client, attaching itself to the event loop and ready for requests.
+    explicit Client(oxen::quic::Network* loop);
+
+    // Non-copyable, non-movable
+    Client(const Client&) = delete;
+    Client(Client&&) = delete;
+    Client& operator=(const Client&) = delete;
+    Client& operator=(Client&&) = delete;
+
+    // Kills all current requests and shuts down.  Callbacks on pending requests are *not* invoked.
+    ~Client();
+
+    // Initiates a new POST request.  When the request complete (or times out) `cb` will be invoked
+    // with the cpr::Response object.  Note that cb is invoked inside the event loop context, so it
+    // should try to be fast and definitely not do anything blocking.
+    void post(
+            response_callback cb,
+            std::string url,
+            std::string payload,
+            std::chrono::milliseconds timeout,
+            std::optional<std::string> host_override = std::nullopt,
+            bool https_disable_validation = false);
+
+  private:
+    // FIXME: in future (dev, as of writing) versions of libquic we should a shared_ptr<Loop>
+    // instead of this raw Network pointer, but currently Network doesn't give us a public interface
+    // to accessing its Loop, so we whole the bigger Network here instead.  (It is still named
+    // "loop", though, because we really only want a loop and Network has proxy functions for all
+    // the Loop functions we use that should work fine without needing other changes when this gets
+    // fixed in the future).
+    oxen::quic::Network* loop;
+    event* ev_timeout;
+    std::shared_ptr<const bool> alive = std::make_shared<bool>(true);
+    CURLM* curl_multi;
+    std::unordered_map<std::shared_ptr<cpr::Session>, response_callback> active_reqs;
+
+    friend struct curl_context;
+
+    static void curl_perform_c(int fd, short event, void* cctx);
+    static void on_timeout_c(evutil_socket_t fd, short events, void* arg);
+    static int start_timeout_c(CURLM* multi, long timeout_ms, void* userp);
+    static int handle_socket_c(CURL* easy, curl_socket_t s, int action, void* self, void* socketp);
+
+    void check_multi_info();
+};
+
+}  // namespace oxenss::http
diff --git a/oxenss/rpc/CMakeLists.txt b/oxenss/rpc/CMakeLists.txt
index 4307655c..bc29b95d 100644
--- a/oxenss/rpc/CMakeLists.txt
+++ b/oxenss/rpc/CMakeLists.txt
@@ -13,6 +13,7 @@ target_link_libraries(rpc
     crypto
     server
     snode
+    http
     utils
     logging
     version
diff --git a/oxenss/rpc/request_handler.cpp b/oxenss/rpc/request_handler.cpp
index 1aa2619f..8dc7d1c3 100644
--- a/oxenss/rpc/request_handler.cpp
+++ b/oxenss/rpc/request_handler.cpp
@@ -14,9 +14,7 @@
 #include <oxenss/crypto/channel_encryption.hpp>
 
 #include <chrono>
-#include <future>
 
-#include <cpr/cpr.h>
 #include <nlohmann/json.hpp>
 #include <oxenc/base32z.h>
 #include <oxenc/base64.h>
@@ -27,7 +25,6 @@
 #include <sodium/crypto_scalarmult_curve25519.h>
 #include <sodium/crypto_scalarmult_ed25519.h>
 #include <sodium/crypto_sign.h>
-#include <stdexcept>
 #include <type_traits>
 #include <variant>
 
@@ -375,15 +372,7 @@ std::string computeMessageHash(const user_pubkey& pubkey, namespace_id ns, std::
 
 RequestHandler::RequestHandler(
         snode::ServiceNode& sn, const crypto::ChannelEncryption& ce, crypto::ed25519_seckey edsk) :
-        service_node_{sn}, channel_cipher_(ce), ed25519_sk_{std::move(edsk)} {
-    // Periodically clean up any proxy request futures
-    service_node_.omq_server()->add_timer(
-            [this] {
-                pending_proxy_requests_.remove_if(
-                        [](auto& f) { return f.wait_for(0ms) == std::future_status::ready; });
-            },
-            1s);
-}
+        service_node_{sn}, channel_cipher_(ce), ed25519_sk_{std::move(edsk)} {}
 
 Response RequestHandler::handle_wrong_swarm(const user_pubkey& pubKey) {
     log::trace(logcat, "Got client request to a wrong swarm");
@@ -1755,39 +1744,37 @@ void RequestHandler::process_onion_req(RelayToServerInfo&& info, OnionRequestMet
 
     service_node_.record_proxy_request();
 
-    pending_proxy_requests_.emplace_front(cpr::PostCallback(
-            [&omq = *service_node_.omq_server(), cb = std::move(data.cb)](cpr::Response r) {
-                Response res;
-                if (r.error.code != cpr::ErrorCode::OK) {
-                    log::debug(
-                            logcat,
-                            "Onion proxied request to {} failed: {}",
-                            r.url.str(),
-                            r.error.message);
-                    res.body = r.error.message;
-                    if (r.error.code == cpr::ErrorCode::OPERATION_TIMEDOUT)
-                        res.status = http::GATEWAY_TIMEOUT;
-                    else
-                        res.status = http::BAD_GATEWAY;
-                } else {
-                    res.status.first = r.status_code;
-                    res.status.second = r.status_line;
-                    for (auto& [k, v] : r.header)
-                        res.headers.emplace_back(std::move(k), std::move(v));
-                    res.body = std::move(r.text);
-                }
+    if (auto http = http_.lock()) {
+        http->post(
+                [&omq = *service_node_.omq_server(), cb = std::move(data.cb)](cpr::Response r) {
+                    Response res;
+                    if (r.error.code != cpr::ErrorCode::OK) {
+                        log::debug(
+                                logcat,
+                                "Onion proxied request to {} failed: {}",
+                                r.url.str(),
+                                r.error.message);
+                        res.body = r.error.message;
+                        if (r.error.code == cpr::ErrorCode::OPERATION_TIMEDOUT)
+                            res.status = http::GATEWAY_TIMEOUT;
+                        else
+                            res.status = http::BAD_GATEWAY;
+                    } else {
+                        res.status.first = r.status_code;
+                        res.status.second = r.status_line;
+                        for (auto& [k, v] : r.header)
+                            res.headers.emplace_back(std::move(k), std::move(v));
+                        res.body = std::move(r.text);
+                    }
 
-                cb(std::move(res));
-            },
-            cpr::Url{std::move(urlstr)},
-            cpr::Header{
-                    {"User-Agent",
-                     "Oxen Storage Server/" + std::string{STORAGE_SERVER_VERSION_STRING}},
-                    {"Content-Type", "application/octet-stream"}},
-            cpr::Timeout{ONION_URL_TIMEOUT},
-            cpr::Ssl(cpr::ssl::TLSv1_2{}),
-            cpr::Redirect{0L},
-            cpr::Body{std::move(info.payload)}));
+                    cb(std::move(res));
+                },
+                std::move(urlstr),
+                std::move(info.payload),
+                ONION_URL_TIMEOUT);
+    } else {
+        log::warning(logcat, "Ignoring onion relay request received during shutdown");
+    }
 }
 
 void RequestHandler::process_onion_req(
diff --git a/oxenss/rpc/request_handler.h b/oxenss/rpc/request_handler.h
index 8897eafa..7b9a6948 100644
--- a/oxenss/rpc/request_handler.h
+++ b/oxenss/rpc/request_handler.h
@@ -9,17 +9,15 @@
 #include <oxenss/utils/string_utils.hpp>
 #include <oxenss/server/utils.h>
 #include <oxenss/utils/time.hpp>
+#include <oxenss/http/http_client.h>
 
 #include <chrono>
-#include <forward_list>
-#include <future>
 #include <string>
 #include <string_view>
 #include <type_traits>
+#include <variant>
 
 #include <nlohmann/json_fwd.hpp>
-#include <cpr/async_wrapper.h>
-#include <variant>
 
 namespace oxenss::rpc {
 
@@ -151,8 +149,7 @@ class RequestHandler {
     snode::ServiceNode& service_node_;
     const crypto::ChannelEncryption& channel_cipher_;
     const crypto::ed25519_seckey ed25519_sk_;
-
-    std::forward_list<cpr::AsyncWrapper<void>> pending_proxy_requests_;
+    std::weak_ptr<http::Client> http_;
 
     // Wrap response `res` to an intermediate node
     Response wrap_proxy_response(
@@ -185,6 +182,10 @@ class RequestHandler {
             const crypto::ChannelEncryption& ce,
             crypto::ed25519_seckey ed_sk);
 
+    // Sets the http client needed to perform proxied onion requests.  This must be set up before
+    // incoming requests are accepted.
+    void set_http_client(std::weak_ptr<http::Client> client) { http_ = std::move(client); }
+
     // Handlers for parsed client requests
     void process_client_req(rpc::store&& req, std::function<void(Response)> cb);
     void process_client_req(rpc::retrieve&& req, std::function<void(Response)> cb);
diff --git a/oxenss/server/quic.cpp b/oxenss/server/quic.cpp
index b91d6601..6abb2ec9 100644
--- a/oxenss/server/quic.cpp
+++ b/oxenss/server/quic.cpp
@@ -38,9 +38,8 @@ QUIC::QUIC(
         const Address& bind,
         const crypto::ed25519_seckey& sk) :
         local{bind},
-        network{std::make_unique<quic::Network>()},
         tls_creds{quic::GNUTLSCreds::make_from_ed_seckey(sk.str())},
-        ep{network->endpoint(
+        ep{network.endpoint(
                 local,
                 make_endpoint_static_secret(sk),
                 quic::opt::inbound_alpns{{uALPN}},
diff --git a/oxenss/server/quic.h b/oxenss/server/quic.h
index 5a7b181b..44b3ef74 100644
--- a/oxenss/server/quic.h
+++ b/oxenss/server/quic.h
@@ -49,9 +49,11 @@ class QUIC : public MQBase {
 
     void reachability_test(std::shared_ptr<snode::sn_test> test) override;
 
+    oxen::quic::Network& net() { return network; }
+
   private:
     const Address local;
-    std::unique_ptr<quic::Network> network;
+    quic::Network network;
     std::shared_ptr<quic::GNUTLSCreds> tls_creds;
     std::shared_ptr<quic::Endpoint> ep;
 
diff --git a/oxenss/snode/service_node.cpp b/oxenss/snode/service_node.cpp
index d7448d33..036a812c 100644
--- a/oxenss/snode/service_node.cpp
+++ b/oxenss/snode/service_node.cpp
@@ -13,7 +13,6 @@
 #include <oxenss/utils/random.hpp>
 
 #include <chrono>
-#include <cpr/cpr.h>
 #include <mutex>
 #include <nlohmann/json.hpp>
 #include <oxenc/base32z.h>
@@ -62,14 +61,6 @@ ServiceNode::ServiceNode(
             },
             Database::CLEANUP_PERIOD);
 
-    // Periodically clean up any https request futures
-    omq_server_->add_timer(
-            [this] {
-                outstanding_https_reqs_.remove_if(
-                        [](auto& f) { return f.wait_for(0ms) == std::future_status::ready; });
-            },
-            1s);
-
     // We really want to make sure nodes don't get stuck in "syncing" mode,
     // so if we are still "syncing" after a long time, activate SN regardless
     auto delay_timer = std::make_shared<oxenmq::TimerID>();
@@ -791,6 +782,12 @@ void ServiceNode::test_reachability(const sn_record& sn, int previous_failures)
             previous_failures > 0 ? "previously failing" : "random",
             sn.pubkey_legacy);
 
+    auto http = http_.lock();
+    if (!http) {
+        log::debug(logcat, "Skipping reachability test during shutdown");
+        return;
+    }
+
     if (sn.ip == "0.0.0.0") {
         // oxend won't accept 0.0.0.0 in an uptime proof, which means if we see this the node
         // hasn't sent an uptime proof; we could treat it as a failure, but that seems
@@ -810,18 +807,13 @@ void ServiceNode::test_reachability(const sn_record& sn, int previous_failures)
     for (auto* mq : mq_servers_)
         mq->reachability_test(test);
 
-    cpr::Url url{fmt::format("https://{}:{}/ping_test/v1", sn.ip, sn.port)};
-    cpr::Body body{""};
-    cpr::Header headers{
-            {"Host",
-             sn.pubkey_ed25519 ? oxenc::to_base32z(sn.pubkey_ed25519.view()) + ".snode"
-                               : "service-node.snode"},
-            {"Content-Type", "application/octet-stream"},
-            {"User-Agent", "Oxen Storage Server/" + std::string{STORAGE_SERVER_VERSION_STRING}},
-    };
+    auto url = fmt::format("https://{}:{}/ping_test/v1", sn.ip, sn.port);
+    std::optional<std::string> host;
+    if (sn.pubkey_ed25519)
+        host = oxenc::to_base32z(sn.pubkey_ed25519.view()) + ".snode";
 
-    log::debug(logcat, "Sending HTTPS ping to {} @ {}", sn.pubkey_legacy, url.str());
-    outstanding_https_reqs_.emplace_front(cpr::PostCallback(
+    log::debug(logcat, "Sending HTTPS ping to {} @ {}", sn.pubkey_legacy, url);
+    http->post(
             [test](cpr::Response r) {
                 auto& sn = test->sn;
                 auto& pk = sn.pubkey_legacy;
@@ -858,15 +850,10 @@ void ServiceNode::test_reachability(const sn_record& sn, int previous_failures)
                 test->add_result(success);
             },
             std::move(url),
-            cpr::Timeout{SN_PING_TIMEOUT},
-            cpr::Ssl(
-                    cpr::ssl::TLSv1_2{},
-                    cpr::ssl::VerifyHost{false},
-                    cpr::ssl::VerifyPeer{false},
-                    cpr::ssl::VerifyStatus{false}),
-            cpr::Redirect{0L},
-            std::move(headers),
-            std::move(body)));
+            ""s /*body*/,
+            SN_PING_TIMEOUT,
+            std::move(host),
+            true /*disable https validation*/);
 }
 
 void ServiceNode::oxend_ping() {
diff --git a/oxenss/snode/service_node.h b/oxenss/snode/service_node.h
index e0c6ab23..c9f1c69d 100644
--- a/oxenss/snode/service_node.h
+++ b/oxenss/snode/service_node.h
@@ -1,8 +1,6 @@
 #pragma once
 
 #include <chrono>
-#include <forward_list>
-#include <future>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -10,10 +8,10 @@
 #include <string>
 #include <string_view>
 
-#include <cpr/async_wrapper.h>
 #include <oxenss/storage/database.hpp>
 #include <oxenss/crypto/keys.h>
 #include <oxenss/server/mqbase.h>
+#include <oxenss/http/http_client.h>
 #include "reachability_testing.h"
 #include "stats.h"
 #include "swarm.h"
@@ -89,6 +87,7 @@ class ServiceNode {
     std::string block_hash_;
     std::unique_ptr<Swarm> swarm_;
     std::unique_ptr<Database> db_;
+    std::weak_ptr<http::Client> http_;
 
     SnodeStatus status_ = SnodeStatus::UNKNOWN;
 
@@ -115,8 +114,6 @@ class ServiceNode {
 
     mutable std::recursive_mutex sn_mutex_;
 
-    std::forward_list<cpr::AsyncWrapper<void>> outstanding_https_reqs_;
-
     void send_notifies(message m);
 
     // Save multiple messages to the database at once (i.e. in a single transaction)
@@ -188,6 +185,9 @@ class ServiceNode {
     // should not be added.
     void register_mq_server(server::MQBase* server);
 
+    // Sets the http client needed to perform HTTPS reachability tests
+    void set_http_client(std::weak_ptr<http::Client> client) { http_ = std::move(client); }
+
     // Return info about this node as it is advertised to other nodes
     const sn_record& own_address() { return our_address_; }
 

From 59a61ce740fab724db7794d6f69d7dfad2cb2ad5 Mon Sep 17 00:00:00 2001
From: Jason Rhinelander <jason@imaginary.ca>
Date: Fri, 19 Jul 2024 11:49:16 -0300
Subject: [PATCH 2/5] Fix missing linking to libevent

---
 oxenss/http/CMakeLists.txt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/oxenss/http/CMakeLists.txt b/oxenss/http/CMakeLists.txt
index 052d0e80..10123ad2 100644
--- a/oxenss/http/CMakeLists.txt
+++ b/oxenss/http/CMakeLists.txt
@@ -1,4 +1,11 @@
 
+if(NOT TARGET libevent::core)
+    add_library(libevent_core INTERFACE)
+    pkg_check_modules(LIBEVENT_core libevent_core>=2.1 IMPORTED_TARGET REQUIRED)
+    target_link_libraries(libevent_core INTERFACE PkgConfig::LIBEVENT_core)
+    add_library(libevent::core ALIAS libevent_core)
+endif()
+
 add_library(http STATIC
     http_client.cpp)
 
@@ -6,6 +13,7 @@ target_link_libraries(http
     PRIVATE
     version
     oxen::logging
+    libevent::core
     PUBLIC
     cpr::cpr
     quic::quic

From 06f3ef1be6ce8be8141f01340ed38450735262d2 Mon Sep 17 00:00:00 2001
From: Jason Rhinelander <jason@imaginary.ca>
Date: Fri, 19 Jul 2024 19:44:51 -0300
Subject: [PATCH 3/5] Fix oxen logging path stripping

---
 external/CMakeLists.txt | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt
index 621f1466..269c2ce4 100644
--- a/external/CMakeLists.txt
+++ b/external/CMakeLists.txt
@@ -60,12 +60,8 @@ macro(system_or_submodule BIGNAME smallname pkgconf subdir)
   endif()
 endmacro()
 
-set(OXEN_LOGGING_SOURCE_ROOT "${PROJECT_SOURCE_DIR}/oxenss" CACHE INTERNAL "")
-
 system_or_submodule(OXENC oxenc liboxenc>=1.0.10 oxenc)
 
-set(OXEN_LOGGING_SOURCE_ROOT "${OXEN_LOGGING_SOURCE_ROOT};${PROJECT_SOURCE_DIR}" CACHE INTERNAL "")
-
 set(LIBQUIC_BUILD_TESTS OFF CACHE BOOL "")
 system_or_submodule(OXENQUIC quic liboxenquic>=1.1.0 oxen-libquic)
 
@@ -81,6 +77,8 @@ endif()
 if(NOT TARGET oxen::logging)
     add_subdirectory(oxen-logging)
 endif()
+oxen_logging_add_source_dir("${PROJECT_SOURCE_DIR}")
+oxen_logging_add_source_dir("${PROJECT_SOURCE_DIR}/oxenss")
 
 
 # uSockets doesn't really have a proper build system (just a very simple Makefile) so build it

From ed3991d1a1ff0ecc72eb2fa279a74d5898c6887a Mon Sep 17 00:00:00 2001
From: Jason Rhinelander <jason@imaginary.ca>
Date: Mon, 22 Jul 2024 11:40:18 -0300
Subject: [PATCH 4/5] Bump version to 2.8.0

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 19dd0ea2..3f9098e0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,7 +13,7 @@ endif()
 cmake_minimum_required(VERSION 3.13)
 
 project(oxenss
-    VERSION 2.7.1
+    VERSION 2.8.0
     LANGUAGES CXX C)
 
 set(CMAKE_CXX_STANDARD 20)

From 75eb69babfeec4267b1114309dd22cbb6e7121d6 Mon Sep 17 00:00:00 2001
From: Jason Rhinelander <jason@imaginary.ca>
Date: Mon, 22 Jul 2024 11:45:04 -0300
Subject: [PATCH 5/5] Bump oxen-libquic and oxen-logging to latest

---
 external/oxen-libquic | 2 +-
 external/oxen-logging | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/external/oxen-libquic b/external/oxen-libquic
index 75d4f825..bc7167f9 160000
--- a/external/oxen-libquic
+++ b/external/oxen-libquic
@@ -1 +1 @@
-Subproject commit 75d4f825d5e39a99f7736ea0863ad227d23ccedc
+Subproject commit bc7167f90e71643b43c2ea9cf7d1fefa5045f8d4
diff --git a/external/oxen-logging b/external/oxen-logging
index 21dae0f8..5e0e0f06 160000
--- a/external/oxen-logging
+++ b/external/oxen-logging
@@ -1 +1 @@
-Subproject commit 21dae0f88c998374fee32d020dd6a647cba3076e
+Subproject commit 5e0e0f064ebd721a1e280af8850eda8397d70173