Skip to content

Commit

Permalink
remove incorrect usage of curls Multi API
Browse files Browse the repository at this point in the history
There are several issues introduced by #29

When running an application that is rarely restarted, that links to this
sdk I encountered hangs inside of the select() call inside of http.c
multiple times, that required a restart of the application.

Looking into the man pages of curl, it is quite obvious, that the usage of select
is a bad idea here.

On one hand, the requests.fdset() call may return no
filedescriptors, if there is currently nothing to wait for.
(https://curl.se/libcurl/c/curl_multi_fdset.html)

when fdset() returns no filedescriptors, maxfd is set to -1.
this leads to a select call, that has nfds set to 0 and a disabled
timeout. -> this leads to an infinite select() until either the program
is terminated or a signal is received.

On the other hand, if your application is running for some time, new
sockets may have filedescriptors larger than FD_SETSIZE (1024),
according to the docs (https://curl.se/libcurl/c/curl_multi_fdset.html),
fdset does not return any filedescriptors in that case, which leads to
the infinite select() again.

There are two possible solutions to this:
1. waiting for this PR to be merged: jpbarrette/curlpp#173
   - Use the poll() or wait() call from there
2. revert to the Easy API again.
   - If I understand it correctly the only reason for using the Multi
     API was to be able to abort transfers inside of the ResponseCallback,
     but that was implemented incorrectly anyway.
     Currently the request is removed by requests.remove(request) inside
     of the ResponseCallback, which is forbidden by:
       'It is fine to remove a handle at any time during a transfer, just
        not from within any libcurl callback function.'
        (https://curl.se/libcurl/c/curl_multi_remove_handle.html)

Due to these reasons I propose to revert back to the Easy API.
To keep the possibility to abort a running request, CURL_WRITEFUNC_ERROR
is returned at the corresponding locations inside of the
ResponseCallback() function.
  • Loading branch information
Florian Kaiser committed Jan 5, 2025
1 parent b8ebbf3 commit a2526cc
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 41 deletions.
4 changes: 1 addition & 3 deletions include/miniocpp/http.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
#define MINIO_CPP_HTTP_H_INCLUDED

#include <curlpp/Easy.hpp>
#include <curlpp/Multi.hpp>
#include <exception>
#include <functional>
#include <iostream>
Expand Down Expand Up @@ -142,8 +141,7 @@ struct Response {
Response() = default;
~Response() = default;

size_t ResponseCallback(curlpp::Multi* const requests,
curlpp::Easy* const request, const char* const buffer,
size_t ResponseCallback(curlpp::Easy* const request, const char* const buffer,
size_t size, size_t length);

explicit operator bool() const {
Expand Down
50 changes: 12 additions & 38 deletions src/http.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
#include <curlpp/Easy.hpp>
#include <curlpp/Exception.hpp>
#include <curlpp/Infos.hpp>
#include <curlpp/Multi.hpp>
#include <curlpp/Options.hpp>
#include <curlpp/cURLpp.hpp>
#include <exception>
Expand All @@ -48,6 +47,10 @@
#include <arpa/inet.h>
#endif

#ifndef CURL_WRITEFUNC_ERROR
#define CURL_WRITEFUNC_ERROR static_cast<size_t>(0xFFFFFFFF)
#endif

namespace minio::http {

// MethodToString converts http Method enum to string.
Expand Down Expand Up @@ -269,16 +272,14 @@ error::Error Response::ReadHeaders() {
return error::SUCCESS;
}

size_t Response::ResponseCallback(curlpp::Multi* const requests,
curlpp::Easy* const request,
size_t Response::ResponseCallback(curlpp::Easy* const request,
const char* const buffer, size_t size,
size_t length) {
size_t realsize = size * length;

// If error occurred previously, just cancel the request.
if (!error.empty()) {
requests->remove(request);
return realsize;
return CURL_WRITEFUNC_ERROR;
}

if (!status_code_read_ || !headers_read_) {
Expand All @@ -288,8 +289,7 @@ size_t Response::ResponseCallback(curlpp::Multi* const requests,
if (!status_code_read_) {
if (error::Error err = ReadStatusCode()) {
error = err.String();
requests->remove(request);
return realsize;
return CURL_WRITEFUNC_ERROR;
}

if (!status_code_read_) return realsize;
Expand All @@ -298,8 +298,7 @@ size_t Response::ResponseCallback(curlpp::Multi* const requests,
if (!headers_read_) {
if (error::Error err = ReadHeaders()) {
error = err.String();
requests->remove(request);
return realsize;
return CURL_WRITEFUNC_ERROR;
}

if (!headers_read_ || response_.empty()) return realsize;
Expand All @@ -308,7 +307,7 @@ size_t Response::ResponseCallback(curlpp::Multi* const requests,
if (datafunc != nullptr && status_code >= 200 && status_code <= 299) {
DataFunctionArgs args(request, this, std::string(this->response_),
userdata);
if (!datafunc(args)) requests->remove(request);
if (!datafunc(args)) return CURL_WRITEFUNC_ERROR;
} else {
body = response_;
}
Expand All @@ -319,7 +318,7 @@ size_t Response::ResponseCallback(curlpp::Multi* const requests,
// If data function is set and the request is successful, send data.
if (datafunc != nullptr && status_code >= 200 && status_code <= 299) {
DataFunctionArgs args(request, this, std::string(buffer, length), userdata);
if (!datafunc(args)) requests->remove(request);
if (!datafunc(args)) return CURL_WRITEFUNC_ERROR;
} else {
body.append(buffer, length);
}
Expand All @@ -339,7 +338,6 @@ Request::Request(Method method, Url url) {
Response Request::execute() {
curlpp::Cleanup cleaner;
curlpp::Easy request;
curlpp::Multi requests;

// Request settings.
request.setOpt(new curlpp::options::CustomRequest{MethodToString(method)});
Expand Down Expand Up @@ -399,8 +397,7 @@ Response Request::execute() {

using namespace std::placeholders;
request.setOpt(new curlpp::options::WriteFunction(
std::bind(&Response::ResponseCallback, &response, &requests, &request, _1,
_2, _3)));
std::bind(&Response::ResponseCallback, &response, &request, _1, _2, _3)));

auto progress =
[&progressfunc = progressfunc, &progress_userdata = progress_userdata](
Expand All @@ -421,31 +418,8 @@ Response Request::execute() {
request.setOpt(new curlpp::options::ProgressFunction(progress));
}

int left = 0;
requests.add(&request);

// Execute.
while (!requests.perform(&left)) {
}
while (left) {
fd_set fdread{};
fd_set fdwrite{};
fd_set fdexcep{};
int maxfd = 0;

FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);

requests.fdset(&fdread, &fdwrite, &fdexcep, &maxfd);

if (select(maxfd + 1, &fdread, &fdwrite, &fdexcep, nullptr) < 0) {
std::cerr << "select() failed; this should not happen" << std::endl;
std::terminate();
}
while (!requests.perform(&left)) {
}
}
request.perform();

if (progressfunc != nullptr) {
ProgressFunctionArgs args;
Expand Down

0 comments on commit a2526cc

Please sign in to comment.