Skip to content

Commit

Permalink
Merge pull request #4 from deedy5/response_with_pytypes
Browse files Browse the repository at this point in the history 
Response with Pytypes
  • Loading branch information
@deedy5
deedy5 authored Apr 24, 2024
2 parents 1a40718 + 5f93bbe commit ac70f51
Show file tree
Hide file tree
Showing 6 changed files with 129 additions and 103 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "pyreqwest_impersonate"
version = "0.2.2"
version = "0.2.3"
edition = "2021"
description = "HTTP client that can impersonate web browsers, mimicking their headers and `TLS/JA3/JA4/HTTP2` fingerprints"
authors = ["deedy5"]
Expand Down
10 changes: 5 additions & 5 deletions benchmark/1_threads.csv
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name,threads,cpu_time 50k,cpu_time 5k,time 50k,time 5k
curl_cffi 0.6.2,1,4.753,1.178,6.636,2.266
httpx 0.27.0,1,2.5,1.821,3.939,3.514
pyreqwest_impersonate 0.2.1,1,1.189,0.26,2.387,0.801
requests 2.31.0,1,4.074,3.742,5.941,5.589
tls_client 1.0.1,1,4.893,1.632,5.458,2.44
curl_cffi 0.6.2,1,5.011,1.42,6.987,2.52
httpx 0.27.0,1,2.34,1.964,3.669,3.362
pyreqwest_impersonate 0.2.2,1,1.394,0.223,2.897,0.759
requests 2.31.0,1,4.244,3.231,6.144,4.974
tls_client 1.0.1,1,4.802,1.501,5.471,2.527
10 changes: 5 additions & 5 deletions benchmark/4_threads.csv
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name,threads,cpu_time 50k,cpu_time 5k,time 50k,time 5k
curl_cffi 0.6.2,4,3.732,1.111,1.347,0.689
httpx 0.27.0,4,2.026,1.452,1.386,1.249
pyreqwest_impersonate 0.2.1,4,1.061,0.388,0.934,0.622
requests 2.31.0,4,4.305,3.007,3.433,2.799
tls_client 1.0.1,4,3.5,1.101,1.246,0.708
curl_cffi 0.6.2,4,3.966,1.056,1.465,0.672
httpx 0.27.0,4,1.91,1.303,1.306,1.092
pyreqwest_impersonate 0.2.2,4,1.271,0.327,1.095,0.679
requests 2.31.0,4,4.329,2.912,3.46,2.709
tls_client 1.0.1,4,3.454,1.094,1.214,0.714
104 changes: 66 additions & 38 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ use std::time::Duration;

use pyo3::exceptions;
use pyo3::prelude::*;
use pyo3::types::PyBytes;
use pyo3::types::{PyDict, PyString};
use reqwest_impersonate::blocking::multipart;
use reqwest_impersonate::header::{HeaderMap, HeaderName, HeaderValue};
use reqwest_impersonate::impersonate::Impersonate;
Expand Down Expand Up @@ -206,6 +208,7 @@ impl Client {
/// * `PyException` - If there is an error making the request.
fn request(
&self,
py: Python,
method: &str,
url: &str,
params: Option<HashMap<String, String>>,
Expand Down Expand Up @@ -307,55 +310,66 @@ impl Client {
request_builder = request_builder.timeout(Duration::from_secs_f64(seconds));
}

// Send request
let mut resp = Python::with_gil(|py| {
py.allow_threads(|| {
request_builder.send().map_err(|e| {
PyErr::new::<exceptions::PyException, _>(format!("Error in request: {}", e))
})
// Send request | release GIL
let resp = py.allow_threads(|| {
request_builder.send().map_err(|e| {
PyErr::new::<exceptions::PyException, _>(format!("Error in request: {}", e))
})
})?;

// Response items
let mut raw: Vec<u8> = vec![];
resp.copy_to(&mut raw).map_err(|e| {
PyErr::new::<exceptions::PyIOError, _>(format!("Error in get resp.raw: {}", e))
})?;
let cookies: HashMap<String, String> = resp
.cookies()
.map(|cookie| (cookie.name().to_string(), cookie.value().to_string()))
.collect();
let cookies_dict = PyDict::new_bound(py);
for cookie in resp.cookies() {
let key = cookie.name().to_string();
let value = cookie.value().to_string();
cookies_dict.set_item(key, value)?;
}
let cookies = cookies_dict.unbind();

// Encoding from "Content-Type" header or "UTF-8"
let encoding = resp
.headers()
.get("Content-Type")
.and_then(|ct| ct.to_str().ok())
.and_then(|ct| {
ct.split(';').find_map(|param| {
let mut kv = param.splitn(2, '=');
let key = kv.next()?.trim();
let value = kv.next()?.trim();
if key.eq_ignore_ascii_case("charset") {
Some(value.to_string())
} else {
None
}
let encoding = {
let encoding_str = resp
.headers()
.get("Content-Type")
.and_then(|ct| ct.to_str().ok())
.and_then(|ct| {
ct.split(';').find_map(|param| {
let mut kv = param.splitn(2, '=');
let key = kv.next()?.trim();
let value = kv.next()?.trim();
if key.eq_ignore_ascii_case("charset") {
Some(value.to_string())
} else {
None
}
})
})
})
.unwrap_or("UTF-8".to_string());
let headers: HashMap<String, String> = resp
.headers()
.iter()
.map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string()))
.collect();
let status_code = resp.status().as_u16();
let url = resp.url().to_string();
.unwrap_or("UTF-8".to_string());
PyString::new_bound(py, &encoding_str).unbind()
};

let headers_dict = PyDict::new_bound(py);
for (key, value) in resp.headers().iter() {
let key_str = key.as_str();
let value_str = value.to_str().unwrap_or("");
headers_dict.set_item(key_str, value_str)?;
}
let headers = headers_dict.unbind();

let status_code = resp.status().as_u16().into_py(py);

let url = PyString::new_bound(py, &resp.url().to_string()).into();

let buf = resp.bytes().map_err(|e| {
PyErr::new::<exceptions::PyException, _>(format!("Error reading response bytes: {}", e))
})?;
let content = PyBytes::new_bound(py, &buf).unbind();

Ok(Response {
content,
cookies,
encoding,
headers,
raw,
status_code,
url,
})
Expand All @@ -364,13 +378,15 @@ impl Client {
fn get(
&self,
url: &str,
py: Python,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
auth: Option<(String, Option<String>)>,
auth_bearer: Option<String>,
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"GET",
url,
params,
Expand All @@ -385,6 +401,7 @@ impl Client {
}
fn head(
&self,
py: Python,
url: &str,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
Expand All @@ -393,6 +410,7 @@ impl Client {
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"HEAD",
url,
params,
Expand All @@ -407,6 +425,7 @@ impl Client {
}
fn options(
&self,
py: Python,
url: &str,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
Expand All @@ -415,6 +434,7 @@ impl Client {
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"OPTIONS",
url,
params,
Expand All @@ -429,6 +449,7 @@ impl Client {
}
fn delete(
&self,
py: Python,
url: &str,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
Expand All @@ -437,6 +458,7 @@ impl Client {
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"DELETE",
url,
params,
Expand All @@ -452,6 +474,7 @@ impl Client {

fn post(
&self,
py: Python,
url: &str,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
Expand All @@ -463,6 +486,7 @@ impl Client {
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"POST",
url,
params,
Expand All @@ -477,6 +501,7 @@ impl Client {
}
fn put(
&self,
py: Python,
url: &str,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
Expand All @@ -488,6 +513,7 @@ impl Client {
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"PUT",
url,
params,
Expand All @@ -502,6 +528,7 @@ impl Client {
}
fn patch(
&self,
py: Python,
url: &str,
params: Option<HashMap<String, String>>,
headers: Option<HashMap<String, String>>,
Expand All @@ -513,6 +540,7 @@ impl Client {
timeout: Option<f64>,
) -> PyResult<Response> {
self.request(
py,
"PATCH",
url,
params,
Expand Down
Loading

0 comments on commit ac70f51

Please sign in to comment.