Using the pythonize crate to handle data and json parameters, and return the responce as json

Cargo.toml

@@ -16,9 +16,10 @@ pyo3 = { version = "0.22", features = ["extension-module", "abi3-py38", "indexma
 anyhow = "1"
 log = "0.4"
 pyo3-log = "0.11"
-rquest = { version = "0.26", features = [ 
+rquest = { version = "0.27", features = [ 
     "cookies",
     "multipart",
+    "json",
     "socks",
     "gzip",
     "brotli",
@@ -31,6 +32,8 @@ indexmap = { version = "2", features = ["serde"] }
 tokio = { version = "1", features = ["full"] }
 html2text = "0.13"
 bytes = "1"
+pythonize = "0.22"
+serde_json = "1"
 
 [profile.release]
 codegen-units = 1

src/lib.rs

@@ -10,19 +10,20 @@ use indexmap::IndexMap;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
 use pyo3::types::{PyBytes, PyDict};
+use pythonize::depythonize;
 use rquest::boring::x509::{store::X509StoreBuilder, X509};
 use rquest::header::{HeaderMap, HeaderName, HeaderValue, COOKIE};
 use rquest::tls::Impersonate;
 use rquest::multipart;
 use rquest::redirect::Policy;
 use rquest::Method;
+use serde_json::Value;
 use tokio::runtime::{self, Runtime};
 
 mod response;
 use response::Response;
 
 mod utils;
-use utils::{json_dumps, url_encode};
 
 // Tokio global one-thread runtime
 static RUNTIME: LazyLock<Runtime> = LazyLock::new(|| {
@@ -120,9 +121,7 @@ impl Client {
         }
 
         // Client builder
-        let mut client_builder = rquest::Client::builder()
-            .enable_ech_grease()
-            .permute_extensions();
+        let mut client_builder = rquest::Client::builder();
 
         // Impersonate
         if let Some(impersonation_type) = impersonate {
@@ -270,14 +269,8 @@ impl Client {
         }?;
         let params = params.or(self.params.clone());
         let cookies = cookies.or(self.cookies.clone());
-        // Converts 'data' (if any) into a URL-encoded string for sending the data as `application/x-www-form-urlencoded` content type.
-        let data_str = data
-            .map(|data| url_encode(py, &data.as_unbound()))
-            .transpose()?;
-        // Converts 'json' (if any) into a JSON string for sending the data as `application/json` content type.
-        let json_str = json
-            .map(|pydict| json_dumps(py, &pydict.as_unbound()))
-            .transpose()?;
+        let data_value: Option<Value> = data.map(|data| depythonize(&data)).transpose()?;
+        let json_value: Option<Value> = json.map(|json| depythonize(&json)).transpose()?;
         let auth = auth.or(self.auth.clone());
         let auth_bearer = auth_bearer.or(self.auth_bearer.clone());
         if auth.is_some() && auth_bearer.is_some() {
@@ -325,16 +318,12 @@ impl Client {
                     request_builder = request_builder.body(content);
                 }
                 // Data
-                if let Some(url_encoded_data) = data_str {
-                    request_builder = request_builder
-                        .header("Content-Type", "application/x-www-form-urlencoded")
-                        .body(url_encoded_data);
+                if let Some(form_data) = data_value {
+                    request_builder = request_builder.form(&form_data);
                 }
                 // Json
-                if let Some(json_str) = json_str {
-                    request_builder = request_builder
-                        .header("Content-Type", "application/json")
-                        .body(json_str);
+                if let Some(json_data) = json_value {
+                    request_builder = request_builder.json(&json_data);
                 }
                 // Files
                 if let Some(files) = files {

src/response.rs

@@ -1,4 +1,4 @@
-use crate::utils::{get_encoding_from_content, get_encoding_from_headers, json_loads};
+use crate::utils::{get_encoding_from_content, get_encoding_from_headers};
 use ahash::RandomState;
 use anyhow::{anyhow, Result};
 use encoding_rs::Encoding;
@@ -8,6 +8,8 @@ use html2text::{
 };
 use indexmap::IndexMap;
 use pyo3::{prelude::*, types::PyBytes};
+use pythonize::pythonize;
+use serde_json::from_slice;
 
 /// A struct representing an HTTP response.
 ///
@@ -78,7 +80,8 @@ impl Response {
     }
 
     fn json(&mut self, py: Python) -> Result<PyObject> {
-        let result = json_loads(py, &self.content)?;
+        let json_value: serde_json::Value = from_slice(self.content.as_bytes(py))?;
+        let result = pythonize(py, &json_value).unwrap().into();
         Ok(result)
     }
 

src/utils.rs

@@ -1,62 +1,7 @@
 use std::cmp::min;
 
 use ahash::RandomState;
-use anyhow::Result;
 use indexmap::IndexMap;
-use pyo3::prelude::*;
-use pyo3::sync::GILOnceCell;
-use pyo3::types::{PyBool, PyBytes, PyDict};
-
-static JSON_DUMPS: GILOnceCell<Py<PyAny>> = GILOnceCell::new();
-static JSON_LOADS: GILOnceCell<Py<PyAny>> = GILOnceCell::new();
-static URLLIB_PARSE_URLENCODE: GILOnceCell<Py<PyAny>> = GILOnceCell::new();
-
-/// python json.dumps
-pub fn json_dumps(py: Python<'_>, pydict: &Py<PyDict>) -> Result<String> {
-    let json_dumps = JSON_DUMPS
-        .get_or_init(py, || {
-            py.import_bound("json")
-                .unwrap()
-                .getattr("dumps")
-                .unwrap()
-                .unbind()
-        })
-        .bind(py);
-    let result = json_dumps.call1((pydict,))?.extract::<String>()?;
-    Ok(result)
-}
-
-/// python json.loads
-pub fn json_loads(py: Python<'_>, content: &Py<PyBytes>) -> Result<PyObject> {
-    let json_loads = JSON_LOADS
-        .get_or_init(py, || {
-            py.import_bound("json")
-                .unwrap()
-                .getattr("loads")
-                .unwrap()
-                .unbind()
-        })
-        .bind(py);
-    let result = json_loads.call1((content,))?.extract::<PyObject>()?;
-    Ok(result)
-}
-
-/// python urllib.parse.urlencode
-pub fn url_encode(py: Python, pydict: &Py<PyDict>) -> Result<String> {
-    let urlencode = URLLIB_PARSE_URLENCODE
-        .get_or_init(py, || {
-            py.import_bound("urllib.parse")
-                .unwrap()
-                .getattr("urlencode")
-                .unwrap()
-                .unbind()
-        })
-        .bind(py);
-    let result: String = urlencode
-        .call1((pydict, ("doseq", py.get_type_bound::<PyBool>().call1(())?)))?
-        .extract()?;
-    Ok(result)
-}
 
 /// Get encoding from the "Content-Type" header
 pub fn get_encoding_from_headers(

tests/test_client.py

@@ -4,7 +4,7 @@
 import primp  # type: ignore
 
 
-def retry(max_retries=5, delay=1):
+def retry(max_retries=3, delay=1):
     def decorator(func):
         def wrapper(*args, **kwargs):
             for attempt in range(max_retries):
@@ -147,32 +147,6 @@ def test_client_post_data():
     assert json_data["form"] == {"key1": "value1", "key2": "value2"}
 
 
-@retry()
-def test_client_post_data2():
-    client = primp.Client()
-    auth_bearer = "bearerXXXXXXXXXXXXXXXXXXXX"
-    headers = {"X-Test": "test"}
-    cookies = {"ccc": "ddd", "cccc": "dddd"}
-    params = {"x": "aaa", "y": "bbb"}
-    data = {"key1": "value1", "key2": ["value2_1", "value2_2"]}
-    response = client.post(
-        "https://httpbin.org/anything",
-        auth_bearer=auth_bearer,
-        headers=headers,
-        cookies=cookies,
-        params=params,
-        data=data,
-    )
-    assert response.status_code == 200
-    json_data = response.json()
-    assert json_data["method"] == "POST"
-    assert json_data["headers"]["X-Test"] == "test"
-    assert json_data["headers"]["Cookie"] == "ccc=ddd; cccc=dddd"
-    assert json_data["headers"]["Authorization"] == "Bearer bearerXXXXXXXXXXXXXXXXXXXX"
-    assert json_data["args"] == {"x": "aaa", "y": "bbb"}
-    assert json_data["form"] == {"key1": "value1", "key2": ["value2_1", "value2_2"]}
-
-
 @retry()
 def test_client_post_json():
     client = primp.Client()
@@ -226,16 +200,14 @@ def test_client_post_files():
 
 
 @retry()
-def test_client_impersonate_chrome126():
+def test_client_impersonate_chrome130():
     client = primp.Client(
-        impersonate="chrome_126",
+        impersonate="chrome_130",
     )
-    response = client.get("https://tls.peet.ws/api/all")
+    # response = client.get("https://tls.peet.ws/api/all")
+    response = client.get("https://tls.http.rw/api/clean")
     assert response.status_code == 200
     json_data = response.json()
-    assert json_data["http_version"] == "h2"
-    assert json_data["tls"]["ja4"].startswith("t13d")
-    assert (
-        json_data["http2"]["akamai_fingerprint_hash"]
-        == "90224459f8bf70b7d0a8797eb916dbc9"
-    )
+    assert json_data["ja4"] == "t13d1516h2_8daaf6152771_b1ff8ab2d16f"
+    assert json_data["akamai_hash"] == "90224459f8bf70b7d0a8797eb916dbc9"
+    assert json_data["peetprint_hash"] == "b8ce945a4d9a7a9b5b6132e3658fe033"

tests/test_defs.py

@@ -4,7 +4,7 @@
 import primp  # type: ignore
 
 
-def retry(max_retries=5, delay=1):
+def retry(max_retries=3, delay=1):
     def decorator(func):
         def wrapper(*args, **kwargs):
             for attempt in range(max_retries):
@@ -171,31 +171,6 @@ def test_post_data():
     assert json_data["form"] == {"key1": "value1", "key2": "value2"}
 
 
-@retry()
-def test_post_data2():
-    auth_bearer = "bearerXXXXXXXXXXXXXXXXXXXX"
-    headers = {"X-Test": "test"}
-    cookies = {"ccc": "ddd", "cccc": "dddd"}
-    params = {"x": "aaa", "y": "bbb"}
-    data = {"key1": "value1", "key2": ["value2_1", "value2_2"]}
-    response = primp.post(
-        "https://httpbin.org/anything",
-        auth_bearer=auth_bearer,
-        headers=headers,
-        cookies=cookies,
-        params=params,
-        data=data,
-    )
-    assert response.status_code == 200
-    json_data = response.json()
-    assert json_data["method"] == "POST"
-    assert json_data["headers"]["X-Test"] == "test"
-    assert json_data["headers"]["Cookie"] == "ccc=ddd; cccc=dddd"
-    assert json_data["headers"]["Authorization"] == "Bearer bearerXXXXXXXXXXXXXXXXXXXX"
-    assert json_data["args"] == {"x": "aaa", "y": "bbb"}
-    assert json_data["form"] == {"key1": "value1", "key2": ["value2_1", "value2_2"]}
-
-
 @retry()
 def test_post_json():
     auth_bearer = "bearerXXXXXXXXXXXXXXXXXXXX"
@@ -297,16 +272,14 @@ def test_put():
 
 
 @retry()
-def test_get_impersonate_chrome126():
+def test_get_impersonate_chrome130():
     response = primp.get(
-        "https://tls.peet.ws/api/all",
-        impersonate="chrome_126",
+        # "https://tls.peet.ws/api/clean",
+        "https://tls.http.rw/api/clean",
+        impersonate="chrome_130",
     )
     assert response.status_code == 200
     json_data = response.json()
-    assert json_data["http_version"] == "h2"
-    assert json_data["tls"]["ja4"].startswith("t13d")
-    assert (
-        json_data["http2"]["akamai_fingerprint_hash"]
-        == "90224459f8bf70b7d0a8797eb916dbc9"
-    )
+    assert json_data["ja4"] == "t13d1516h2_8daaf6152771_b1ff8ab2d16f"
+    assert json_data["akamai_hash"] == "90224459f8bf70b7d0a8797eb916dbc9"
+    assert json_data["peetprint_hash"] == "b8ce945a4d9a7a9b5b6132e3658fe033"