add steal_forwarded_for config in http interception

g3proxy/doc/conf.py

@@ -9,7 +9,7 @@
 project = 'g3proxy'
 copyright = '2024, Zhang Jingqiang'
 author = 'Zhang Jingqiang'
-release = '1.9.0'
+release = '1.9.2'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
@@ -21,14 +21,12 @@
 templates_path = ['_templates']
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
-
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
 html_theme = 'alabaster'
 html_static_path = ['_static']
 
-
 # -- Custom Options ----------------------------------------------------------
 
 # Set the master document, which contains the root toctree directive.

g3proxy/doc/configuration/servers/http_proxy.rst

@@ -263,4 +263,9 @@ steal_forwarded_for
 
 Set if we should delete the *Forwarded* and *X-Forwarded-For* headers from the client's request.
 
+.. note::
+
+  If you want to remove those headers from https traffic, you need to enable TLS interception and also set this in
+  auditor's :ref:`h1 interception <conf_auditor_h1_interception>` config.
+
 **default**: false

g3proxy/doc/configuration/values/dpi.rst

@@ -1,4 +1,3 @@
-
 .. _configure_dpi_value_types:
 
 ***
@@ -550,6 +549,16 @@ The keys are:
 
   **default**: 8192
 
+* steal_forwarded_for
+
+  **optional**, **type**: bool
+
+  Set if we should delete the *Forwarded* and *X-Forwarded-For* headers from the client's intercepted transparent request.
+
+  **default**: false
+
+  .. versionadded:: 1.9.2
+
 .. _conf_value_dpi_h2_interception:
 
 h2 interception

g3proxy/src/inspect/http/v1/pipeline/request.rs

@@ -156,7 +156,11 @@ where
 
                 match tokio::time::timeout(
                     http_config.req_head_recv_timeout,
-                    HttpTransparentRequest::parse(&mut io.clt_r, http_config.req_head_max_size),
+                    HttpTransparentRequest::parse(
+                        &mut io.clt_r,
+                        http_config.req_head_max_size,
+                        http_config.steal_forwarded_for,
+                    ),
                 )
                 .await
                 {

g3proxy/src/inspect/http/v2/stream.rs

@@ -30,13 +30,17 @@ use crate::config::server::ServerConfig;
 use crate::inspect::StreamInspectContext;
 
 pub(super) async fn transfer<SC>(
-    clt_req: Request<RecvStream>,
+    mut clt_req: Request<RecvStream>,
     clt_send_rsp: SendResponse<Bytes>,
     h2s: SendRequest<Bytes>,
     ctx: StreamInspectContext<SC>,
 ) where
     SC: ServerConfig + Send + Sync + 'static,
 {
+    if ctx.h1_interception().steal_forwarded_for {
+        clt_req.headers_mut().remove(http::header::FORWARDED);
+        clt_req.headers_mut().remove("x-forwarded-for");
+    }
     let clt_stream_id = clt_send_rsp.stream_id();
     if clt_req.method().eq(&Method::CONNECT) {
         if let Some(protocol) = clt_req.extensions().get::<Protocol>() {

g3proxy/src/serve/http_proxy/task/protocol/request.rs

@@ -42,7 +42,7 @@ where
         reader: &mut HttpClientReader<CDR>,
         sender: mpsc::Sender<Option<HttpClientReader<CDR>>>,
         max_header_size: usize,
-        steal_forwarder_for: bool,
+        steal_forwarded_for: bool,
         allow_custom_host: bool,
         version: &mut Version,
     ) -> Result<(Self, bool), HttpRequestParseError> {
@@ -57,7 +57,7 @@ where
                         return req.parse_header_connection(header);
                     }
                     "forwarded" | "x-forwarded-for" => {
-                        if steal_forwarder_for {
+                        if steal_forwarded_for {
                             return Ok(());
                         }
                     }

lib/g3-dpi/src/config/http.rs

@@ -25,6 +25,7 @@ pub struct H1InterceptionConfig {
     pub req_head_max_size: usize,
     pub rsp_head_max_size: usize,
     pub body_line_max_len: usize,
+    pub steal_forwarded_for: bool,
 }
 
 impl Default for H1InterceptionConfig {
@@ -37,6 +38,7 @@ impl Default for H1InterceptionConfig {
             req_head_max_size: 65536,
             rsp_head_max_size: 65536,
             body_line_max_len: 8192,
+            steal_forwarded_for: false,
         }
     }
 }

lib/g3-http/src/server/transparent.rs

@@ -32,6 +32,7 @@ pub struct HttpTransparentRequest {
     pub method: Method,
     pub version: Version,
     pub uri: Uri,
+    steal_forwarded_for: bool,
     pub end_to_end_headers: HttpHeaderMap,
     pub hop_by_hop_headers: HttpHeaderMap,
     /// the port may be 0
@@ -55,6 +56,7 @@ impl HttpTransparentRequest {
             version,
             method,
             uri,
+            steal_forwarded_for: false,
             end_to_end_headers: HttpHeaderMap::default(),
             hop_by_hop_headers: HttpHeaderMap::default(),
             host: None,
@@ -82,6 +84,7 @@ impl HttpTransparentRequest {
             version: adapted.version,
             method: adapted.method,
             uri: adapted.uri,
+            steal_forwarded_for: false,
             end_to_end_headers: adapted.headers,
             hop_by_hop_headers,
             host: None,
@@ -139,6 +142,7 @@ impl HttpTransparentRequest {
     pub async fn parse<R>(
         reader: &mut R,
         max_header_size: usize,
+        steal_forwarded_for: bool,
     ) -> Result<(Self, Bytes), HttpRequestParseError>
     where
         R: AsyncBufRead + Unpin,
@@ -165,6 +169,7 @@ impl HttpTransparentRequest {
             Version::HTTP_11 => req.keep_alive = true,
             _ => {}
         }
+        req.steal_forwarded_for = steal_forwarded_for;
 
         loop {
             let header_size = head_bytes.len();
@@ -357,6 +362,11 @@ impl HttpTransparentRequest {
             "te" | "proxy-authorization" => {
                 return self.insert_hop_by_hop_header(name, &header);
             }
+            "forwarded" | "x-forwarded-for" => {
+                if self.steal_forwarded_for {
+                    return Ok(());
+                }
+            }
             // ignore "expect"
             _ => {}
         }
@@ -506,15 +516,15 @@ mod tests {
         let stream = tokio_stream::iter(vec![Result::Ok(Bytes::from_static(content))]);
         let stream = StreamReader::new(stream);
         let mut buf_stream = BufReader::new(stream);
-        let (request, data) = HttpTransparentRequest::parse(&mut buf_stream, 4096)
+        let (request, data) = HttpTransparentRequest::parse(&mut buf_stream, 4096, false)
             .await
             .unwrap();
         assert_eq!(data.as_ref(), content.as_slice());
         assert_eq!(request.method, &Method::GET);
         assert!(request.keep_alive());
         assert!(request.body_type().is_none());
 
-        let result = HttpTransparentRequest::parse(&mut buf_stream, 4096).await;
+        let result = HttpTransparentRequest::parse(&mut buf_stream, 4096, false).await;
         assert!(result.is_err());
     }
 
@@ -528,7 +538,7 @@ mod tests {
         let stream = tokio_stream::iter(vec![Result::Ok(Bytes::from_static(content))]);
         let stream = StreamReader::new(stream);
         let mut buf_stream = BufReader::new(stream);
-        let (request, data) = HttpTransparentRequest::parse(&mut buf_stream, 4096)
+        let (request, data) = HttpTransparentRequest::parse(&mut buf_stream, 4096, false)
             .await
             .unwrap();
         assert_eq!(data.as_ref(), content.as_slice());

lib/g3-yaml/src/value/dpi/http.rs

@@ -58,6 +58,10 @@ pub fn as_h1_interception_config(value: &Yaml) -> anyhow::Result<H1InterceptionC
                     .context(format!("invalid usize value for key {k}"))?;
                 Ok(())
             }
+            "steal_forwarded_for" => {
+                config.steal_forwarded_for = crate::value::as_bool(v)?;
+                Ok(())
+            }
             _ => Err(anyhow!("invalid key {k}")),
         })?;