g3proxy: add support for periodic task logs

g3proxy/doc/configuration/servers/http_proxy.rst

@@ -23,6 +23,9 @@ The following common keys are supported:
 * :ref:`tcp_misc_opts <conf_server_common_tcp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 The auth scheme supported by the server is determined by the type of the specified user group.

g3proxy/doc/configuration/servers/http_rproxy.rst

@@ -20,6 +20,9 @@ The following common keys are supported:
 * :ref:`tcp_misc_opts <conf_server_common_tcp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 The auth scheme supported by the server is determined by the type of the specified user group.

g3proxy/doc/configuration/servers/index.rst

@@ -303,6 +303,50 @@ The task will be closed if the idle check return IDLE the times as this value.
 
 **default**: 1
 
+.. _conf_server_common_flush_task_log_on_created:
+
+flush_task_log_on_created
+-------------------------
+
+**optional**, **type**: bool
+
+Log when task get created.
+
+**default**: false
+
+.. versionadded:: 1.11.0
+
+.. _conf_server_common_flush_task_log_on_connected:
+
+flush_task_log_on_connected
+---------------------------
+
+**optional**, **type**: bool
+
+Log when upstream connected.
+
+**default**: false
+
+.. versionadded:: 1.11.0
+
+.. _conf_server_common_task_log_flush_interval:
+
+task_log_flush_interval
+-----------------------
+
+**optional**, **type**: :ref:`humanize duration <conf_value_humanize_duration>`
+
+Enable periodic task log and set the flush interval.
+
+.. note::
+
+  There will be no periodic task log if protocol inspection is enabled, as intercept and inspect logs will be available
+  in this case.
+
+**default**: not set
+
+.. versionadded:: 1.11.0
+
 .. _conf_server_common_extra_metrics_tags:
 
 extra_metrics_tags

g3proxy/doc/configuration/servers/sni_proxy.rst

@@ -18,6 +18,9 @@ The following common keys are supported:
 * :ref:`tcp_misc_opts <conf_server_common_tcp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 listen

g3proxy/doc/configuration/servers/socks_proxy.rst

@@ -26,6 +26,9 @@ The following common keys are supported:
 * :ref:`udp_misc_opts <conf_server_common_udp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 The auth type supported by the server is determined by the type of the specified user group.

g3proxy/doc/configuration/servers/tcp_stream.rst

@@ -18,6 +18,9 @@ The following common keys are supported:
 * :ref:`tcp_misc_opts <conf_server_common_tcp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 listen

g3proxy/doc/configuration/servers/tcp_tproxy.rst

@@ -22,6 +22,9 @@ The following common keys are supported:
 * :ref:`tcp_misc_opts <conf_server_common_tcp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 listen

g3proxy/doc/configuration/servers/tls_stream.rst

@@ -23,6 +23,9 @@ The following common keys are supported:
 * :ref:`tcp_misc_opts <conf_server_common_tcp_misc_opts>`
 * :ref:`task_idle_check_duration <conf_server_common_task_idle_check_duration>`
 * :ref:`task_idle_max_count <conf_server_common_task_idle_max_count>`
+* :ref:`flush_task_log_on_created <conf_server_common_flush_task_log_on_created>`
+* :ref:`flush_task_log_on_connected <conf_server_common_flush_task_log_on_connected>`
+* :ref:`task_log_flush_interval <conf_server_common_task_log_flush_interval>`
 * :ref:`extra_metrics_tags <conf_server_common_extra_metrics_tags>`
 
 listen

g3proxy/doc/log/task/http_forward.rst

@@ -19,7 +19,7 @@ Show the time spent from the receive of the http request header to the creation
 reuse_connection
 ----------------
 
-**required**, **type**: bool
+**optional**, **type**: bool
 
 Show if this task reuse old remote connection.
 

g3proxy/doc/log/task/index.rst

@@ -41,6 +41,24 @@ UUID of the task.
 
 The *task_id* will appear in other logs such as escape log if they have any association with this task.
 
+task_event
+----------
+
+**optional**, **type**: string
+
+Show the event that trigger this log.
+
+The event can be
+
+  - created
+  - connected
+  - periodic
+  - finished
+
+This field can be omitted if the value is *finished*.
+
+.. versionadded:: 1.11.0
+
 stage
 -----
 
@@ -108,7 +126,7 @@ The selected escaper name.
 reason
 ------
 
-**required**, **type**: enum string
+**optional**, **type**: enum string
 
 The brief reason why the task ends.
 
@@ -137,9 +155,9 @@ and the remote channel have been established. The value may be empty if the task
 total_time
 ----------
 
-**required**, **type**: time duration string
+**optional**, **type**: time duration string
 
-Show the time from the creation of the task to the end of the task.
+Show the time from the creation of the task to the time of this log.
 
 Sub Types
 =========

g3proxy/src/config/server/http_proxy.rs

@@ -83,6 +83,9 @@ pub(crate) struct HttpProxyServerConfig {
     pub(crate) timeout: HttpProxyServerTimeoutConfig,
     pub(crate) task_idle_check_duration: Duration,
     pub(crate) task_idle_max_count: i32,
+    pub(crate) flush_task_log_on_created: bool,
+    pub(crate) flush_task_log_on_connected: bool,
+    pub(crate) task_log_flush_interval: Option<Duration>,
     pub(crate) tcp_copy: LimitedCopyConfig,
     pub(crate) tcp_misc_opts: TcpMiscSockOpts,
     pub(crate) req_hdr_max_size: usize,
@@ -126,6 +129,9 @@ impl HttpProxyServerConfig {
             timeout: HttpProxyServerTimeoutConfig::default(),
             task_idle_check_duration: IDLE_CHECK_DEFAULT_DURATION,
             task_idle_max_count: 1,
+            flush_task_log_on_created: false,
+            flush_task_log_on_connected: false,
+            task_log_flush_interval: None,
             tcp_copy: Default::default(),
             tcp_misc_opts: Default::default(),
             req_hdr_max_size: 65536, // 64KiB
@@ -292,6 +298,20 @@ impl HttpProxyServerConfig {
                     g3_yaml::value::as_i32(v).context(format!("invalid i32 value for key {k}"))?;
                 Ok(())
             }
+            "flush_task_log_on_created" => {
+                self.flush_task_log_on_created = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "flush_task_log_on_connected" => {
+                self.flush_task_log_on_connected = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "task_log_flush_interval" => {
+                let interval = g3_yaml::humanize::as_duration(v)
+                    .context(format!("invalid humanize duration value for key {k}"))?;
+                self.task_log_flush_interval = Some(interval);
+                Ok(())
+            }
             "req_header_recv_timeout" => {
                 self.timeout.recv_req_header = g3_yaml::humanize::as_duration(v)
                     .context(format!("invalid humanize duration value for key {k}"))?;

g3proxy/src/config/server/http_rproxy/mod.rs

@@ -76,6 +76,9 @@ pub(crate) struct HttpRProxyServerConfig {
     pub(crate) timeout: HttpRProxyServerTimeoutConfig,
     pub(crate) task_idle_check_duration: Duration,
     pub(crate) task_idle_max_count: i32,
+    pub(crate) flush_task_log_on_created: bool,
+    pub(crate) flush_task_log_on_connected: bool,
+    pub(crate) task_log_flush_interval: Option<Duration>,
     pub(crate) tcp_copy: LimitedCopyConfig,
     pub(crate) tcp_misc_opts: TcpMiscSockOpts,
     pub(crate) req_hdr_max_size: usize,
@@ -113,6 +116,9 @@ impl HttpRProxyServerConfig {
             timeout: HttpRProxyServerTimeoutConfig::default(),
             task_idle_check_duration: IDLE_CHECK_DEFAULT_DURATION,
             task_idle_max_count: 1,
+            flush_task_log_on_created: false,
+            flush_task_log_on_connected: false,
+            task_log_flush_interval: None,
             tcp_copy: Default::default(),
             tcp_misc_opts: Default::default(),
             req_hdr_max_size: 65536, // 64KiB
@@ -232,6 +238,20 @@ impl HttpRProxyServerConfig {
                     g3_yaml::value::as_i32(v).context(format!("invalid i32 value for key {k}"))?;
                 Ok(())
             }
+            "flush_task_log_on_created" => {
+                self.flush_task_log_on_created = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "flush_task_log_on_connected" => {
+                self.flush_task_log_on_connected = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "task_log_flush_interval" => {
+                let interval = g3_yaml::humanize::as_duration(v)
+                    .context(format!("invalid humanize duration value for key {k}"))?;
+                self.task_log_flush_interval = Some(interval);
+                Ok(())
+            }
             "req_header_recv_timeout" => {
                 self.timeout.recv_req_header = g3_yaml::humanize::as_duration(v)
                     .context(format!("invalid humanize duration value for key {k}"))?;

g3proxy/src/config/server/sni_proxy/mod.rs

@@ -49,6 +49,9 @@ pub(crate) struct SniProxyServerConfig {
     pub(crate) tcp_sock_speed_limit: TcpSockSpeedLimitConfig,
     pub(crate) task_idle_check_duration: Duration,
     pub(crate) task_idle_max_count: i32,
+    pub(crate) flush_task_log_on_created: bool,
+    pub(crate) flush_task_log_on_connected: bool,
+    pub(crate) task_log_flush_interval: Option<Duration>,
     pub(crate) tcp_copy: LimitedCopyConfig,
     pub(crate) tcp_misc_opts: TcpMiscSockOpts,
     pub(crate) tls_max_client_hello_size: u32,
@@ -75,6 +78,9 @@ impl SniProxyServerConfig {
             tcp_sock_speed_limit: TcpSockSpeedLimitConfig::default(),
             task_idle_check_duration: Duration::from_secs(300),
             task_idle_max_count: 1,
+            flush_task_log_on_created: false,
+            flush_task_log_on_connected: false,
+            task_log_flush_interval: None,
             tcp_copy: Default::default(),
             tcp_misc_opts: Default::default(),
             tls_max_client_hello_size: 1 << 16,
@@ -179,6 +185,20 @@ impl SniProxyServerConfig {
                     g3_yaml::value::as_i32(v).context(format!("invalid i32 value for key {k}"))?;
                 Ok(())
             }
+            "flush_task_log_on_created" => {
+                self.flush_task_log_on_created = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "flush_task_log_on_connected" => {
+                self.flush_task_log_on_connected = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "task_log_flush_interval" => {
+                let interval = g3_yaml::humanize::as_duration(v)
+                    .context(format!("invalid humanize duration value for key {k}"))?;
+                self.task_log_flush_interval = Some(interval);
+                Ok(())
+            }
             "request_wait_timeout" => {
                 self.request_wait_timeout = g3_yaml::humanize::as_duration(v)
                     .context(format!("invalid humanize duration value for key {k}"))?;

g3proxy/src/config/server/socks_proxy.rs

@@ -82,6 +82,9 @@ pub(crate) struct SocksProxyServerConfig {
     pub(crate) timeout: SocksProxyServerTimeoutConfig,
     pub(crate) task_idle_check_duration: Duration,
     pub(crate) task_idle_max_count: i32,
+    pub(crate) flush_task_log_on_created: bool,
+    pub(crate) flush_task_log_on_connected: bool,
+    pub(crate) task_log_flush_interval: Option<Duration>,
     pub(crate) tcp_copy: LimitedCopyConfig,
     pub(crate) udp_relay: LimitedUdpRelayConfig,
     pub(crate) tcp_misc_opts: TcpMiscSockOpts,
@@ -114,6 +117,9 @@ impl SocksProxyServerConfig {
             timeout: SocksProxyServerTimeoutConfig::default(),
             task_idle_check_duration: IDLE_CHECK_DEFAULT_DURATION,
             task_idle_max_count: 1,
+            flush_task_log_on_created: false,
+            flush_task_log_on_connected: false,
+            task_log_flush_interval: None,
             tcp_copy: Default::default(),
             udp_relay: Default::default(),
             tcp_misc_opts: Default::default(),
@@ -295,6 +301,20 @@ impl SocksProxyServerConfig {
                     g3_yaml::value::as_i32(v).context(format!("invalid i32 value for key {k}"))?;
                 Ok(())
             }
+            "flush_task_log_on_created" => {
+                self.flush_task_log_on_created = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "flush_task_log_on_connected" => {
+                self.flush_task_log_on_connected = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "task_log_flush_interval" => {
+                let interval = g3_yaml::humanize::as_duration(v)
+                    .context(format!("invalid humanize duration value for key {k}"))?;
+                self.task_log_flush_interval = Some(interval);
+                Ok(())
+            }
             "transmute_udp_echo_ip" | "auto_reply_local_ip_map" => {
                 if let Yaml::Hash(_) = v {
                     let map = g3_yaml::value::as_hashmap(

g3proxy/src/config/server/tcp_stream.rs

@@ -52,6 +52,9 @@ pub(crate) struct TcpStreamServerConfig {
     pub(crate) tcp_sock_speed_limit: TcpSockSpeedLimitConfig,
     pub(crate) task_idle_check_duration: Duration,
     pub(crate) task_idle_max_count: i32,
+    pub(crate) flush_task_log_on_created: bool,
+    pub(crate) flush_task_log_on_connected: bool,
+    pub(crate) task_log_flush_interval: Option<Duration>,
     pub(crate) tcp_copy: LimitedCopyConfig,
     pub(crate) tcp_misc_opts: TcpMiscSockOpts,
     pub(crate) extra_metrics_tags: Option<Arc<StaticMetricsTags>>,
@@ -75,6 +78,9 @@ impl TcpStreamServerConfig {
             tcp_sock_speed_limit: TcpSockSpeedLimitConfig::default(),
             task_idle_check_duration: Duration::from_secs(300),
             task_idle_max_count: 1,
+            flush_task_log_on_created: false,
+            flush_task_log_on_connected: false,
+            task_log_flush_interval: None,
             tcp_copy: Default::default(),
             tcp_misc_opts: Default::default(),
             extra_metrics_tags: None,
@@ -205,6 +211,20 @@ impl TcpStreamServerConfig {
                     g3_yaml::value::as_i32(v).context(format!("invalid i32 value for key {k}"))?;
                 Ok(())
             }
+            "flush_task_log_on_created" => {
+                self.flush_task_log_on_created = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "flush_task_log_on_connected" => {
+                self.flush_task_log_on_connected = g3_yaml::value::as_bool(v)?;
+                Ok(())
+            }
+            "task_log_flush_interval" => {
+                let interval = g3_yaml::humanize::as_duration(v)
+                    .context(format!("invalid humanize duration value for key {k}"))?;
+                self.task_log_flush_interval = Some(interval);
+                Ok(())
+            }
             _ => Err(anyhow!("invalid key {k}")),
         }
     }