bytedance · zh-jq · Nov 22, 2024 · Nov 22, 2024 · Nov 22, 2024
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -89,6 +89,9 @@ jobs:
       - name: Install binutils
         run: |
           cargo install cargo-binutils
+      - name: Install nginx
+        run: |
+          sudo apt-get install nginx-light
       - name: Install and run c-icap
         run: |
           sudo apt-get install c-icap

diff --git a/g3proxy/ci/python3+curl/test_httpbin.py b/g3proxy/ci/python3+curl/test_httpbin.py
@@ -26,6 +26,7 @@ def setUp(self):
         self.buffer = BytesIO()
 
         self.c = pycurl.Curl()
+        self.c.setopt(pycurl.HTTP_VERSION,  pycurl.CURL_HTTP_VERSION_1_1)
         self.c.setopt(pycurl.WRITEFUNCTION, self.buffer.write)
         self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON])
         if target_ca_cert is not None:

diff --git a/g3proxy/ci/python3+curl/test_httpbin_h2.py b/g3proxy/ci/python3+curl/test_httpbin_h2.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+import unittest
+import base64
+from io import BytesIO
+from urllib.parse import urlencode
+
+import pycurl
+
+target_site = 'https://httpbin.org'
+target_ca_cert = None
+target_proxy = None
+proxy_ca_cert = None
+local_resolve = None
+
+ACCEPT_JSON = 'Accept: application/json'
+ACCEPT_HTML = 'Accept: text/html'
+
+
+class TestHttpBin(unittest.TestCase):
+    def setUp(self):
+        self.buffer = BytesIO()
+
+        self.c = pycurl.Curl()
+        self.c.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2)
+        self.c.setopt(pycurl.WRITEFUNCTION, self.buffer.write)
+        self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON])
+        if target_ca_cert is not None:
+            self.c.setopt(pycurl.CAINFO, target_ca_cert)
+        if target_proxy is not None:
+            self.c.setopt(pycurl.PROXY, target_proxy)
+            if proxy_ca_cert is not None:
+                self.c.setopt(pycurl.PROXY_CAINFO, proxy_ca_cert)
+        if local_resolve is not None:
+            self.c.setopt(pycurl.RESOLVE, [local_resolve])
+
+    def tearDown(self):
+        self.c.close()
+
+    def set_url_and_request_target(self, path: str):
+        self.c.setopt(pycurl.URL, f"{target_site}{path}")
+
+    def test_simple_get(self):
+        self.set_url_and_request_target('/get')
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200)
+
+    def test_basic_auth_get(self):
+        self.set_url_and_request_target('/basic-auth/name/pass')
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 401)
+
+        auth_header = "Authorization: Basic {}".format(base64.standard_b64encode(b'name:pass').decode('utf-8'))
+        self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON, auth_header])
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200)
+
+        auth_header = "Authorization: Basic {}".format(base64.standard_b64encode(b'name:pas').decode('utf-8'))
+        self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON, auth_header])
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 401)
+
+    def test_base64_decode(self):
+        self.set_url_and_request_target('/base64/SFRUUEJJTiBpcyBhd2Vzb21l')
+        self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_HTML])
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200)
+        self.assertEqual(self.buffer.getvalue(), b"HTTPBIN is awesome")
+
+    def test_post_small(self):
+        data = "Content to post"
+
+        self.set_url_and_request_target('/post')
+        self.c.setopt(pycurl.POSTFIELDS, data)
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200)
+
+    def test_post_large(self):
+        post_data = {'data': "Content to post" * 1024 * 100}
+        post_fields = urlencode(post_data)
+
+        # curl won't send Expect with HTTP2.0
+        self.set_url_and_request_target('/post')
+        self.c.setopt(pycurl.POSTFIELDS, post_fields)
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200)
+
+    def test_put_file(self):
+        self.set_url_and_request_target('/put')
+        self.c.setopt(pycurl.UPLOAD, 1)
+        file = open(__file__)
+        self.c.setopt(pycurl.READDATA, file)
+        self.c.perform()
+        self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200)
+        file.close()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--proxy', '-x', nargs='?', help='Proxy URL')
+    parser.add_argument('--site', '-T', nargs='?', help='Target Site', default=target_site)
+    parser.add_argument('--ca-cert', nargs='?', help='CA Cert')
+    parser.add_argument('--proxy-ca-cert', nargs='?', help='Proxy CA Cert')
+    parser.add_argument('--resolve', nargs='?', help='Local Resolve Record for curl')
+
+    (args, left_args) = parser.parse_known_args()
+
+    if args.ca_cert is not None:
+        target_ca_cert = args.ca_cert
+    if args.proxy is not None:
+        target_proxy = args.proxy
+    if args.proxy_ca_cert is not None:
+        proxy_ca_cert = args.proxy_ca_cert
+    if args.resolve is not None:
+        local_resolve = args.resolve
+    target_site = args.site
+
+    left_args.insert(0, sys.argv[0])
+
+    unittest.main(argv=left_args)
diff --git a/scripts/coverage/g3proxy.sh b/scripts/coverage/g3proxy.sh
@@ -22,8 +22,10 @@ all_objects=$(find target/debug/deps/ -type f -perm /111 -not -name "*.so" | awk
 # generate resource files
 "${SCRIPTS_DIR}"/g3proxy/mkcert.sh
 
-# start g3fcgen
+# start nginx
+/usr/sbin/nginx -c "${PROJECT_DIR}"/scripts/coverage/g3proxy/nginx.conf
 
+# start g3fcgen
 "${PROJECT_DIR}"/target/debug/g3fcgen -c "${SCRIPTS_DIR}"/g3proxy/g3fcgen.yaml -G port2999 &
 FCGEN_PID=$!
 
@@ -60,6 +62,8 @@ done
 set +x
 
 kill -INT $FCGEN_PID
+NGINX_PID=$(cat /tmp/nginx.pid)
+kill -INT $NGINX_PID
 
 ## g3proxy-ftp
 

diff --git a/scripts/coverage/g3proxy/0006_chain_http_proxy/testcases.sh b/scripts/coverage/g3proxy/0006_chain_http_proxy/testcases.sh
@@ -4,42 +4,63 @@
 test_http_proxy_https_connect()
 {
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE}
 
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE}
 }
 
 
 test_http_proxy_https_forward()
 {
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T http://httpbin.local --no-auth --request-target-prefix https://httpbin.local:9443
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T http://httpbin.local --no-auth --request-target-prefix https://httpbin.local:2443
+}
+
+
+test_http_proxy_h2()
+{
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin_h2.py" -x ${HTTP_PROXY} -T https://httpbin.local:2443 --ca-cert ${SSL_CERT_FILE}
 }
 
 
 test_https_proxy_https_connect()
 {
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:9443 --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:2443 --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --ca-cert ${SSL_CERT_FILE}
 
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE}
 }
 
 
 test_https_proxy_https_forward()
 {
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T http://httpbin.local --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --request-target-prefix https://httpbin.local:9443
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T http://httpbin.local --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --request-target-prefix https://httpbin.local:2443
+}
+
+
+test_https_proxy_h2()
+{
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin_h2.py" -x ${HTTPS_PROXY} -T https://httpbin.local:2443 --proxy-ca-cert ${SSL_CERT_FILE} --ca-cert ${SSL_CERT_FILE}
 }
 
 
 test_socks5_proxy_https()
 {
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE}
 
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE}
 }
 
 
 test_socks4_proxy_https()
 {
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS4_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE}
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS4_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE}
 }
 
 

diff --git a/scripts/coverage/g3proxy/0010_escaper_direct_float/g3proxy.yaml b/scripts/coverage/g3proxy/0010_escaper_direct_float/g3proxy.yaml
@@ -17,6 +17,8 @@ escaper:
     type: direct_float
     resolver: default
     resolve_strategy: IPv4First
+    cache_ipv4: ipv4_ip_cache.json
+    cache_ipv6: ipv6_ip_cache.json
     egress_net_filter:
       default: allow
       allow: 127.0.0.1

diff --git a/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml b/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml
@@ -24,10 +24,12 @@ auditor:
   - name: default
     protocol_inspection: { }
     tls_cert_generator: { }
+    tls_interception_client:
+      ca-certificate: ../rootCA.pem
     tls_ticketer: { }
     tls_stream_dump: { }
-    icap_reqmod_service: icap://127.0.0.1:1344/echo
-    icap_respmod_service: icap://127.0.0.1:1344/echo
+    #icap_reqmod_service: icap://127.0.0.1:1344/echo
+    #icap_respmod_service: icap://127.0.0.1:1344/echo
 
 server:
   - name: rss

diff --git a/scripts/coverage/g3proxy/0020_base_audit/testcases.sh b/scripts/coverage/g3proxy/0020_base_audit/testcases.sh
@@ -6,6 +6,7 @@ test_http_proxy_http_forward
 test_http_proxy_ftp_over_http
 test_http_proxy_https_connect
 test_http_proxy_https_forward
+test_http_proxy_h2
 
 
 SOCKS5_PROXY="socks5h://127.0.0.1:1080"

diff --git a/scripts/coverage/g3proxy/g3proxy.yaml b/scripts/coverage/g3proxy/g3proxy.yaml
@@ -119,14 +119,6 @@ escaper:
     fallback_node: route3
 
 server:
-  - name: tls9443
-    escaper: direct
-    type: tls_stream
-    listen: 127.0.0.1:9443
-    tls_server:
-      certificate: httpbin.local.pem
-      private-key: httpbin.local-key.pem
-    upstream: 127.0.0.1:80
   - name: http_route13128
     type: http_proxy
     listen: 127.0.0.1:13128

diff --git a/scripts/coverage/g3proxy/nginx.conf b/scripts/coverage/g3proxy/nginx.conf
@@ -11,19 +11,23 @@ http {
     server {
         server_name httpbin.local;
 
-        listen 8080;
-        listen [::]:8080;
+        listen 2080;
+        listen [::]:2080;
 
-        listen 8443 ssl;
-        listen [::]:8443 ssl;
+        listen 2443 ssl http2;
+        listen [::]:2443 ssl http2;
 
-        http2 on;
+        # requires nginx 1.25.1
+        # http2 on;
 
         ssl_certificate httpbin.local.pem;
         ssl_certificate_key httpbin.local-key.pem;
 
         location / {
-                proxy_pass http://127.0.0.1:8000/;
+                proxy_pass http://127.0.0.1:80/;
+                client_max_body_size 10m;
+                proxy_buffering off;
+                proxy_request_buffering off;
         }
     }
 }
diff --git a/scripts/coverage/g3proxy/testcases.sh b/scripts/coverage/g3proxy/testcases.sh
@@ -13,5 +13,5 @@ for proxy in $all_proxies
 do
 	echo "-- ${proxy}"
 	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${proxy} -T http://httpbin.local || :
-	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${proxy} -T https://httpbin.local:9443 --ca-cert "${SCRIPTS_DIR}/g3proxy/rootCA.pem" || :
+	python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${proxy} -T https://httpbin.local:2443 --ca-cert "${SCRIPTS_DIR}/g3proxy/rootCA.pem" || :
 done