From eabe06fd67366a277c95eaa9e6006952bbdcd309 Mon Sep 17 00:00:00 2001 From: Zhang Jingqiang Date: Fri, 22 Nov 2024 14:51:00 +0800 Subject: [PATCH 1/2] g3proxy: add h2 testcase --- .github/workflows/codecov.yml | 3 + g3proxy/ci/python3+curl/test_httpbin.py | 1 + g3proxy/ci/python3+curl/test_httpbin_h2.py | 122 ++++++++++++++++++ scripts/coverage/g3proxy.sh | 6 +- .../0006_chain_http_proxy/testcases.sh | 21 +++ .../0010_escaper_direct_float/g3proxy.yaml | 2 + .../g3proxy/0020_base_audit/g3proxy.yaml | 2 + .../g3proxy/0020_base_audit/testcases.sh | 1 + scripts/coverage/g3proxy/g3proxy.yaml | 8 -- scripts/coverage/g3proxy/nginx.conf | 16 ++- scripts/coverage/g3proxy/testcases.sh | 2 +- 11 files changed, 168 insertions(+), 16 deletions(-) create mode 100644 g3proxy/ci/python3+curl/test_httpbin_h2.py diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 84bd43f0..a4549489 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -89,6 +89,9 @@ jobs: - name: Install binutils run: | cargo install cargo-binutils + - name: Install nginx + run: | + sudo apt-get install nginx-light - name: Install and run c-icap run: | sudo apt-get install c-icap diff --git a/g3proxy/ci/python3+curl/test_httpbin.py b/g3proxy/ci/python3+curl/test_httpbin.py index 59d7634c..ecdde3ed 100644 --- a/g3proxy/ci/python3+curl/test_httpbin.py +++ b/g3proxy/ci/python3+curl/test_httpbin.py @@ -26,6 +26,7 @@ def setUp(self): self.buffer = BytesIO() self.c = pycurl.Curl() + self.c.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_1_1) self.c.setopt(pycurl.WRITEFUNCTION, self.buffer.write) self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON]) if target_ca_cert is not None: diff --git a/g3proxy/ci/python3+curl/test_httpbin_h2.py b/g3proxy/ci/python3+curl/test_httpbin_h2.py new file mode 100644 index 00000000..ad3556b9 --- /dev/null +++ b/g3proxy/ci/python3+curl/test_httpbin_h2.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import argparse +import sys +import unittest +import base64 +from io import BytesIO +from urllib.parse import urlencode + +import pycurl + +target_site = 'https://httpbin.org' +target_ca_cert = None +target_proxy = None +proxy_ca_cert = None +local_resolve = None + +ACCEPT_JSON = 'Accept: application/json' +ACCEPT_HTML = 'Accept: text/html' + + +class TestHttpBin(unittest.TestCase): + def setUp(self): + self.buffer = BytesIO() + + self.c = pycurl.Curl() + self.c.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2) + self.c.setopt(pycurl.WRITEFUNCTION, self.buffer.write) + self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON]) + if target_ca_cert is not None: + self.c.setopt(pycurl.CAINFO, target_ca_cert) + if target_proxy is not None: + self.c.setopt(pycurl.PROXY, target_proxy) + if proxy_ca_cert is not None: + self.c.setopt(pycurl.PROXY_CAINFO, proxy_ca_cert) + if local_resolve is not None: + self.c.setopt(pycurl.RESOLVE, [local_resolve]) + + def tearDown(self): + self.c.close() + + def set_url_and_request_target(self, path: str): + self.c.setopt(pycurl.URL, f"{target_site}{path}") + + def test_simple_get(self): + self.set_url_and_request_target('/get') + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200) + + def test_basic_auth_get(self): + self.set_url_and_request_target('/basic-auth/name/pass') + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 401) + + auth_header = "Authorization: Basic {}".format(base64.standard_b64encode(b'name:pass').decode('utf-8')) + self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON, auth_header]) + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200) + + auth_header = "Authorization: Basic {}".format(base64.standard_b64encode(b'name:pas').decode('utf-8')) + self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_JSON, auth_header]) + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 401) + + def test_base64_decode(self): + self.set_url_and_request_target('/base64/SFRUUEJJTiBpcyBhd2Vzb21l') + self.c.setopt(pycurl.HTTPHEADER, [ACCEPT_HTML]) + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200) + self.assertEqual(self.buffer.getvalue(), b"HTTPBIN is awesome") + + def test_post_small(self): + data = "Content to post" + + self.set_url_and_request_target('/post') + self.c.setopt(pycurl.POSTFIELDS, data) + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200) + + def test_post_large(self): + post_data = {'data': "Content to post" * 1024 * 100} + post_fields = urlencode(post_data) + + # curl won't send Expect with HTTP2.0 + self.set_url_and_request_target('/post') + self.c.setopt(pycurl.POSTFIELDS, post_fields) + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200) + + def test_put_file(self): + self.set_url_and_request_target('/put') + self.c.setopt(pycurl.UPLOAD, 1) + file = open(__file__) + self.c.setopt(pycurl.READDATA, file) + self.c.perform() + self.assertEqual(self.c.getinfo(pycurl.RESPONSE_CODE), 200) + file.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--proxy', '-x', nargs='?', help='Proxy URL') + parser.add_argument('--site', '-T', nargs='?', help='Target Site', default=target_site) + parser.add_argument('--ca-cert', nargs='?', help='CA Cert') + parser.add_argument('--proxy-ca-cert', nargs='?', help='Proxy CA Cert') + parser.add_argument('--resolve', nargs='?', help='Local Resolve Record for curl') + + (args, left_args) = parser.parse_known_args() + + if args.ca_cert is not None: + target_ca_cert = args.ca_cert + if args.proxy is not None: + target_proxy = args.proxy + if args.proxy_ca_cert is not None: + proxy_ca_cert = args.proxy_ca_cert + if args.resolve is not None: + local_resolve = args.resolve + target_site = args.site + + left_args.insert(0, sys.argv[0]) + + unittest.main(argv=left_args) diff --git a/scripts/coverage/g3proxy.sh b/scripts/coverage/g3proxy.sh index 823ec680..2e674930 100755 --- a/scripts/coverage/g3proxy.sh +++ b/scripts/coverage/g3proxy.sh @@ -22,8 +22,10 @@ all_objects=$(find target/debug/deps/ -type f -perm /111 -not -name "*.so" | awk # generate resource files "${SCRIPTS_DIR}"/g3proxy/mkcert.sh -# start g3fcgen +# start nginx +/usr/sbin/nginx -c "${PROJECT_DIR}"/scripts/coverage/g3proxy/nginx.conf +# start g3fcgen "${PROJECT_DIR}"/target/debug/g3fcgen -c "${SCRIPTS_DIR}"/g3proxy/g3fcgen.yaml -G port2999 & FCGEN_PID=$! @@ -60,6 +62,8 @@ done set +x kill -INT $FCGEN_PID +NGINX_PID=$(cat /tmp/nginx.pid) +kill -INT $NGINX_PID ## g3proxy-ftp diff --git a/scripts/coverage/g3proxy/0006_chain_http_proxy/testcases.sh b/scripts/coverage/g3proxy/0006_chain_http_proxy/testcases.sh index e84b84db..5a22299e 100644 --- a/scripts/coverage/g3proxy/0006_chain_http_proxy/testcases.sh +++ b/scripts/coverage/g3proxy/0006_chain_http_proxy/testcases.sh @@ -4,42 +4,63 @@ test_http_proxy_https_connect() { python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE} python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTP_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE} } test_http_proxy_https_forward() { python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T http://httpbin.local --no-auth --request-target-prefix https://httpbin.local:9443 + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTP_PROXY} -T http://httpbin.local --no-auth --request-target-prefix https://httpbin.local:2443 +} + + +test_http_proxy_h2() +{ + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin_h2.py" -x ${HTTP_PROXY} -T https://httpbin.local:2443 --ca-cert ${SSL_CERT_FILE} } test_https_proxy_https_connect() { python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:9443 --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:2443 --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --ca-cert ${SSL_CERT_FILE} python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${HTTPS_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE} } test_https_proxy_https_forward() { python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T http://httpbin.local --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --request-target-prefix https://httpbin.local:9443 + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${HTTPS_PROXY} -T http://httpbin.local --no-auth --proxy-ca-cert ${SSL_CERT_FILE} --request-target-prefix https://httpbin.local:2443 +} + + +test_https_proxy_h2() +{ + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin_h2.py" -x ${HTTPS_PROXY} -T https://httpbin.local:2443 --proxy-ca-cert ${SSL_CERT_FILE} --ca-cert ${SSL_CERT_FILE} } test_socks5_proxy_https() { python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE} python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${SOCKS5_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE} } test_socks4_proxy_https() { python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS4_PROXY} -T https://httpbin.local:9443 --no-auth --ca-cert ${SSL_CERT_FILE} + python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -x ${SOCKS4_PROXY} -T https://httpbin.local:2443 --no-auth --ca-cert ${SSL_CERT_FILE} } diff --git a/scripts/coverage/g3proxy/0010_escaper_direct_float/g3proxy.yaml b/scripts/coverage/g3proxy/0010_escaper_direct_float/g3proxy.yaml index 928787d2..cf442f97 100644 --- a/scripts/coverage/g3proxy/0010_escaper_direct_float/g3proxy.yaml +++ b/scripts/coverage/g3proxy/0010_escaper_direct_float/g3proxy.yaml @@ -17,6 +17,8 @@ escaper: type: direct_float resolver: default resolve_strategy: IPv4First + cache_ipv4: ipv4_ip_cache.json + cache_ipv6: ipv6_ip_cache.json egress_net_filter: default: allow allow: 127.0.0.1 diff --git a/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml b/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml index 99981144..30c45948 100644 --- a/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml +++ b/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml @@ -24,6 +24,8 @@ auditor: - name: default protocol_inspection: { } tls_cert_generator: { } + tls_interception_client: + ca-certificate: ../rootCA.pem tls_ticketer: { } tls_stream_dump: { } icap_reqmod_service: icap://127.0.0.1:1344/echo diff --git a/scripts/coverage/g3proxy/0020_base_audit/testcases.sh b/scripts/coverage/g3proxy/0020_base_audit/testcases.sh index 658feda4..a4001afe 100644 --- a/scripts/coverage/g3proxy/0020_base_audit/testcases.sh +++ b/scripts/coverage/g3proxy/0020_base_audit/testcases.sh @@ -6,6 +6,7 @@ test_http_proxy_http_forward test_http_proxy_ftp_over_http test_http_proxy_https_connect test_http_proxy_https_forward +test_http_proxy_h2 SOCKS5_PROXY="socks5h://127.0.0.1:1080" diff --git a/scripts/coverage/g3proxy/g3proxy.yaml b/scripts/coverage/g3proxy/g3proxy.yaml index b611680a..5f08b24b 100644 --- a/scripts/coverage/g3proxy/g3proxy.yaml +++ b/scripts/coverage/g3proxy/g3proxy.yaml @@ -119,14 +119,6 @@ escaper: fallback_node: route3 server: - - name: tls9443 - escaper: direct - type: tls_stream - listen: 127.0.0.1:9443 - tls_server: - certificate: httpbin.local.pem - private-key: httpbin.local-key.pem - upstream: 127.0.0.1:80 - name: http_route13128 type: http_proxy listen: 127.0.0.1:13128 diff --git a/scripts/coverage/g3proxy/nginx.conf b/scripts/coverage/g3proxy/nginx.conf index 5dd39fc8..5d479715 100644 --- a/scripts/coverage/g3proxy/nginx.conf +++ b/scripts/coverage/g3proxy/nginx.conf @@ -11,19 +11,23 @@ http { server { server_name httpbin.local; - listen 8080; - listen [::]:8080; + listen 2080; + listen [::]:2080; - listen 8443 ssl; - listen [::]:8443 ssl; + listen 2443 ssl http2; + listen [::]:2443 ssl http2; - http2 on; + # requires nginx 1.25.1 + # http2 on; ssl_certificate httpbin.local.pem; ssl_certificate_key httpbin.local-key.pem; location / { - proxy_pass http://127.0.0.1:8000/; + proxy_pass http://127.0.0.1:80/; + client_max_body_size 10m; + proxy_buffering off; + proxy_request_buffering off; } } } diff --git a/scripts/coverage/g3proxy/testcases.sh b/scripts/coverage/g3proxy/testcases.sh index 1d78ecc4..2d1b50b2 100644 --- a/scripts/coverage/g3proxy/testcases.sh +++ b/scripts/coverage/g3proxy/testcases.sh @@ -13,5 +13,5 @@ for proxy in $all_proxies do echo "-- ${proxy}" python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${proxy} -T http://httpbin.local || : - python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${proxy} -T https://httpbin.local:9443 --ca-cert "${SCRIPTS_DIR}/g3proxy/rootCA.pem" || : + python3 "${PROJECT_DIR}/g3proxy/ci/python3+requests/test_httpbin.py" -x ${proxy} -T https://httpbin.local:2443 --ca-cert "${SCRIPTS_DIR}/g3proxy/rootCA.pem" || : done From 750be427a968de294df37350a687d3f79ce1289b Mon Sep 17 00:00:00 2001 From: Zhang Jingqiang Date: Fri, 22 Nov 2024 17:05:18 +0800 Subject: [PATCH 2/2] g3proxy: disable ICAP testcase --- scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml b/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml index 30c45948..706f21c4 100644 --- a/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml +++ b/scripts/coverage/g3proxy/0020_base_audit/g3proxy.yaml @@ -28,8 +28,8 @@ auditor: ca-certificate: ../rootCA.pem tls_ticketer: { } tls_stream_dump: { } - icap_reqmod_service: icap://127.0.0.1:1344/echo - icap_respmod_service: icap://127.0.0.1:1344/echo + #icap_reqmod_service: icap://127.0.0.1:1344/echo + #icap_respmod_service: icap://127.0.0.1:1344/echo server: - name: rss