Restricting network access to http_request functions (#127)

* Restricting network access to http_request functions * update * update * Add host whitelist
GuanceCloud · Dec 25, 2024 · 60d3422 · 60d3422
1 parent febc32f
commit 60d3422
Show file tree

Hide file tree

Showing 2 changed files with 227 additions and 4 deletions.
diff --git a/pipeline/ptinput/funcs/fn_http_request.go b/pipeline/ptinput/funcs/fn_http_request.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -27,6 +28,78 @@ var defaultTransport http.RoundTripper = &http.Transport{
 	ExpectContinueTimeout: 1 * time.Second,
 }
 
+var gDisableInternalNet bool
+var gCIDRsWhitelist []string
+var gHostWhitelist []string
+
+func SetNetFilter(disableInternal bool, cidrWList, hostWList []string) {
+	gDisableInternalNet = disableInternal
+	gCIDRsWhitelist = append(gCIDRsWhitelist, cidrWList...)
+	gHostWhitelist = append(gHostWhitelist, hostWList...)
+}
+
+func filterHost(host string, disableInternal bool, cidrsWhite []string, hostWhite []string) bool {
+	// host whitelist
+	for i := range hostWhite {
+		if host == hostWhite[i] {
+			return false
+		}
+	}
+
+	var ips []net.IP
+	if len(cidrsWhite) > 0 || disableInternal {
+		var err error
+		if ips, err = net.LookupIP(host); err != nil {
+			return true
+		}
+	}
+
+	// cidr whitelist
+	for _, cidr := range cidrsWhite {
+		if _, ipNet, err := net.ParseCIDR(cidr); err != nil {
+			l.Debug("parse cidr %s failed: %s", cidr, err)
+			continue
+		} else if ipNet != nil {
+			for i := range ips {
+				if ipNet.Contains(ips[i]) {
+					return false
+				}
+			}
+		}
+	}
+	if len(hostWhite) > 0 || len(cidrsWhite) > 0 {
+		return true
+	}
+
+	// disable internal netwrok
+	if disableInternal {
+		for _, ip := range ips {
+			if ip.IsLoopback() ||
+				ip.IsPrivate() ||
+				ip.IsLinkLocalUnicast() ||
+				ip.IsLinkLocalMulticast() ||
+				ip.IsUnspecified() {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func filterURL(urlStr string, disable bool, cidrs, hosts []string) bool {
+	urlP, err := url.Parse(urlStr)
+	if err != nil || urlP == nil {
+		return true
+	}
+
+	if urlP.Scheme != "http" && urlP.Scheme != "https" {
+		return true
+	}
+
+	return filterHost(urlP.Hostname(), disable, cidrs, hosts)
+}
+
 func HTTPRequestChecking(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError {
 	if err := normalizeFuncArgsDeprecated(funcExpr, []string{
 		"method", "url", "headers", "body",
@@ -57,6 +130,11 @@ func HTTPRequest(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError {
 			funcExpr.Param[1].StartPos())
 	}
 
+	if filterURL(url.(string), gDisableInternalNet, gCIDRsWhitelist, gHostWhitelist) {
+		ctx.Regs.ReturnAppend(nil, ast.Nil)
+		return nil
+	}
+
 	var headers any
 	if funcExpr.Param[2] != nil {
 		var headersType ast.DType

diff --git a/pipeline/ptinput/funcs/fn_http_request_test.go b/pipeline/ptinput/funcs/fn_http_request_test.go
@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 	"time"
 
@@ -16,6 +17,139 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestFilter(t *testing.T) {
+	cases := []struct {
+		url             string
+		filterResult    bool
+		disableInternal bool
+		cidrs, hosts    []string
+	}{
+		{
+			url:             "http://0.0.0.0/",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://localhost/",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://127.0.0.0.1/",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://1.0.0.1/",
+			filterResult:    false,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://1.0.0.1:1234/",
+			filterResult:    false,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://[::]:1234/",
+			filterResult:    false,
+			disableInternal: false,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://[::]:1234/",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://[::]/",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://1.0.0.1",
+			filterResult:    false,
+			disableInternal: true,
+			cidrs:           []string{"1.0.0.0/16"},
+		},
+		{
+			url:             "http://10.0.0.1",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://192.168.0.1",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://10.0.0.1",
+			filterResult:    false,
+			disableInternal: false,
+			cidrs:           nil,
+		},
+		{
+			url:             "http://10.0.0.1",
+			filterResult:    false,
+			disableInternal: false,
+			cidrs:           []string{"10.0.0.1/16"},
+		},
+		{
+			url:             "file://ccc/",
+			filterResult:    true,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:             "https://guance.com",
+			filterResult:    false,
+			disableInternal: true,
+			cidrs:           nil,
+		},
+		{
+			url:          "https://guance.com",
+			hosts:        []string{"guance.com"},
+			filterResult: false,
+		},
+		{
+			url:          "https://guance.com",
+			hosts:        []string{"guancez.com"},
+			filterResult: true,
+		},
+		{
+			url:             "https://127.0.0.1",
+			hosts:           []string{"127.0.0.1"},
+			filterResult:    false,
+			disableInternal: true,
+		},
+		{
+			url:             "https://127.0.0.1",
+			cidrs:           []string{"127.0.0.1/32"},
+			filterResult:    false,
+			disableInternal: true,
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r := filterURL(
+				c.url, c.disableInternal, c.cidrs, c.hosts,
+			)
+			if r != c.filterResult {
+				assert.Equal(t, c.filterResult, r)
+			}
+		})
+	}
+}
+
 func TestBuildBody(t *testing.T) {
 	cases := []struct {
 		val    any
@@ -86,20 +220,31 @@ func TestHTTPRequest(t *testing.T) {
 		{
 			name: "test_post",
 			pl: fmt.Sprintf(`
-			resp = http_request("POST", %s, {"extraHeader": "1", 
+			resp = http_request("POST", %s, {"extraHeader": "1",
 			"extraHeader": "1"}, {"a": "1"})
-			add_key(abc, resp["body"])	
+			add_key(abc, resp["body"])
 			`, url),
 			in:       `[]`,
 			outkey:   "abc",
 			expected: `{"a":"1"}`,
 		},
+		{
+			name: "test_file",
+			pl: `
+			resp = http_request("POST", "file:///etc/", {"extraHeader": "1", 
+			"extraHeader": "1"}, {"a": "1"})
+			add_key(abc, resp)
+			`,
+			in:       `[]`,
+			outkey:   "abc",
+			expected: nil,
+		},
 		{
 			name: "test_put",
 			pl: fmt.Sprintf(`
-			resp = http_request("put", %s, {"extraHeader": "1", 
+			resp = http_request("put", %s, {"extraHeader": "1",
 			"extraHeader": "1"}, {"a": "1"})
-			add_key(abc, resp["body"])	
+			add_key(abc, resp["body"])
 			`, url),
 			in:       `[]`,
 			outkey:   "abc",