limit pipeline http_request function

pipeline/ptinput/funcs/fn_http_request.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -27,6 +28,61 @@ var defaultTransport http.RoundTripper = &http.Transport{
 	ExpectContinueTimeout: 1 * time.Second,
 }
 
+var disableInternalHost bool
+var cidrs []string
+
+func SetNetFilter(disableInternal bool, cidrList []string) {
+	disableInternalHost = disableInternal
+	cidrs = append(cidrs, cidrList...)
+}
+
+func filterHost(host string, disableInternal bool, cidrs []string) bool {
+	ips, err := net.LookupIP(host)
+	if err != nil {
+		return true
+	}
+
+	for _, cidr := range cidrs {
+		_, ipNet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			l.Debug("parse cidr %s failed: %s", cidr, err)
+			continue
+		}
+		for _, ip := range ips {
+			if ipNet.Contains(ip) {
+				return true
+			}
+		}
+	}
+
+	if disableInternal {
+		for _, ip := range ips {
+			if ip.IsLoopback() ||
+				ip.IsPrivate() ||
+				ip.IsLinkLocalUnicast() ||
+				ip.IsLinkLocalMulticast() ||
+				ip.IsUnspecified() {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func filterURL(urlStr string, disable bool, cidrs []string) bool {
+	urlP, err := url.Parse(urlStr)
+	if err != nil || urlP == nil {
+		return true
+	}
+
+	if urlP.Scheme != "http" && urlP.Scheme != "https" {
+		return true
+	}
+
+	return filterHost(urlP.Hostname(), disable, cidrs)
+}
+
 func HTTPRequestChecking(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError {
 	if err := normalizeFuncArgsDeprecated(funcExpr, []string{
 		"method", "url", "headers", "body",
@@ -57,6 +113,11 @@ func HTTPRequest(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError {
 			funcExpr.Param[1].StartPos())
 	}
 
+	if filterURL(url.(string), disableInternalHost, cidrs) {
+		ctx.Regs.ReturnAppend(nil, ast.Nil)
+		return nil
+	}
+
 	var headers any
 	if funcExpr.Param[2] != nil {
 		var headersType ast.DType

pipeline/ptinput/funcs/fn_http_request_test.go

@@ -16,6 +16,102 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestFilter(t *testing.T) {
+	cases := []struct {
+		url     string
+		filter  bool
+		disable bool
+		cidrs   []string
+	}{
+		{
+			"http://0.0.0.0/",
+			true,
+			true,
+			nil,
+		},
+		{
+			"http://localhost/",
+			true,
+			true,
+			nil,
+		},
+		{
+			"http://1.0.0.1/",
+			false,
+			true,
+			nil,
+		},
+		{
+			"http://1.0.0.1:1234/",
+			false,
+			true,
+			nil,
+		},
+		{
+			"http://[::]:1234/",
+			false,
+			false,
+			nil,
+		},
+		{
+			"http://[::]:1234/",
+			true,
+			true,
+			nil,
+		},
+		{
+			"http://[::]/",
+			true,
+			true,
+			nil,
+		},
+		{
+			"http://1.0.0.1",
+			true,
+			true,
+			[]string{"1.0.0.0/16"},
+		},
+		{
+			"http://10.0.0.1",
+			true,
+			true,
+			nil,
+		},
+		{
+			"http://10.0.0.1",
+			false,
+			false,
+			nil,
+		},
+		{
+			"http://10.0.0.1",
+			true,
+			false,
+			[]string{"10.0.0.1/16"},
+		},
+		{
+			"file://ccc/",
+			true,
+			true,
+			nil,
+		},
+		{
+			"https://guance.com",
+			false,
+			true,
+			nil,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.url, func(t *testing.T) {
+			assert.Equal(t, c.filter, filterURL(
+				c.url, c.disable, c.cidrs,
+			))
+		})
+	}
+}
+
 func TestBuildBody(t *testing.T) {
 	cases := []struct {
 		val    any
@@ -86,20 +182,31 @@ func TestHTTPRequest(t *testing.T) {
 		{
 			name: "test_post",
 			pl: fmt.Sprintf(`
-			resp = http_request("POST", %s, {"extraHeader": "1", 
+			resp = http_request("POST", %s, {"extraHeader": "1",
 			"extraHeader": "1"}, {"a": "1"})
-			add_key(abc, resp["body"])	
+			add_key(abc, resp["body"])
 			`, url),
 			in:       `[]`,
 			outkey:   "abc",
 			expected: `{"a":"1"}`,
 		},
+		{
+			name: "test_file",
+			pl: `
+			resp = http_request("POST", "file:///etc/", {"extraHeader": "1", 
+			"extraHeader": "1"}, {"a": "1"})
+			add_key(abc, resp)
+			`,
+			in:       `[]`,
+			outkey:   "abc",
+			expected: nil,
+		},
 		{
 			name: "test_put",
 			pl: fmt.Sprintf(`
-			resp = http_request("put", %s, {"extraHeader": "1", 
+			resp = http_request("put", %s, {"extraHeader": "1",
 			"extraHeader": "1"}, {"a": "1"})
-			add_key(abc, resp["body"])	
+			add_key(abc, resp["body"])
 			`, url),
 			in:       `[]`,
 			outkey:   "abc",