feat:support configuring xff trusted cidrs

internal/ir/xds.go

@@ -35,47 +35,49 @@
 )
 
 var (
-	ErrListenerNameEmpty                       = errors.New("field Name must be specified")
-	ErrListenerAddressInvalid                  = errors.New("field Address must be a valid IP address")
-	ErrListenerPortInvalid                     = errors.New("field Port specified is invalid")
-	ErrHTTPListenerHostnamesEmpty              = errors.New("field Hostnames must be specified with at least a single hostname entry")
-	ErrTCPRouteSNIsEmpty                       = errors.New("field SNIs must be specified with at least a single server name entry")
-	ErrTLSServerCertEmpty                      = errors.New("field ServerCertificate must be specified")
-	ErrTLSPrivateKey                           = errors.New("field PrivateKey must be specified")
-	ErrRouteNameEmpty                          = errors.New("field Name must be specified")
-	ErrHTTPRouteHostnameEmpty                  = errors.New("field Hostname must be specified")
-	ErrDestinationNameEmpty                    = errors.New("field Name must be specified")
-	ErrDestEndpointHostInvalid                 = errors.New("field Address must be a valid IP or FQDN address")
-	ErrDestEndpointPortInvalid                 = errors.New("field Port specified is invalid")
-	ErrDestEndpointUDSPortInvalid              = errors.New("field Port must not be specified for Unix Domain Socket address")
-	ErrDestEndpointUDSHostInvalid              = errors.New("field Host must not be specified for Unix Domain Socket address")
-	ErrStringMatchConditionInvalid             = errors.New("only one of the Exact, Prefix, SafeRegex or Distinct fields must be set")
-	ErrStringMatchInvertDistinctInvalid        = errors.New("only one of the Invert or Distinct fields can be set")
-	ErrStringMatchNameIsEmpty                  = errors.New("field Name must be specified")
-	ErrDirectResponseStatusInvalid             = errors.New("only HTTP status codes 100 - 599 are supported for DirectResponse")
-	ErrRedirectUnsupportedStatus               = errors.New("only HTTP status codes 301 and 302 are supported for redirect filters")
-	ErrRedirectUnsupportedScheme               = errors.New("only http and https are supported for the scheme in redirect filters")
-	ErrHTTPPathModifierDoubleReplace           = errors.New("redirect filter cannot have a path modifier that supplies more than one of fullPathReplace, prefixMatchReplace and regexMatchReplace")
-	ErrHTTPPathModifierNoReplace               = errors.New("redirect filter cannot have a path modifier that does not supply either fullPathReplace, prefixMatchReplace or regexMatchReplace")
-	ErrHTTPPathRegexModifierNoSetting          = errors.New("redirect filter cannot have a path modifier that does not supply either fullPathReplace, prefixMatchReplace or regexMatchReplace")
-	ErrHTTPHostModifierDoubleReplace           = errors.New("redirect filter cannot have a host modifier that supplies more than one of Hostname, Header and Backend")
-	ErrAddHeaderEmptyName                      = errors.New("header modifier filter cannot configure a header without a name to be added")
-	ErrAddHeaderDuplicate                      = errors.New("header modifier filter attempts to add the same header more than once (case insensitive)")
-	ErrRemoveHeaderDuplicate                   = errors.New("header modifier filter attempts to remove the same header more than once (case insensitive)")
-	ErrLoadBalancerInvalid                     = errors.New("loadBalancer setting is invalid, only one setting can be set")
-	ErrHealthCheckTimeoutInvalid               = errors.New("field HealthCheck.Timeout must be specified")
-	ErrHealthCheckIntervalInvalid              = errors.New("field HealthCheck.Interval must be specified")
-	ErrHealthCheckUnhealthyThresholdInvalid    = errors.New("field HealthCheck.UnhealthyThreshold should be greater than 0")
-	ErrHealthCheckHealthyThresholdInvalid      = errors.New("field HealthCheck.HealthyThreshold should be greater than 0")
-	ErrHealthCheckerInvalid                    = errors.New("health checker setting is invalid, only one health checker can be set")
-	ErrHCHTTPHostInvalid                       = errors.New("field HTTPHealthChecker.Host should be specified")
-	ErrHCHTTPPathInvalid                       = errors.New("field HTTPHealthChecker.Path should be specified")
-	ErrHCHTTPMethodInvalid                     = errors.New("only one of the GET, HEAD, POST, DELETE, OPTIONS, TRACE, PATCH of HTTPHealthChecker.Method could be set")
-	ErrHCHTTPExpectedStatusesInvalid           = errors.New("field HTTPHealthChecker.ExpectedStatuses should be specified")
-	ErrHealthCheckPayloadInvalid               = errors.New("one of Text, Binary fields must be set in payload")
-	ErrHTTPStatusInvalid                       = errors.New("HTTPStatus should be in [200,600)")
-	ErrOutlierDetectionBaseEjectionTimeInvalid = errors.New("field OutlierDetection.BaseEjectionTime must be specified")
-	ErrOutlierDetectionIntervalInvalid         = errors.New("field OutlierDetection.Interval must be specified")
+	ErrListenerNameEmpty                        = errors.New("field Name must be specified")
+	ErrListenerAddressInvalid                   = errors.New("field Address must be a valid IP address")
+	ErrListenerPortInvalid                      = errors.New("field Port specified is invalid")
+	ErrHTTPListenerHostnamesEmpty               = errors.New("field Hostnames must be specified with at least a single hostname entry")
+	ErrTCPRouteSNIsEmpty                        = errors.New("field SNIs must be specified with at least a single server name entry")
+	ErrTLSServerCertEmpty                       = errors.New("field ServerCertificate must be specified")
+	ErrTLSPrivateKey                            = errors.New("field PrivateKey must be specified")
+	ErrRouteNameEmpty                           = errors.New("field Name must be specified")
+	ErrHTTPRouteHostnameEmpty                   = errors.New("field Hostname must be specified")
+	ErrDestinationNameEmpty                     = errors.New("field Name must be specified")
+	ErrDestEndpointHostInvalid                  = errors.New("field Address must be a valid IP or FQDN address")
+	ErrDestEndpointPortInvalid                  = errors.New("field Port specified is invalid")
+	ErrDestEndpointUDSPortInvalid               = errors.New("field Port must not be specified for Unix Domain Socket address")
+	ErrDestEndpointUDSHostInvalid               = errors.New("field Host must not be specified for Unix Domain Socket address")
+	ErrStringMatchConditionInvalid              = errors.New("only one of the Exact, Prefix, SafeRegex or Distinct fields must be set")
+	ErrStringMatchInvertDistinctInvalid         = errors.New("only one of the Invert or Distinct fields can be set")
+	ErrStringMatchNameIsEmpty                   = errors.New("field Name must be specified")
+	ErrDirectResponseStatusInvalid              = errors.New("only HTTP status codes 100 - 599 are supported for DirectResponse")
+	ErrRedirectUnsupportedStatus                = errors.New("only HTTP status codes 301 and 302 are supported for redirect filters")
+	ErrRedirectUnsupportedScheme                = errors.New("only http and https are supported for the scheme in redirect filters")
+	ErrHTTPPathModifierDoubleReplace            = errors.New("redirect filter cannot have a path modifier that supplies more than one of fullPathReplace, prefixMatchReplace and regexMatchReplace")
+	ErrHTTPPathModifierNoReplace                = errors.New("redirect filter cannot have a path modifier that does not supply either fullPathReplace, prefixMatchReplace or regexMatchReplace")
+	ErrHTTPPathRegexModifierNoSetting           = errors.New("redirect filter cannot have a path modifier that does not supply either fullPathReplace, prefixMatchReplace or regexMatchReplace")
+	ErrHTTPHostModifierDoubleReplace            = errors.New("redirect filter cannot have a host modifier that supplies more than one of Hostname, Header and Backend")
+	ErrAddHeaderEmptyName                       = errors.New("header modifier filter cannot configure a header without a name to be added")
+	ErrAddHeaderDuplicate                       = errors.New("header modifier filter attempts to add the same header more than once (case insensitive)")
+	ErrRemoveHeaderDuplicate                    = errors.New("header modifier filter attempts to remove the same header more than once (case insensitive)")
+	ErrLoadBalancerInvalid                      = errors.New("loadBalancer setting is invalid, only one setting can be set")
+	ErrHealthCheckTimeoutInvalid                = errors.New("field HealthCheck.Timeout must be specified")
+	ErrHealthCheckIntervalInvalid               = errors.New("field HealthCheck.Interval must be specified")
+	ErrHealthCheckUnhealthyThresholdInvalid     = errors.New("field HealthCheck.UnhealthyThreshold should be greater than 0")
+	ErrHealthCheckHealthyThresholdInvalid       = errors.New("field HealthCheck.HealthyThreshold should be greater than 0")
+	ErrHealthCheckerInvalid                     = errors.New("health checker setting is invalid, only one health checker can be set")
+	ErrHCHTTPHostInvalid                        = errors.New("field HTTPHealthChecker.Host should be specified")
+	ErrHCHTTPPathInvalid                        = errors.New("field HTTPHealthChecker.Path should be specified")
+	ErrHCHTTPMethodInvalid                      = errors.New("only one of the GET, HEAD, POST, DELETE, OPTIONS, TRACE, PATCH of HTTPHealthChecker.Method could be set")
+	ErrHCHTTPExpectedStatusesInvalid            = errors.New("field HTTPHealthChecker.ExpectedStatuses should be specified")
+	ErrHealthCheckPayloadInvalid                = errors.New("one of Text, Binary fields must be set in payload")
+	ErrHTTPStatusInvalid                        = errors.New("HTTPStatus should be in [200,600)")
+	ErrOutlierDetectionBaseEjectionTimeInvalid  = errors.New("field OutlierDetection.BaseEjectionTime must be specified")
+	ErrOutlierDetectionIntervalInvalid          = errors.New("field OutlierDetection.Interval must be specified")
+	ErrBothXForwardedForAndCustomHeaderInvalid  = errors.New("only one of ClientIPDetection.XForwardedFor and ClientIPDetection.CustomHeader must be set")
+	ErrBothNumTrustedHopsAndTrustedCIDRsInvalid = errors.New("only one of ClientIPDetection.XForwardedFor.NumTrustedHops and ClientIPDetection.XForwardedFor.TrustedCIDRs must be set")
 
 	redacted = []byte("[redacted]")
 )
@@ -345,6 +347,15 @@
 			errs = errors.Join(errs, err)
 		}
 	}
+	if h.ClientIPDetection != nil {
+		if h.ClientIPDetection.XForwardedFor != nil && h.ClientIPDetection.CustomHeader != nil {
+			errs = errors.Join(errs, ErrBothXForwardedForAndCustomHeaderInvalid)
+		} else if h.ClientIPDetection.XForwardedFor != nil {
+			if h.ClientIPDetection.XForwardedFor.NumTrustedHops != nil && h.ClientIPDetection.XForwardedFor.TrustedCIDRs != nil {
+				errs = errors.Join(errs, ErrBothNumTrustedHopsAndTrustedCIDRsInvalid)
+			}
+		}
+	}
 	return errs
 }
 

internal/xds/translator/listener.go

@@ -7,6 +7,7 @@ package translator
 
 import (
 	"errors"
+	"net"
 	"strconv"
 	"strings"
 
@@ -23,12 +24,14 @@ import (
 	early_header_mutationv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/early_header_mutation/header_mutation/v3"
 	preservecasev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/header_formatters/preserve_case/v3"
 	customheaderv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/original_ip_detection/custom_header/v3"
+	xffv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/original_ip_detection/xff/v3"
 	quicv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/quic/v3"
 	tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	typev3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"github.com/envoyproxy/go-control-plane/pkg/resource/v3"
 	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 	"k8s.io/utils/ptr"
@@ -108,9 +111,13 @@ func http2ProtocolOptions(opts *ir.HTTP2Settings) *corev3.Http2ProtocolOptions {
 	return out
 }
 
+// xffNumTrustedHops returns the number of hops to be configured in proxy
+// Need to decrement number of hops configured by EGW user by 1 for backward compatibility
+// See for more: https://github.com/envoyproxy/envoy/issues/34241
 func xffNumTrustedHops(clientIPDetection *ir.ClientIPDetectionSettings) uint32 {
-	if clientIPDetection != nil && clientIPDetection.XForwardedFor != nil && clientIPDetection.XForwardedFor.NumTrustedHops != nil {
-		return *clientIPDetection.XForwardedFor.NumTrustedHops
+	if clientIPDetection != nil && clientIPDetection.XForwardedFor != nil &&
+		clientIPDetection.XForwardedFor.NumTrustedHops != nil && *clientIPDetection.XForwardedFor.NumTrustedHops > 0 {
+		return *clientIPDetection.XForwardedFor.NumTrustedHops - 1
 	}
 	return 0
 }
@@ -141,6 +148,34 @@ func originalIPDetectionExtensions(clientIPDetection *ir.ClientIPDetectionSettin
 			Name:        "envoy.extensions.http.original_ip_detection.custom_header",
 			TypedConfig: customHeaderConfigAny,
 		})
+	} else if clientIPDetection.XForwardedFor != nil {
+		var xffHeaderConfigAny *anypb.Any
+		if clientIPDetection.XForwardedFor.TrustedCIDRs != nil {
+			trustedCidrs := make([]*corev3.CidrRange, 0)
+			for _, cidr := range clientIPDetection.XForwardedFor.TrustedCIDRs {
+				ip, nw, _ := net.ParseCIDR(string(cidr))
+				prefixLen, _ := nw.Mask.Size()
+				trustedCidrs = append(trustedCidrs, &corev3.CidrRange{
+					AddressPrefix: ip.String(),
+					PrefixLen:     wrapperspb.UInt32(uint32(prefixLen)),
+				})
+			}
+			xffHeaderConfigAny, _ = protocov.ToAnyWithValidation(&xffv3.XffConfig{
+				XffTrustedCidrs: &xffv3.XffTrustedCidrs{
+					Cidrs: trustedCidrs,
+				},
+				SkipXffAppend: wrapperspb.Bool(false),
+			})
+		} else if clientIPDetection.XForwardedFor.NumTrustedHops != nil {
+			xffHeaderConfigAny, _ = protocov.ToAnyWithValidation(&xffv3.XffConfig{
+				XffNumTrustedHops: xffNumTrustedHops(clientIPDetection),
+				SkipXffAppend:     wrapperspb.Bool(false),
+			})
+		}
+		extensionConfig = append(extensionConfig, &corev3.TypedExtensionConfig{
+			Name:        "envoy.extensions.http.original_ip_detection.xff",
+			TypedConfig: xffHeaderConfigAny,
+		})
 	}
 
 	return extensionConfig
@@ -292,7 +327,6 @@ func (t *Translator) addHCMToXDSListener(xdsListener *listenerv3.Listener, irLis
 		Http2ProtocolOptions: http2ProtocolOptions(irListener.HTTP2),
 		// https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
 		UseRemoteAddress:              &wrapperspb.BoolValue{Value: useRemoteAddress},
-		XffNumTrustedHops:             xffNumTrustedHops(irListener.ClientIPDetection),
 		OriginalIpDetectionExtensions: originalIPDetectionExtensions,
 		// normalize paths according to RFC 3986
 		NormalizePath:                &wrapperspb.BoolValue{Value: true},

internal/xds/translator/testdata/in/xds-ir/client-ip-detection.yaml

@@ -52,3 +52,23 @@ http:
     customHeader:
       name: "x-my-custom-header"
       failClosed: true
+- name: "fourth-listener"
+  address: "::"
+  port: 8084
+  hostnames:
+  - "*"
+  routes:
+  - name: "fourth-route"
+    hostname: "*"
+    destination:
+      name: "fourth-route-dest"
+      settings:
+      - endpoints:
+        - host: "4.4.4.4"
+          port: 8084
+  clientIPDetection:
+    xForwardedFor:
+      trustedCidrs:
+      - "192.168.1.0/24"
+      - "10.0.0.0/16"
+      - "172.16.0.0/12"

internal/xds/translator/testdata/out/xds-ir/client-ip-detection.clusters.yaml

@@ -49,3 +49,20 @@
   name: third-route-dest
   perConnectionBufferLimitBytes: 32768
   type: EDS
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_PREFERRED
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+    serviceName: fourth-route-dest
+  ignoreHealthOnHostRemoval: true
+  lbPolicy: LEAST_REQUEST
+  name: fourth-route-dest
+  perConnectionBufferLimitBytes: 32768
+  type: EDS

internal/xds/translator/testdata/out/xds-ir/client-ip-detection.endpoints.yaml

@@ -34,3 +34,15 @@
     loadBalancingWeight: 1
     locality:
       region: third-route-dest/backend/0
+- clusterName: fourth-route-dest
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 4.4.4.4
+            portValue: 8084
+      loadBalancingWeight: 1
+    loadBalancingWeight: 1
+    locality:
+      region: fourth-route-dest/backend/0

internal/xds/translator/testdata/out/xds-ir/client-ip-detection.listeners.yaml

@@ -19,15 +19,20 @@
             '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
             suppressEnvoyHeaders: true
         normalizePath: true
+        originalIpDetectionExtensions:
+        - name: envoy.extensions.http.original_ip_detection.xff
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.http.original_ip_detection.xff.v3.XffConfig
+            skipXffAppend: false
+            xffNumTrustedHops: 1
         rds:
           configSource:
             ads: {}
             resourceApiVersion: V3
           routeConfigName: first-listener
         serverHeaderTransformation: PASS_THROUGH
         statPrefix: http-8081
-        useRemoteAddress: true
-        xffNumTrustedHops: 2
+        useRemoteAddress: false
     name: first-listener
   name: first-listener
   perConnectionBufferLimitBytes: 32768
@@ -109,3 +114,48 @@
     name: third-listener
   name: third-listener
   perConnectionBufferLimitBytes: 32768
+- address:
+    socketAddress:
+      address: '::'
+      portValue: 8084
+  defaultFilterChain:
+    filters:
+    - name: envoy.filters.network.http_connection_manager
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+        commonHttpProtocolOptions:
+          headersWithUnderscoresAction: REJECT_REQUEST
+        http2ProtocolOptions:
+          initialConnectionWindowSize: 1048576
+          initialStreamWindowSize: 65536
+          maxConcurrentStreams: 100
+        httpFilters:
+        - name: envoy.filters.http.router
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+            suppressEnvoyHeaders: true
+        normalizePath: true
+        originalIpDetectionExtensions:
+        - name: envoy.extensions.http.original_ip_detection.xff
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.http.original_ip_detection.xff.v3.XffConfig
+            skipXffAppend: false
+            xffTrustedCidrs:
+              cidrs:
+              - addressPrefix: 192.168.1.0
+                prefixLen: 24
+              - addressPrefix: 10.0.0.0
+                prefixLen: 16
+              - addressPrefix: 172.16.0.0
+                prefixLen: 12
+        rds:
+          configSource:
+            ads: {}
+            resourceApiVersion: V3
+          routeConfigName: fourth-listener
+        serverHeaderTransformation: PASS_THROUGH
+        statPrefix: http-8084
+        useRemoteAddress: false
+    name: fourth-listener
+  name: fourth-listener
+  perConnectionBufferLimitBytes: 32768

internal/xds/translator/testdata/out/xds-ir/client-ip-detection.routes.yaml

@@ -40,3 +40,17 @@
         cluster: third-route-dest
         upgradeConfigs:
         - upgradeType: websocket
+- ignorePortInHostMatching: true
+  name: fourth-listener
+  virtualHosts:
+  - domains:
+    - '*'
+    name: fourth-listener/*
+    routes:
+    - match:
+        prefix: /
+      name: fourth-route
+      route:
+        cluster: fourth-route-dest
+        upgradeConfigs:
+        - upgradeType: websocket