diff --git a/api/v1alpha1/backend_types.go b/api/v1alpha1/backend_types.go index a2022c2c5ef0..bb407981af87 100644 --- a/api/v1alpha1/backend_types.go +++ b/api/v1alpha1/backend_types.go @@ -131,7 +131,6 @@ type BackendSpec struct { // the health of the active backends falls below 72%. // // +optional - // +notImplementedHide Fallback *bool `json:"fallback,omitempty"` } diff --git a/api/v1alpha1/httproutefilter_types.go b/api/v1alpha1/httproutefilter_types.go index 7f56ca07d7cb..9ae8be598420 100644 --- a/api/v1alpha1/httproutefilter_types.go +++ b/api/v1alpha1/httproutefilter_types.go @@ -33,6 +33,8 @@ type HTTPRouteFilter struct { type HTTPRouteFilterSpec struct { // +optional URLRewrite *HTTPURLRewriteFilter `json:"urlRewrite,omitempty"` + // +optional + DirectResponse *HTTPDirectResponseFilter `json:"directResponse,omitempty"` } // HTTPURLRewriteFilter define rewrites of HTTP URL components such as path and host @@ -49,6 +51,24 @@ type HTTPURLRewriteFilter struct { Path *HTTPPathModifier `json:"path,omitempty"` } +// HTTPDirectResponseFilter defines the configuration to return a fixed response. +type HTTPDirectResponseFilter struct { + // Content Type of the response. This will be set in the Content-Type header. + // + // +optional + ContentType *string `json:"contentType,omitempty"` + + // Body of the Response + // + // +optional + Body *CustomResponseBody `json:"body,omitempty"` + + // Status Code of the HTTP response + // If unset, defaults to 200. + // +optional + StatusCode *int `json:"statusCode,omitempty"` +} + // HTTPPathModifierType defines the type of path redirect or rewrite. type HTTPPathModifierType string diff --git a/api/v1alpha1/oidc_types.go b/api/v1alpha1/oidc_types.go index 73e1dea8c6e0..78c32287cde2 100644 --- a/api/v1alpha1/oidc_types.go +++ b/api/v1alpha1/oidc_types.go @@ -107,7 +107,8 @@ type OIDC struct { // OIDCProvider defines the OIDC Provider configuration. // +kubebuilder:validation:XValidation:rule="!has(self.backendRef)",message="BackendRefs must be used, backendRef is not supported." -// +kubebuilder:validation:XValidation:rule="has(self.backendRefs)? self.backendRefs.size() > 1 : true",message="Only one backendRefs is allowed." +// +kubebuilder:validation:XValidation:rule="has(self.backendSettings)? (has(self.backendSettings.retry)?(has(self.backendSettings.retry.perRetry)? !has(self.backendSettings.retry.perRetry.timeout):true):true):true",message="Retry timeout is not supported." +// +kubebuilder:validation:XValidation:rule="has(self.backendSettings)? (has(self.backendSettings.retry)?(has(self.backendSettings.retry.retryOn)? !has(self.backendSettings.retry.retryOn.httpStatusCodes):true):true):true",message="HTTPStatusCodes is not supported." type OIDCProvider struct { // BackendRefs is used to specify the address of the OIDC Provider. // If the BackendRefs is not specified, The host and port of the OIDC Provider's token endpoint diff --git a/api/v1alpha1/ratelimit_types.go b/api/v1alpha1/ratelimit_types.go index deb859400f88..d7a43751077e 100644 --- a/api/v1alpha1/ratelimit_types.go +++ b/api/v1alpha1/ratelimit_types.go @@ -101,8 +101,6 @@ type RateLimitSelectCondition struct { // meaning, a request MUST match all the specified headers. // At least one of headers or sourceCIDR condition must be specified. // - // +listType=map - // +listMapKey=name // +optional // +kubebuilder:validation:MaxItems=16 Headers []HeaderMatch `json:"headers,omitempty"` @@ -138,13 +136,6 @@ type SourceMatch struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=256 Value string `json:"value"` - - // Invert specifies whether the value match result will be inverted. - // - // +optional - // +kubebuilder:default=false - // +notImplementedHide - Invert *bool `json:"invert,omitempty"` } // HeaderMatch defines the match attributes within the HTTP Headers of the request. diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 8e98b904e351..fe795c833db0 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -669,7 +669,9 @@ type CustomResponse struct { ContentType *string `json:"contentType,omitempty"` // Body of the Custom Response - Body CustomResponseBody `json:"body"` + // + // +optional + Body *CustomResponseBody `json:"body,omitempty"` } // ResponseValueType defines the types of values for the response body supported by Envoy Gateway. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a5e8dc183ff4..696c99259fb2 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1148,7 +1148,11 @@ func (in *CustomResponse) DeepCopyInto(out *CustomResponse) { *out = new(string) **out = **in } - in.Body.DeepCopyInto(&out.Body) + if in.Body != nil { + in, out := &in.Body, &out.Body + *out = new(CustomResponseBody) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomResponse. @@ -2688,6 +2692,36 @@ func (in *HTTPClientTimeout) DeepCopy() *HTTPClientTimeout { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPDirectResponseFilter) DeepCopyInto(out *HTTPDirectResponseFilter) { + *out = *in + if in.ContentType != nil { + in, out := &in.ContentType, &out.ContentType + *out = new(string) + **out = **in + } + if in.Body != nil { + in, out := &in.Body, &out.Body + *out = new(CustomResponseBody) + (*in).DeepCopyInto(*out) + } + if in.StatusCode != nil { + in, out := &in.StatusCode, &out.StatusCode + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPDirectResponseFilter. +func (in *HTTPDirectResponseFilter) DeepCopy() *HTTPDirectResponseFilter { + if in == nil { + return nil + } + out := new(HTTPDirectResponseFilter) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPExtAuthService) DeepCopyInto(out *HTTPExtAuthService) { *out = *in @@ -2820,6 +2854,11 @@ func (in *HTTPRouteFilterSpec) DeepCopyInto(out *HTTPRouteFilterSpec) { *out = new(HTTPURLRewriteFilter) (*in).DeepCopyInto(*out) } + if in.DirectResponse != nil { + in, out := &in.DirectResponse, &out.DirectResponse + *out = new(HTTPDirectResponseFilter) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRouteFilterSpec. @@ -5075,11 +5114,6 @@ func (in *SourceMatch) DeepCopyInto(out *SourceMatch) { *out = new(SourceMatchType) **out = **in } - if in.Invert != nil { - in, out := &in.Invert, &out.Invert - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceMatch. diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml index c0dce5938040..7b2e937312db 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml @@ -727,19 +727,11 @@ spec: type: object maxItems: 16 type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map sourceCIDR: description: |- SourceCIDR is the client IP Address range to match on. At least one of headers or sourceCIDR condition must be specified. properties: - invert: - default: false - description: Invert specifies whether the - value match result will be inverted. - type: boolean type: default: Exact enum: @@ -870,19 +862,11 @@ spec: type: object maxItems: 16 type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map sourceCIDR: description: |- SourceCIDR is the client IP Address range to match on. At least one of headers or sourceCIDR condition must be specified. properties: - invert: - default: false - description: Invert specifies whether the - value match result will be inverted. - type: boolean type: default: Exact enum: @@ -1051,8 +1035,6 @@ spec: description: Content Type of the response. This will be set in the Content-Type header. type: string - required: - - body type: object required: - match diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_httproutefilters.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_httproutefilters.yaml index 7a55ec8871f2..8a75fec42116 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_httproutefilters.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_httproutefilters.yaml @@ -49,6 +49,66 @@ spec: spec: description: Spec defines the desired state of HTTPRouteFilter. properties: + directResponse: + description: HTTPDirectResponseFilter defines the configuration to + return a fixed response. + properties: + body: + description: Body of the Response + properties: + inline: + description: Inline contains the value as an inline string. + type: string + type: + description: Type is the type of method to use to read the + body value. + enum: + - Inline + - ValueRef + type: string + valueRef: + description: |- + ValueRef contains the contents of the body + specified as a local object reference. + Only a reference to ConfigMap is supported. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For example + "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + required: + - type + type: object + contentType: + description: Content Type of the response. This will be set in + the Content-Type header. + type: string + statusCode: + description: |- + Status Code of the HTTP response + If unset, defaults to 200. + type: integer + type: object urlRewrite: description: HTTPURLRewriteFilter define rewrites of HTTP URL components such as path and host diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml index ad2c81818c20..b6a040f8c421 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml @@ -3261,9 +3261,12 @@ spec: x-kubernetes-validations: - message: BackendRefs must be used, backendRef is not supported. rule: '!has(self.backendRef)' - - message: Only one backendRefs is allowed. - rule: 'has(self.backendRefs)? self.backendRefs.size() > 1 : - true' + - message: Retry timeout is not supported. + rule: has(self.backendSettings)? (has(self.backendSettings.retry)?(has(self.backendSettings.retry.perRetry)? + !has(self.backendSettings.retry.perRetry.timeout):true):true):true + - message: HTTPStatusCodes is not supported. + rule: has(self.backendSettings)? (has(self.backendSettings.retry)?(has(self.backendSettings.retry.retryOn)? + !has(self.backendSettings.retry.retryOn.httpStatusCodes):true):true):true redirectURL: description: |- The redirect URL to be used in the OIDC diff --git a/charts/gateway-helm/templates/infra-manager-rbac.yaml b/charts/gateway-helm/templates/infra-manager-rbac.yaml index 02382bc271c5..74c0ec6282e7 100644 --- a/charts/gateway-helm/templates/infra-manager-rbac.yaml +++ b/charts/gateway-helm/templates/infra-manager-rbac.yaml @@ -11,10 +11,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -25,6 +27,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -36,6 +39,7 @@ rules: - create - get - delete + - deletecollection - patch --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/extension-server/go.mod b/examples/extension-server/go.mod index c4a08fecafca..92af0438105b 100644 --- a/examples/extension-server/go.mod +++ b/examples/extension-server/go.mod @@ -4,7 +4,7 @@ go 1.23.1 require ( github.com/envoyproxy/gateway v1.0.2 - github.com/envoyproxy/go-control-plane v0.13.1-0.20240917224354-20d038a70568 + github.com/envoyproxy/go-control-plane v0.13.1 github.com/urfave/cli/v2 v2.27.4 google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.35.1 diff --git a/examples/extension-server/go.sum b/examples/extension-server/go.sum index 7d995a604244..1df719e00af0 100644 --- a/examples/extension-server/go.sum +++ b/examples/extension-server/go.sum @@ -11,8 +11,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.13.1-0.20240917224354-20d038a70568 h1:bUMUmkPtm/z62/8WiVbxtqTK8I7AzXGYn+qB8JAzAXw= -github.com/envoyproxy/go-control-plane v0.13.1-0.20240917224354-20d038a70568/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= +github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= +github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= diff --git a/go.mod b/go.mod index 482af942a9a3..0b7c29eb542e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/docker/cli v27.3.1+incompatible github.com/dominikbraun/graph v0.23.0 - github.com/envoyproxy/go-control-plane v0.13.1-0.20240917224354-20d038a70568 + github.com/envoyproxy/go-control-plane v0.13.1 github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7 github.com/evanphx/json-patch/v5 v5.9.0 github.com/fatih/color v1.17.0 @@ -159,7 +159,7 @@ require ( github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/sys/user v0.2.0 // indirect + github.com/moby/sys/user v0.3.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/go.sum b/go.sum index c5d453f9e0d5..5aa804824a4d 100644 --- a/go.sum +++ b/go.sum @@ -222,8 +222,8 @@ github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.13.1-0.20240917224354-20d038a70568 h1:bUMUmkPtm/z62/8WiVbxtqTK8I7AzXGYn+qB8JAzAXw= -github.com/envoyproxy/go-control-plane v0.13.1-0.20240917224354-20d038a70568/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= +github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= +github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= @@ -601,8 +601,8 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/user v0.2.0 h1:OnpapJsRp25vkhw8TFG6OLJODNh/3rEwRWtJ3kakwRM= -github.com/moby/sys/user v0.2.0/go.mod h1:RYstrcWOJpVh+6qzUqp2bU3eaRpdiQeKGlKitaH0PM8= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 25add4c85419..a4c9d3e97137 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -6,12 +6,15 @@ package cmd import ( + "context" + "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/admin" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/envoygateway/config/loader" extensionregistry "github.com/envoyproxy/gateway/internal/extension/registry" "github.com/envoyproxy/gateway/internal/extension/types" gatewayapirunner "github.com/envoyproxy/gateway/internal/gatewayapi/runner" @@ -51,6 +54,20 @@ func server() error { return err } + ctx := ctrl.SetupSignalHandler() + hook := func(c context.Context, cfg *config.Server) error { + cfg.Logger.Info("Setup runners") + if err := setupRunners(c, cfg); err != nil { + cfg.Logger.Error(err, "failed to setup runners") + return err + } + return nil + } + l := loader.New(cfgPath, cfg, hook) + if err := l.Start(ctx); err != nil { + return err + } + // Init eg admin servers. if err := admin.Init(cfg); err != nil { return err @@ -60,10 +77,10 @@ func server() error { return err } - // init eg runners. - if err := setupRunners(cfg); err != nil { - return err - } + // Wait exit signal + <-ctx.Done() + + cfg.Logger.Info("shutting down") return nil } @@ -110,11 +127,7 @@ func getConfigByPath(cfgPath string) (*config.Server, error) { // setupRunners starts all the runners required for the Envoy Gateway to // fulfill its tasks. -func setupRunners(cfg *config.Server) (err error) { - // TODO - Setup a Config Manager - // https://github.com/envoyproxy/gateway/issues/43 - ctx := ctrl.SetupSignalHandler() - +func setupRunners(ctx context.Context, cfg *config.Server) (err error) { // Setup the Extension Manager var extMgr types.Manager if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { @@ -212,7 +225,7 @@ func setupRunners(cfg *config.Server) (err error) { infraIR.Close() xds.Close() - cfg.Logger.Info("shutting down") + cfg.Logger.Info("runners are shutting down") if extMgr != nil { // Close connections to extension services diff --git a/internal/envoygateway/config/loader/configloader.go b/internal/envoygateway/config/loader/configloader.go new file mode 100644 index 000000000000..9523c7a432ee --- /dev/null +++ b/internal/envoygateway/config/loader/configloader.go @@ -0,0 +1,113 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package loader + +import ( + "context" + "time" + + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/filewatcher" + "github.com/envoyproxy/gateway/internal/logging" +) + +type HookFunc func(c context.Context, cfg *config.Server) error + +type Loader struct { + cfgPath string + cfg *config.Server + logger logging.Logger + cancel context.CancelFunc + hook HookFunc + + w filewatcher.FileWatcher +} + +func New(cfgPath string, cfg *config.Server, f HookFunc) *Loader { + return &Loader{ + cfgPath: cfgPath, + cfg: cfg, + logger: cfg.Logger.WithName("config-loader"), + hook: f, + w: filewatcher.NewWatcher(), + } +} + +func (r *Loader) Start(ctx context.Context) error { + r.runHook() + + if r.cfgPath == "" { + r.logger.Info("no config file provided, skipping config watcher") + return nil + } + + r.logger.Info("watching for changes to the EnvoyGateway configuration", "path", r.cfgPath) + if err := r.w.Add(r.cfgPath); err != nil { + r.logger.Error(err, "failed to add config file to watcher") + return err + } + + go func() { + defer func() { + _ = r.w.Close() + }() + for { + select { + case e := <-r.w.Events(r.cfgPath): + r.logger.Info("received fsnotify events", "name", e.Name, "op", e.Op.String()) + + // Load the config file. + eg, err := config.Decode(r.cfgPath) + if err != nil { + r.logger.Info("failed to decode config file", "name", r.cfgPath, "error", err) + // TODO: add a metric for this? + continue + } + // Set defaults for unset fields + eg.SetEnvoyGatewayDefaults() + r.cfg.EnvoyGateway = eg + // update cfg logger + eg.Logging.SetEnvoyGatewayLoggingDefaults() + r.cfg.Logger = logging.NewLogger(eg.Logging) + + // cancel last + if r.cancel != nil { + r.cancel() + } + + // TODO: we need to make sure that all runners are stopped, before we start the new ones + // Otherwise we might end up with error listening on:8081 + time.Sleep(3 * time.Second) + + r.runHook() + case err := <-r.w.Errors(r.cfgPath): + r.logger.Error(err, "watcher error") + case <-ctx.Done(): + if r.cancel != nil { + r.cancel() + } + return + } + } + }() + + return nil +} + +func (r *Loader) runHook() { + if r.hook == nil { + return + } + + r.logger.Info("running hook") + c, cancel := context.WithCancel(context.TODO()) + r.cancel = cancel + go func(ctx context.Context) { + if err := r.hook(ctx, r.cfg); err != nil { + r.logger.Error(err, "hook error") + } + }(c) +} diff --git a/internal/envoygateway/config/loader/configloader_test.go b/internal/envoygateway/config/loader/configloader_test.go new file mode 100644 index 000000000000..d0420df0f5f3 --- /dev/null +++ b/internal/envoygateway/config/loader/configloader_test.go @@ -0,0 +1,59 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package loader + +import ( + "context" + _ "embed" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/gateway/internal/envoygateway/config" +) + +var ( + //go:embed testdata/default.yaml + defaultConfig string + //go:embed testdata/enable-redis.yaml + redisConfig string +) + +func TestConfigLoader(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "envoy-gateway-configloader-test") + require.NoError(t, err) + defer func(path string) { + _ = os.RemoveAll(path) + }(tmpDir) + + cfgPath := tmpDir + "/config.yaml" + require.NoError(t, os.WriteFile(cfgPath, []byte(defaultConfig), 0o600)) + s, err := config.New() + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.TODO()) + defer func() { + cancel() + }() + + changed := 0 + loader := New(cfgPath, s, func(_ context.Context, cfg *config.Server) error { + changed++ + t.Logf("config changed %d times", changed) + if changed > 1 { + cancel() + } + return nil + }) + + require.NoError(t, loader.Start(ctx)) + go func() { + _ = os.WriteFile(cfgPath, []byte(redisConfig), 0o600) + }() + + <-ctx.Done() +} diff --git a/internal/envoygateway/config/loader/testdata/default.yaml b/internal/envoygateway/config/loader/testdata/default.yaml new file mode 100644 index 000000000000..20463f848e13 --- /dev/null +++ b/internal/envoygateway/config/loader/testdata/default.yaml @@ -0,0 +1,24 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +logging: + level: + default: info +provider: + kubernetes: + rateLimitDeployment: + container: + image: docker.io/envoyproxy/ratelimit:master + patch: + type: StrategicMerge + value: + spec: + template: + spec: + containers: + - imagePullPolicy: IfNotPresent + name: envoy-ratelimit + shutdownManager: + image: docker.io/envoyproxy/gateway-dev:latest + type: Kubernetes diff --git a/internal/envoygateway/config/loader/testdata/enable-redis.yaml b/internal/envoygateway/config/loader/testdata/enable-redis.yaml new file mode 100644 index 000000000000..ed2218ab5abf --- /dev/null +++ b/internal/envoygateway/config/loader/testdata/enable-redis.yaml @@ -0,0 +1,14 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +provider: + type: Kubernetes +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +extensionApis: + enableEnvoyPatchPolicy: true + enableBackend: true +rateLimit: + backend: + type: Redis + redis: + url: redis.redis-system.svc.cluster.local:6379 diff --git a/internal/filewatcher/filewatcher.go b/internal/filewatcher/filewatcher.go new file mode 100644 index 000000000000..4fce5e9aba41 --- /dev/null +++ b/internal/filewatcher/filewatcher.go @@ -0,0 +1,179 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package filewatcher + +import ( + "errors" + "fmt" + "path/filepath" + "sync" + + "github.com/fsnotify/fsnotify" +) + +// FileWatcher is an interface that watches a set of files, +// delivering events to related channel. +type FileWatcher interface { + Add(path string) error + Remove(path string) error + Close() error + Events(path string) chan fsnotify.Event + Errors(path string) chan error +} + +type fileWatcher struct { + mu sync.RWMutex + + // The watcher maintain a map of workers, + // keyed by watched dir (parent dir of watched files). + workers map[string]*workerState + + funcs *patchTable +} + +type workerState struct { + worker *worker + count int +} + +// functions that can be replaced in a test setting +type patchTable struct { + newWatcher func() (*fsnotify.Watcher, error) + addWatcherPath func(*fsnotify.Watcher, string) error +} + +// NewWatcher return with a FileWatcher instance that implemented with fsnotify. +func NewWatcher() FileWatcher { + return &fileWatcher{ + workers: map[string]*workerState{}, + + // replaceable functions for tests + funcs: &patchTable{ + newWatcher: fsnotify.NewWatcher, + addWatcherPath: func(watcher *fsnotify.Watcher, path string) error { + return watcher.Add(path) + }, + }, + } +} + +// Close releases all resources associated with the watcher +func (fw *fileWatcher) Close() error { + fw.mu.Lock() + defer fw.mu.Unlock() + + for _, ws := range fw.workers { + ws.worker.terminate() + } + fw.workers = nil + + return nil +} + +// Add a path to watch +func (fw *fileWatcher) Add(path string) error { + fw.mu.Lock() + defer fw.mu.Unlock() + + ws, cleanedPath, _, err := fw.getWorker(path) + if err != nil { + return err + } + + if err = ws.worker.addPath(cleanedPath); err == nil { + ws.count++ + } + + return err +} + +// Stop watching a path +func (fw *fileWatcher) Remove(path string) error { + fw.mu.Lock() + defer fw.mu.Unlock() + + ws, cleanedPath, parentPath, err := fw.getWorker(path) + if err != nil { + return err + } + + if err = ws.worker.removePath(cleanedPath); err == nil { + ws.count-- + if ws.count == 0 { + ws.worker.terminate() + delete(fw.workers, parentPath) + } + } + + return err +} + +// Events returns an event notification channel for a path +func (fw *fileWatcher) Events(path string) chan fsnotify.Event { + fw.mu.RLock() + defer fw.mu.RUnlock() + + ws, cleanedPath, err := fw.findWorker(path) + if err != nil { + return nil + } + + return ws.worker.eventChannel(cleanedPath) +} + +// Errors returns an error notification channel for a path +func (fw *fileWatcher) Errors(path string) chan error { + fw.mu.RLock() + defer fw.mu.RUnlock() + + ws, cleanedPath, err := fw.findWorker(path) + if err != nil { + return nil + } + + return ws.worker.errorChannel(cleanedPath) +} + +func (fw *fileWatcher) getWorker(path string) (*workerState, string, string, error) { + if fw.workers == nil { + return nil, "", "", errors.New("using a closed watcher") + } + + cleanedPath := filepath.Clean(path) + parentPath, _ := filepath.Split(cleanedPath) + + ws, workerExists := fw.workers[parentPath] + if !workerExists { + wk, err := newWorker(parentPath, fw.funcs) + if err != nil { + return nil, "", "", err + } + + ws = &workerState{ + worker: wk, + } + + fw.workers[parentPath] = ws + } + + return ws, cleanedPath, parentPath, nil +} + +func (fw *fileWatcher) findWorker(path string) (*workerState, string, error) { + if fw.workers == nil { + return nil, "", errors.New("using a closed watcher") + } + + cleanedPath := filepath.Clean(path) + parentPath, _ := filepath.Split(cleanedPath) + + ws, workerExists := fw.workers[parentPath] + if !workerExists { + return nil, "", fmt.Errorf("no path registered for %s", path) + } + + return ws, cleanedPath, nil +} diff --git a/internal/filewatcher/filewatcher_test.go b/internal/filewatcher/filewatcher_test.go new file mode 100644 index 000000000000..5230d7c05ad3 --- /dev/null +++ b/internal/filewatcher/filewatcher_test.go @@ -0,0 +1,321 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package filewatcher + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path" + "runtime" + "sync" + "testing" + + "github.com/fsnotify/fsnotify" + "github.com/stretchr/testify/require" +) + +func newWatchFile(t *testing.T) string { + watchDir := t.TempDir() + watchFile := path.Join(watchDir, "test.conf") + err := os.WriteFile(watchFile, []byte("foo: bar\n"), 0o600) + require.NoError(t, err) + + return watchFile +} + +func newWatchFileThatDoesNotExist(t *testing.T) string { + watchDir := t.TempDir() + + watchFile := path.Join(watchDir, "test.conf") + + return watchFile +} + +// newTwoWatchFile returns with two watch files that exist in the same base dir. +func newTwoWatchFile(t *testing.T) (string, string) { + watchDir := t.TempDir() + + watchFile1 := path.Join(watchDir, "test1.conf") + err := os.WriteFile(watchFile1, []byte("foo: bar\n"), 0o600) + require.NoError(t, err) + + watchFile2 := path.Join(watchDir, "test2.conf") + err = os.WriteFile(watchFile2, []byte("foo: baz\n"), 0o600) + require.NoError(t, err) + + return watchFile1, watchFile2 +} + +// newSymlinkedWatchFile simulates the behavior of k8s configmap/secret. +// Path structure looks like: +// +// /test.conf +// ^ +// | +// +// /data/test.conf +// +// ^ +// | +// +// /data1/test.conf +func newSymlinkedWatchFile(t *testing.T) (string, string) { + watchDir := t.TempDir() + + dataDir1 := path.Join(watchDir, "data1") + err := os.Mkdir(dataDir1, 0o777) + require.NoError(t, err) + + realTestFile := path.Join(dataDir1, "test.conf") + t.Logf("Real test file location: %s\n", realTestFile) + err = os.WriteFile(realTestFile, []byte("foo: bar\n"), 0o600) + require.NoError(t, err) + + // Now, symlink the tmp `data1` dir to `data` in the baseDir + require.NoError(t, os.Symlink(dataDir1, path.Join(watchDir, "data"))) + // And link the `/datadir/test.conf` to `/test.conf` + watchFile := path.Join(watchDir, "test.conf") + require.NoError(t, os.Symlink(path.Join(watchDir, "data", "test.conf"), watchFile)) + t.Logf("Watch file location: %s\n", path.Join(watchDir, "test.conf")) + return watchDir, watchFile +} + +func TestWatchFile(t *testing.T) { + t.Run("file content changed", func(t *testing.T) { + // Given a file being watched + watchFile := newWatchFile(t) + _, err := os.Stat(watchFile) + require.NoError(t, err) + + w := NewWatcher() + require.NoError(t, w.Add(watchFile)) + events := w.Events(watchFile) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + <-events + wg.Done() + }() + + // Overwriting the file and waiting its event to be received. + err = os.WriteFile(watchFile, []byte("foo: baz\n"), 0o600) + require.NoError(t, err) + wg.Wait() + + _ = w.Close() + }) + + t.Run("link to real file changed (for k8s configmap/secret path)", func(t *testing.T) { + // skip if not executed on Linux + if runtime.GOOS != "linux" { + t.Skip("Skipping test as symlink replacements don't work on non-linux environment...") + } + + watchDir, watchFile := newSymlinkedWatchFile(t) + + w := NewWatcher() + require.NoError(t, w.Add(watchFile)) + events := w.Events(watchFile) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + <-events + wg.Done() + }() + + // Link to another `test.conf` file + dataDir2 := path.Join(watchDir, "data2") + err := os.Mkdir(dataDir2, 0o777) + require.NoError(t, err) + + watchFile2 := path.Join(dataDir2, "test.conf") + err = os.WriteFile(watchFile2, []byte("foo: baz\n"), 0o600) + require.NoError(t, err) + + // change the symlink using the `ln -sfn` command + err = exec.Command("ln", "-sfn", dataDir2, path.Join(watchDir, "data")).Run() + require.NoError(t, err) + + // Wait its event to be received. + wg.Wait() + + _ = w.Close() + }) + + t.Run("file added later", func(t *testing.T) { + // Given a file being watched + watchFile := newWatchFileThatDoesNotExist(t) + + w := NewWatcher() + require.NoError(t, w.Add(watchFile)) + events := w.Events(watchFile) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + <-events + wg.Done() + }() + + // Overwriting the file and waiting its event to be received. + err := os.WriteFile(watchFile, []byte("foo: baz\n"), 0o600) + require.NoError(t, err) + wg.Wait() + + _ = w.Close() + }) +} + +func TestWatcherLifecycle(t *testing.T) { + watchFile1, watchFile2 := newTwoWatchFile(t) + + w := NewWatcher() + + // Validate Add behavior + err := w.Add(watchFile1) + require.NoError(t, err) + err = w.Add(watchFile2) + require.NoError(t, err) + + // Validate events and errors channel are fulfilled. + events1 := w.Events(watchFile1) + require.NotNil(t, events1) + events2 := w.Events(watchFile2) + require.NotNil(t, events2) + + errors1 := w.Errors(watchFile1) + require.NotNil(t, errors1) + errors2 := w.Errors(watchFile2) + require.NotNil(t, errors2) + + // Validate Remove behavior + err = w.Remove(watchFile1) + require.NoError(t, err) + events1 = w.Events(watchFile1) + require.Nil(t, events1) + errors1 = w.Errors(watchFile1) + require.Nil(t, errors1) + events2 = w.Events(watchFile2) + require.NotNil(t, events2) + errors2 = w.Errors(watchFile2) + require.NotNil(t, errors2) + + fmt.Printf("2\n") + // Validate Close behavior + err = w.Close() + require.NoError(t, err) + events1 = w.Events(watchFile1) + require.Nil(t, events1) + errors1 = w.Errors(watchFile1) + require.Nil(t, errors1) + events2 = w.Events(watchFile2) + require.Nil(t, events2) + errors2 = w.Errors(watchFile2) + require.Nil(t, errors2) +} + +func TestErrors(t *testing.T) { + w := NewWatcher() + + if ch := w.Errors("XYZ"); ch != nil { + t.Error("Expected no channel") + } + + if ch := w.Events("XYZ"); ch != nil { + t.Error("Expected no channel") + } + + name := newWatchFile(t) + _ = w.Add(name) + _ = w.Remove(name) + + if ch := w.Errors("XYZ"); ch != nil { + t.Error("Expected no channel") + } + + if ch := w.Events(name); ch != nil { + t.Error("Expected no channel") + } + + _ = w.Close() + + if err := w.Add(name); err == nil { + t.Error("Expecting error") + } + + if err := w.Remove(name); err == nil { + t.Error("Expecting error") + } + + if ch := w.Errors(name); ch != nil { + t.Error("Expecting nil") + } + + if ch := w.Events(name); ch != nil { + t.Error("Expecting nil") + } +} + +func TestBadWatcher(t *testing.T) { + w := NewWatcher() + w.(*fileWatcher).funcs.newWatcher = func() (*fsnotify.Watcher, error) { + return nil, errors.New("FOOBAR") + } + + name := newWatchFile(t) + if err := w.Add(name); err == nil { + t.Errorf("Expecting error, got nil") + } + if err := w.Close(); err != nil { + t.Errorf("Expecting nil, got %v", err) + } +} + +func TestBadAddWatcher(t *testing.T) { + w := NewWatcher() + w.(*fileWatcher).funcs.addWatcherPath = func(*fsnotify.Watcher, string) error { + return errors.New("FOOBAR") + } + + name := newWatchFile(t) + if err := w.Add(name); err == nil { + t.Errorf("Expecting error, got nil") + } + if err := w.Close(); err != nil { + t.Errorf("Expecting nil, got %v", err) + } +} + +func TestDuplicateAdd(t *testing.T) { + w := NewWatcher() + + name := newWatchFile(t) + + if err := w.Add(name); err != nil { + t.Errorf("Expecting nil, got %v", err) + } + + if err := w.Add(name); err == nil { + t.Errorf("Expecting error, got nil") + } + + _ = w.Close() +} + +func TestBogusRemove(t *testing.T) { + w := NewWatcher() + + name := newWatchFile(t) + if err := w.Remove(name); err == nil { + t.Errorf("Expecting error, got nil") + } + + _ = w.Close() +} diff --git a/internal/filewatcher/worker.go b/internal/filewatcher/worker.go new file mode 100644 index 000000000000..6ae9c9f77bae --- /dev/null +++ b/internal/filewatcher/worker.go @@ -0,0 +1,256 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package filewatcher + +import ( + "bufio" + "bytes" + "crypto/sha256" + "fmt" + "io" + "os" + "sync" + + "github.com/fsnotify/fsnotify" +) + +type worker struct { + mu sync.RWMutex + + // watcher is an fsnotify watcher that watches the parent + // dir of watchedFiles. + dirWatcher *fsnotify.Watcher + + // The worker maintains a map of channels keyed by watched file path. + // The worker watches parent path of given path, + // and filters out events of given path, then redirect + // to the result channel. + // Note that for symlink files, the content in received events + // do not have to be related to the file itself. + watchedFiles map[string]*fileTracker + + // tracker lifecycle + retireTrackerCh chan *fileTracker + + // tells the worker to exit + terminateCh chan bool +} + +type fileTracker struct { + events chan fsnotify.Event + errors chan error + + // Hash sum to indicate if a file has been updated. + hash []byte +} + +func newWorker(path string, funcs *patchTable) (*worker, error) { + dirWatcher, err := funcs.newWatcher() + if err != nil { + return nil, err + } + + if err = funcs.addWatcherPath(dirWatcher, path); err != nil { + _ = dirWatcher.Close() + return nil, err + } + + wk := &worker{ + dirWatcher: dirWatcher, + watchedFiles: make(map[string]*fileTracker), + retireTrackerCh: make(chan *fileTracker), + terminateCh: make(chan bool), + } + + go wk.listen() + + return wk, nil +} + +func (wk *worker) listen() { + wk.loop() + + _ = wk.dirWatcher.Close() + + // drain any retiring trackers that may be pending + wk.drainRetiringTrackers() + + // clean up the rest + for _, ft := range wk.watchedFiles { + retireTracker(ft) + } +} + +func (wk *worker) loop() { + for { + select { + case event := <-wk.dirWatcher.Events: + // work on a copy of the watchedFiles map, so that we don't interfere + // with the caller's use of the map + for path, ft := range wk.getTrackers() { + if ft.events == nil { + // tracker has been retired, skip it + continue + } + + sum := getHashSum(path) + if !bytes.Equal(sum, ft.hash) { + ft.hash = sum + + select { + case ft.events <- event: + // nothing to do + + case ft := <-wk.retireTrackerCh: + retireTracker(ft) + + case <-wk.terminateCh: + return + } + } + } + + case err := <-wk.dirWatcher.Errors: + for _, ft := range wk.getTrackers() { + if ft.errors == nil { + // tracker has been retired, skip it + continue + } + + select { + case ft.errors <- err: + // nothing to do + + case ft := <-wk.retireTrackerCh: + retireTracker(ft) + + case <-wk.terminateCh: + return + } + } + + case ft := <-wk.retireTrackerCh: + retireTracker(ft) + + case <-wk.terminateCh: + return + } + } +} + +// used only by the worker goroutine +func (wk *worker) drainRetiringTrackers() { + // cleanup any trackers that were in the process + // of being retired, but didn't get processed due + // to termination + for { + select { + case ft := <-wk.retireTrackerCh: + retireTracker(ft) + default: + return + } + } +} + +// make a local copy of the set of trackers to avoid contention with callers +// used only by the worker goroutine +func (wk *worker) getTrackers() map[string]*fileTracker { + wk.mu.RLock() + + result := make(map[string]*fileTracker, len(wk.watchedFiles)) + for k, v := range wk.watchedFiles { + result[k] = v + } + + wk.mu.RUnlock() + return result +} + +// used only by the worker goroutine +func retireTracker(ft *fileTracker) { + close(ft.events) + close(ft.errors) + ft.events = nil + ft.errors = nil +} + +func (wk *worker) terminate() { + wk.terminateCh <- true +} + +func (wk *worker) addPath(path string) error { + wk.mu.Lock() + + ft := wk.watchedFiles[path] + if ft != nil { + wk.mu.Unlock() + return fmt.Errorf("path %s is already being watched", path) + } + + ft = &fileTracker{ + events: make(chan fsnotify.Event), + errors: make(chan error), + hash: getHashSum(path), + } + + wk.watchedFiles[path] = ft + wk.mu.Unlock() + + return nil +} + +func (wk *worker) removePath(path string) error { + wk.mu.Lock() + + ft := wk.watchedFiles[path] + if ft == nil { + wk.mu.Unlock() + return fmt.Errorf("path %s not found", path) + } + + delete(wk.watchedFiles, path) + wk.mu.Unlock() + + wk.retireTrackerCh <- ft + return nil +} + +func (wk *worker) eventChannel(path string) chan fsnotify.Event { + wk.mu.RLock() + defer wk.mu.RUnlock() + + if ft := wk.watchedFiles[path]; ft != nil { + return ft.events + } + + return nil +} + +func (wk *worker) errorChannel(path string) chan error { + wk.mu.RLock() + defer wk.mu.RUnlock() + + if ft := wk.watchedFiles[path]; ft != nil { + return ft.errors + } + + return nil +} + +// gets the hash of the given file, or nil if there's a problem +func getHashSum(file string) []byte { + f, err := os.Open(file) + if err != nil { + return nil + } + defer f.Close() + r := bufio.NewReader(f) + + h := sha256.New() + _, _ = io.Copy(h, r) + return h.Sum(nil) +} diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go index 12453ea18265..b8f289a9df0e 100644 --- a/internal/gatewayapi/backendtrafficpolicy.go +++ b/internal/gatewayapi/backendtrafficpolicy.go @@ -16,7 +16,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/utils/ptr" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -324,7 +323,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen errs = errors.Join(errs, err) } if policy.Spec.Retry != nil { - rt = t.buildRetry(policy) + rt = buildRetry(policy.Spec.Retry) } if to, err = buildClusterSettingsTimeout(policy.Spec.ClusterSettings, nil); err != nil { err = perr.WithMessage(err, "Timeout") @@ -460,7 +459,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back errs = errors.Join(errs, err) } if policy.Spec.Retry != nil { - rt = t.buildRetry(policy) + rt = buildRetry(policy.Spec.Retry) } if ct, err = buildClusterSettingsTimeout(policy.Spec.ClusterSettings, nil); err != nil { err = perr.WithMessage(err, "Timeout") @@ -812,67 +811,6 @@ func (t *Translator) buildFaultInjection(policy *egv1a1.BackendTrafficPolicy) *i return fi } -func (t *Translator) buildRetry(policy *egv1a1.BackendTrafficPolicy) *ir.Retry { - var rt *ir.Retry - if policy.Spec.Retry != nil { - prt := policy.Spec.Retry - rt = &ir.Retry{} - - if prt.NumRetries != nil { - rt.NumRetries = ptr.To(uint32(*prt.NumRetries)) - } - - if prt.RetryOn != nil { - ro := &ir.RetryOn{} - bro := false - if prt.RetryOn.HTTPStatusCodes != nil { - ro.HTTPStatusCodes = makeIrStatusSet(prt.RetryOn.HTTPStatusCodes) - bro = true - } - - if prt.RetryOn.Triggers != nil { - ro.Triggers = makeIrTriggerSet(prt.RetryOn.Triggers) - bro = true - } - - if bro { - rt.RetryOn = ro - } - } - - if prt.PerRetry != nil { - pr := &ir.PerRetryPolicy{} - bpr := false - - if prt.PerRetry.Timeout != nil { - pr.Timeout = prt.PerRetry.Timeout - bpr = true - } - - if prt.PerRetry.BackOff != nil { - if prt.PerRetry.BackOff.MaxInterval != nil || prt.PerRetry.BackOff.BaseInterval != nil { - bop := &ir.BackOffPolicy{} - if prt.PerRetry.BackOff.MaxInterval != nil { - bop.MaxInterval = prt.PerRetry.BackOff.MaxInterval - } - - if prt.PerRetry.BackOff.BaseInterval != nil { - bop.BaseInterval = prt.PerRetry.BackOff.BaseInterval - } - pr.BackOff = bop - bpr = true - } - } - - if bpr { - rt.PerRetry = pr - } - } - } - - return rt -} - func makeIrStatusSet(in []egv1a1.HTTPStatus) []ir.HTTPStatus { statusSet := sets.NewInt() for _, r := range in { diff --git a/internal/gatewayapi/clustersettings.go b/internal/gatewayapi/clustersettings.go index 742b026249e1..260c1db84d3c 100644 --- a/internal/gatewayapi/clustersettings.go +++ b/internal/gatewayapi/clustersettings.go @@ -71,6 +71,8 @@ func translateTrafficFeatures(policy *egv1a1.ClusterSettings) (*ir.TrafficFeatur ret.HTTP2 = h2 } + ret.Retry = buildRetry(policy.Retry) + // If nothing was set in any of the above calls, return nil instead of an empty // container var empty ir.TrafficFeatures @@ -507,3 +509,64 @@ func translateDNS(policy egv1a1.ClusterSettings) *ir.DNS { DNSRefreshRate: policy.DNS.DNSRefreshRate, } } + +func buildRetry(r *egv1a1.Retry) *ir.Retry { + if r == nil { + return nil + } + + rt := &ir.Retry{} + + if r.NumRetries != nil { + rt.NumRetries = ptr.To(uint32(*r.NumRetries)) + } + + if r.RetryOn != nil { + ro := &ir.RetryOn{} + bro := false + if r.RetryOn.HTTPStatusCodes != nil { + ro.HTTPStatusCodes = makeIrStatusSet(r.RetryOn.HTTPStatusCodes) + bro = true + } + + if r.RetryOn.Triggers != nil { + ro.Triggers = makeIrTriggerSet(r.RetryOn.Triggers) + bro = true + } + + if bro { + rt.RetryOn = ro + } + } + + if r.PerRetry != nil { + pr := &ir.PerRetryPolicy{} + bpr := false + + if r.PerRetry.Timeout != nil { + pr.Timeout = r.PerRetry.Timeout + bpr = true + } + + if r.PerRetry.BackOff != nil { + if r.PerRetry.BackOff.MaxInterval != nil || r.PerRetry.BackOff.BaseInterval != nil { + bop := &ir.BackOffPolicy{} + if r.PerRetry.BackOff.MaxInterval != nil { + bop.MaxInterval = r.PerRetry.BackOff.MaxInterval + } + + if r.PerRetry.BackOff.BaseInterval != nil { + bop.BaseInterval = r.PerRetry.BackOff.BaseInterval + } + pr.BackOff = bop + bpr = true + } + } + + if bpr { + rt.PerRetry = pr + } + } + + return rt +} diff --git a/internal/gatewayapi/envoyextensionpolicy.go b/internal/gatewayapi/envoyextensionpolicy.go index 473945875295..5e61f2eb3aa6 100644 --- a/internal/gatewayapi/envoyextensionpolicy.go +++ b/internal/gatewayapi/envoyextensionpolicy.go @@ -411,7 +411,7 @@ func (t *Translator) buildExtProcs(policy *egv1a1.EnvoyExtensionPolicy, resource for idx, ep := range policy.Spec.ExtProc { name := irConfigNameForExtProc(policy, idx) - extProcIR, err := t.buildExtProc(name, utils.NamespacedName(policy), ep, idx, resources, envoyProxy) + extProcIR, err := t.buildExtProc(name, policy, ep, idx, resources, envoyProxy) if err != nil { return nil, err } @@ -422,59 +422,33 @@ func (t *Translator) buildExtProcs(policy *egv1a1.EnvoyExtensionPolicy, resource func (t *Translator) buildExtProc( name string, - policyNamespacedName types.NamespacedName, + policy *egv1a1.EnvoyExtensionPolicy, extProc egv1a1.ExtProc, extProcIdx int, resources *resource.Resources, envoyProxy *egv1a1.EnvoyProxy, ) (*ir.ExtProc, error) { var ( - ds *ir.DestinationSetting + rd *ir.RouteDestination authority string err error ) - var dsl []*ir.DestinationSetting - for i := range extProc.BackendRefs { - if err = t.validateExtServiceBackendReference( - &extProc.BackendRefs[i].BackendObjectReference, - policyNamespacedName.Namespace, - egv1a1.KindEnvoyExtensionPolicy, - resources); err != nil { - return nil, err - } - - ds, err = t.processExtServiceDestination( - &extProc.BackendRefs[i], - policyNamespacedName, - egv1a1.KindEnvoyExtensionPolicy, - ir.GRPC, - resources, - envoyProxy, - ) - if err != nil { - return nil, err - } - - dsl = append(dsl, ds) - } - - rd := ir.RouteDestination{ - Name: irIndexedExtServiceDestinationName(policyNamespacedName, egv1a1.KindEnvoyExtensionPolicy, extProcIdx), - Settings: dsl, + if rd, err = t.translateExtServiceBackendRefs(policy, extProc.BackendRefs, ir.GRPC, resources, envoyProxy, extProcIdx); err != nil { + return nil, err } if extProc.BackendRefs[0].Port != nil { authority = fmt.Sprintf( "%s.%s:%d", extProc.BackendRefs[0].Name, - NamespaceDerefOr(extProc.BackendRefs[0].Namespace, policyNamespacedName.Namespace), + NamespaceDerefOr(extProc.BackendRefs[0].Namespace, policy.Namespace), *extProc.BackendRefs[0].Port) } else { authority = fmt.Sprintf( "%s.%s", extProc.BackendRefs[0].Name, - NamespaceDerefOr(extProc.BackendRefs[0].Namespace, policyNamespacedName.Namespace)) + NamespaceDerefOr(extProc.BackendRefs[0].Namespace, policy.Namespace)) } traffic, err := translateTrafficFeatures(extProc.BackendCluster.BackendSettings) @@ -484,7 +458,7 @@ func (t *Translator) buildExtProc( extProcIR := &ir.ExtProc{ Name: name, - Destination: rd, + Destination: *rd, Traffic: traffic, Authority: authority, } diff --git a/internal/gatewayapi/ext_service.go b/internal/gatewayapi/ext_service.go index f72296c3187f..93c43b5e5a6a 100644 --- a/internal/gatewayapi/ext_service.go +++ b/internal/gatewayapi/ext_service.go @@ -12,15 +12,66 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/gatewayapi/resource" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/utils" ) -// TODO: zhaohuabing combine this function with the one in the route translator +// translateExtServiceBackendRefs translates external service backend references to route destinations. +func (t *Translator) translateExtServiceBackendRefs( + policy client.Object, + backendRefs []egv1a1.BackendRef, + protocol ir.AppProtocol, + resources *resource.Resources, + envoyProxy *egv1a1.EnvoyProxy, + index int, // index is used to differentiate between multiple external services in the same policy +) (*ir.RouteDestination, error) { + var ( + rs *ir.RouteDestination + ds []*ir.DestinationSetting + err error + ) + + if len(backendRefs) == 0 { + return nil, errors.New("no backendRefs found for external service") + } + + pnn := utils.NamespacedName(policy) + for _, backendRef := range backendRefs { + if err = t.validateExtServiceBackendReference( + &backendRef.BackendObjectReference, + policy.GetNamespace(), + policy.GetObjectKind().GroupVersionKind().Kind, + resources); err != nil { + return nil, err + } + + var extServiceDest *ir.DestinationSetting + if extServiceDest, err = t.processExtServiceDestination( + &backendRef, + pnn, + policy.GetObjectKind().GroupVersionKind().Kind, + protocol, + resources, + envoyProxy, + ); err != nil { + return nil, err + } + ds = append(ds, extServiceDest) + } + + rs = &ir.RouteDestination{ + Name: irIndexedExtServiceDestinationName(pnn, policy.GetObjectKind().GroupVersionKind().Kind, index), + Settings: ds, + } + return rs, nil +} + func (t *Translator) processExtServiceDestination( backendRef *egv1a1.BackendRef, policyNamespacedName types.NamespacedName, diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 13c5987a3acc..974ec38cdd24 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -1727,9 +1727,18 @@ func (t *Translator) processBackendDestinationSetting(backendRef gwapiv1.Backend } } - return &ir.DestinationSetting{ + ds := &ir.DestinationSetting{ Protocol: dstProtocol, Endpoints: dstEndpoints, AddressType: dstAddrType, } + + if backend.Spec.Fallback != nil { + // set only the secondary priority, the backend defaults to a primary priority if unset. + if ptr.Deref(backend.Spec.Fallback, false) { + ds.Priority = ptr.To(uint32(1)) + } + } + + return ds } diff --git a/internal/gatewayapi/securitypolicy.go b/internal/gatewayapi/securitypolicy.go index 2ab9670d5015..c9289bbcfb3e 100644 --- a/internal/gatewayapi/securitypolicy.go +++ b/internal/gatewayapi/securitypolicy.go @@ -329,7 +329,6 @@ func (t *Translator) translateSecurityPolicyForRoute( var ( cors *ir.CORS jwt *ir.JWT - oidc *ir.OIDC basicAuth *ir.BasicAuth authorization *ir.Authorization err, errs error @@ -343,15 +342,6 @@ func (t *Translator) translateSecurityPolicyForRoute( jwt = t.buildJWT(policy.Spec.JWT) } - if policy.Spec.OIDC != nil { - if oidc, err = t.buildOIDC( - policy, - resources); err != nil { - err = perr.WithMessage(err, "OIDC") - errs = errors.Join(errs, err) - } - } - if policy.Spec.BasicAuth != nil { if basicAuth, err = t.buildBasicAuth( policy, @@ -388,6 +378,18 @@ func (t *Translator) translateSecurityPolicyForRoute( errs = errors.Join(errs, err) } } + + var oidc *ir.OIDC + if policy.Spec.OIDC != nil { + if oidc, err = t.buildOIDC( + policy, + resources, + gtwCtx.envoyProxy); err != nil { + err = perr.WithMessage(err, "OIDC") + errs = errors.Join(errs, err) + } + } + irKey := t.getIRKey(gtwCtx.Gateway) for _, listener := range parentRefCtx.listeners { irListener := xdsIR[irKey].GetHTTPListener(irListenerName(listener)) @@ -445,7 +447,8 @@ func (t *Translator) translateSecurityPolicyForGateway( if policy.Spec.OIDC != nil { if oidc, err = t.buildOIDC( policy, - resources); err != nil { + resources, + gateway.envoyProxy); err != nil { err = perr.WithMessage(err, "OIDC") errs = errors.Join(errs, err) } @@ -566,19 +569,30 @@ func (t *Translator) buildJWT(jwt *egv1a1.JWT) *ir.JWT { func (t *Translator) buildOIDC( policy *egv1a1.SecurityPolicy, resources *resource.Resources, + envoyProxy *egv1a1.EnvoyProxy, ) (*ir.OIDC, error) { var ( - oidc = policy.Spec.OIDC - clientSecret *corev1.Secret - provider *ir.OIDCProvider - err error + oidc = policy.Spec.OIDC + provider *ir.OIDCProvider + clientSecret *corev1.Secret + redirectURL = defaultRedirectURL + redirectPath = defaultRedirectPath + logoutPath = defaultLogoutPath + forwardAccessToken = defaultForwardAccessToken + refreshToken = defaultRefreshToken + err error ) + if provider, err = t.buildOIDCProvider(policy, resources, envoyProxy); err != nil { + return nil, err + } + from := crossNamespaceFrom{ group: egv1a1.GroupName, kind: resource.KindSecurityPolicy, namespace: policy.Namespace, } + if clientSecret, err = t.validateSecretRef( false, from, oidc.ClientSecret, resources); err != nil { return nil, err @@ -591,25 +605,8 @@ func (t *Translator) buildOIDC( clientSecret.Namespace, clientSecret.Name) } - // Discover the token and authorization endpoints from the issuer's - // well-known url if not explicitly specified - if provider, err = discoverEndpointsFromIssuer(&oidc.Provider); err != nil { - return nil, err - } - - if err = validateTokenEndpoint(provider.TokenEndpoint); err != nil { - return nil, err - } scopes := appendOpenidScopeIfNotExist(oidc.Scopes) - var ( - redirectURL = defaultRedirectURL - redirectPath = defaultRedirectPath - logoutPath = defaultLogoutPath - forwardAccessToken = defaultForwardAccessToken - refreshToken = defaultRefreshToken - ) - if oidc.RedirectURL != nil { path, err := extractRedirectPath(*oidc.RedirectURL) if err != nil { @@ -668,6 +665,62 @@ func (t *Translator) buildOIDC( }, nil } +func (t *Translator) buildOIDCProvider(policy *egv1a1.SecurityPolicy, resources *resource.Resources, envoyProxy *egv1a1.EnvoyProxy) (*ir.OIDCProvider, error) { + var ( + provider = policy.Spec.OIDC.Provider + tokenEndpoint string + authorizationEndpoint string + protocol ir.AppProtocol + rd *ir.RouteDestination + traffic *ir.TrafficFeatures + err error + ) + + // Discover the token and authorization endpoints from the issuer's + // well-known url if not explicitly specified + if provider.TokenEndpoint == nil || provider.AuthorizationEndpoint == nil { + tokenEndpoint, authorizationEndpoint, err = fetchEndpointsFromIssuer(provider.Issuer) + if err != nil { + return nil, fmt.Errorf("error fetching endpoints from issuer: %w", err) + } + } else { + tokenEndpoint = *provider.TokenEndpoint + authorizationEndpoint = *provider.AuthorizationEndpoint + } + + if err = validateTokenEndpoint(tokenEndpoint); err != nil { + return nil, err + } + + u, err := url.Parse(tokenEndpoint) + if err != nil { + return nil, err + } + + if u.Scheme == "https" { + protocol = ir.HTTPS + } else { + protocol = ir.HTTP + } + + if len(provider.BackendRefs) > 0 { + if rd, err = t.translateExtServiceBackendRefs(policy, provider.BackendRefs, protocol, resources, envoyProxy, 0); err != nil { + return nil, err + } + } + + if traffic, err = translateTrafficFeatures(provider.BackendSettings); err != nil { + return nil, err + } + + return &ir.OIDCProvider{ + Destination: rd, + Traffic: traffic, + AuthorizationEndpoint: authorizationEndpoint, + TokenEndpoint: tokenEndpoint, + }, nil +} + func extractRedirectPath(redirectURL string) (string, error) { schemeDelimiter := strings.Index(redirectURL, "://") if schemeDelimiter <= 0 { @@ -712,26 +765,6 @@ type OpenIDConfig struct { AuthorizationEndpoint string `json:"authorization_endpoint"` } -// discoverEndpointsFromIssuer discovers the token and authorization endpoints from the issuer's well-known url -// return error if failed to fetch the well-known configuration -func discoverEndpointsFromIssuer(provider *egv1a1.OIDCProvider) (*ir.OIDCProvider, error) { - if provider.TokenEndpoint == nil || provider.AuthorizationEndpoint == nil { - tokenEndpoint, authorizationEndpoint, err := fetchEndpointsFromIssuer(provider.Issuer) - if err != nil { - return nil, fmt.Errorf("error fetching endpoints from issuer: %w", err) - } - return &ir.OIDCProvider{ - TokenEndpoint: tokenEndpoint, - AuthorizationEndpoint: authorizationEndpoint, - }, nil - } - - return &ir.OIDCProvider{ - TokenEndpoint: *provider.TokenEndpoint, - AuthorizationEndpoint: *provider.AuthorizationEndpoint, - }, nil -} - func fetchEndpointsFromIssuer(issuerURL string) (string, string, error) { // Fetch the OpenID configuration from the issuer URL resp, err := http.Get(fmt.Sprintf("%s/.well-known/openid-configuration", issuerURL)) @@ -811,7 +844,7 @@ func (t *Translator) buildExtAuth(policy *egv1a1.SecurityPolicy, resources *reso grpc = policy.Spec.ExtAuth.GRPC backends *egv1a1.BackendCluster protocol ir.AppProtocol - ds []*ir.DestinationSetting + rd *ir.RouteDestination authority string err error traffic *ir.TrafficFeatures @@ -833,12 +866,12 @@ func (t *Translator) buildExtAuth(policy *egv1a1.SecurityPolicy, resources *reso backends = &grpc.BackendCluster protocol = ir.GRPC } - pnn := utils.NamespacedName(policy) - for _, backendRef := range backends.BackendRefs { - if err = t.validateExtServiceBackendReference(&backendRef.BackendObjectReference, policy.Namespace, policy.Kind, resources); err != nil { - return nil, err - } + if rd, err = t.translateExtServiceBackendRefs(policy, backends.BackendRefs, protocol, resources, envoyProxy, 0); err != nil { + return nil, err + } + + for _, backendRef := range backends.BackendRefs { // Authority is the calculated hostname that will be used as the Authority header. // If there are multiple backend referenced, simply use the first one - there are no good answers here. // When translated to XDS, the authority is used on the filter level not on the cluster level. @@ -846,23 +879,6 @@ func (t *Translator) buildExtAuth(policy *egv1a1.SecurityPolicy, resources *reso if authority == "" { authority = backendRefAuthority(resources, &backendRef.BackendObjectReference, policy) } - - extServiceDest, err := t.processExtServiceDestination( - &backendRef, - pnn, - resource.KindSecurityPolicy, - protocol, - resources, - envoyProxy, - ) - if err != nil { - return nil, err - } - ds = append(ds, extServiceDest) - } - rd := ir.RouteDestination{ - Name: irIndexedExtServiceDestinationName(pnn, resource.KindSecurityPolicy, 0), - Settings: ds, } if traffic, err = translateTrafficFeatures(backends.BackendSettings); err != nil { @@ -878,14 +894,14 @@ func (t *Translator) buildExtAuth(policy *egv1a1.SecurityPolicy, resources *reso if http != nil { extAuth.HTTP = &ir.HTTPExtAuthService{ - Destination: rd, + Destination: *rd, Authority: authority, Path: ptr.Deref(http.Path, ""), HeadersToBackend: http.HeadersToBackend, } } else { extAuth.GRPC = &ir.GRPCExtAuthService{ - Destination: rd, + Destination: *rd, Authority: authority, } } diff --git a/internal/gatewayapi/testdata/backend-with-fallback.in.yaml b/internal/gatewayapi/testdata/backend-with-fallback.in.yaml new file mode 100644 index 000000000000..667a75a65579 --- /dev/null +++ b/internal/gatewayapi/testdata/backend-with-fallback.in.yaml @@ -0,0 +1,58 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + - matches: + - path: + value: "/" + backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + - group: gateway.envoyproxy.io + kind: Backend + name: backend-2 +backends: + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-1 + namespace: default + spec: + endpoints: + - ip: + address: 1.1.1.1 + port: 3001 + - apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-2 + namespace: default + spec: + fallback: true + endpoints: + - ip: + address: 2.2.2.2 + port: 3001 diff --git a/internal/gatewayapi/testdata/backend-with-fallback.out.yaml b/internal/gatewayapi/testdata/backend-with-fallback.out.yaml new file mode 100644 index 000000000000..74bd61795fe2 --- /dev/null +++ b/internal/gatewayapi/testdata/backend-with-fallback.out.yaml @@ -0,0 +1,180 @@ +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-1 + namespace: default + spec: + endpoints: + - ip: + address: 1.1.1.1 + port: 3001 + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-2 + namespace: default + spec: + endpoints: + - ip: + address: 2.2.2.2 + port: 3001 + fallback: true + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + - group: gateway.envoyproxy.io + kind: Backend + name: backend-2 + matches: + - path: + value: / + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 1.1.1.1 + port: 3001 + weight: 1 + - addressType: IP + endpoints: + - host: 2.2.2.2 + port: 3001 + priority: 1 + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: / diff --git a/internal/gatewayapi/testdata/securitypolicy-with-oidc-backendcluster.in.yaml b/internal/gatewayapi/testdata/securitypolicy-with-oidc-backendcluster.in.yaml new file mode 100644 index 000000000000..67b051e4b313 --- /dev/null +++ b/internal/gatewayapi/testdata/securitypolicy-with-oidc-backendcluster.in.yaml @@ -0,0 +1,101 @@ +secrets: +- apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: client1-secret + data: + client-secret: Y2xpZW50MTpzZWNyZXQK +- apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway-system + name: envoy-oidc-hmac + data: + hmac-secret: qrOYACHXoe7UEDI/raOjNSx+Z9ufXSc/22C3T6X/zPY= +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - www.example.com + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/foo" + backendRefs: + - name: service-1 + port: 8080 +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-fqdn + namespace: envoy-gateway + spec: + endpoints: + - fqdn: + hostname: 'oauth.foo.com' + port: 443 +securityPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: SecurityPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + uid: b8284d0f-de82-4c65-b204-96a0d3f258a1 + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + oidc: + provider: + backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-fqdn + port: 443 + backendSettings: + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 1s + maxInterval: 5s + retryOn: + triggers: ["5xx", "gateway-error", "reset"] + issuer: "https://oauth.foo.com" + authorizationEndpoint: "https://oauth.foo.com/oauth2/v2/auth" + tokenEndpoint: "https://oauth.foo.com/token" + clientID: "client1.apps.googleusercontent.com" + clientSecret: + name: "client1-secret" + redirectURL: "https://www.example.com/bar/oauth2/callback" + logoutPath: "/bar/logout" + forwardAccessToken: true + defaultTokenTTL: 30m + refreshToken: true + defaultRefreshTokenTTL: 24h diff --git a/internal/gatewayapi/testdata/securitypolicy-with-oidc-backendcluster.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-oidc-backendcluster.out.yaml new file mode 100644 index 000000000000..d6f0c4dbc47a --- /dev/null +++ b/internal/gatewayapi/testdata/securitypolicy-with-oidc-backendcluster.out.yaml @@ -0,0 +1,256 @@ +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-fqdn + namespace: envoy-gateway + spec: + endpoints: + - fqdn: + hostname: oauth.foo.com + port: 443 + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - www.example.com + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: /foo + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +securityPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: SecurityPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + uid: b8284d0f-de82-4c65-b204-96a0d3f258a1 + spec: + oidc: + clientID: client1.apps.googleusercontent.com + clientSecret: + group: null + kind: null + name: client1-secret + defaultRefreshTokenTTL: 24h0m0s + defaultTokenTTL: 30m0s + forwardAccessToken: true + logoutPath: /bar/logout + provider: + authorizationEndpoint: https://oauth.foo.com/oauth2/v2/auth + backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-fqdn + port: 443 + backendSettings: + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 1s + maxInterval: 5s + retryOn: + triggers: + - 5xx + - gateway-error + - reset + issuer: https://oauth.foo.com + tokenEndpoint: https://oauth.foo.com/token + redirectURL: https://www.example.com/bar/oauth2/callback + refreshToken: true + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + hostname: www.example.com + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/www_example_com + pathMatch: + distinct: false + name: "" + prefix: /foo + security: + oidc: + clientID: client1.apps.googleusercontent.com + clientSecret: Y2xpZW50MTpzZWNyZXQK + cookieSuffix: b0a1b740 + defaultRefreshTokenTTL: 24h0m0s + defaultTokenTTL: 30m0s + forwardAccessToken: true + hmacSecret: qrOYACHXoe7UEDI/raOjNSx+Z9ufXSc/22C3T6X/zPY= + logoutPath: /bar/logout + name: securitypolicy/envoy-gateway/policy-for-gateway + provider: + authorizationEndpoint: https://oauth.foo.com/oauth2/v2/auth + destination: + name: securitypolicy/envoy-gateway/policy-for-gateway/0 + settings: + - addressType: FQDN + endpoints: + - host: oauth.foo.com + port: 443 + protocol: HTTPS + weight: 1 + tokenEndpoint: https://oauth.foo.com/token + traffic: + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 1s + maxInterval: 5s + retryOn: + triggers: + - 5xx + - gateway-error + - reset + redirectPath: /bar/oauth2/callback + redirectURL: https://www.example.com/bar/oauth2/callback + refreshToken: true + scopes: + - openid diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 04d0a2e0c1f6..16cd72b95457 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -398,6 +398,7 @@ func (i *Infra) deleteServiceAccount(ctx context.Context, r ResourceRender) (err return i.Client.DeleteAllOf(ctx, sa, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) @@ -437,6 +438,7 @@ func (i *Infra) deleteDeployment(ctx context.Context, r ResourceRender) (err err return i.Client.DeleteAllOf(ctx, deployment, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) @@ -476,6 +478,7 @@ func (i *Infra) deleteDaemonSet(ctx context.Context, r ResourceRender) (err erro return i.Client.DeleteAllOf(ctx, daemonSet, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) @@ -510,6 +513,7 @@ func (i *Infra) deleteConfigMap(ctx context.Context, r ResourceRender) (err erro return i.Client.DeleteAllOf(ctx, cm, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) @@ -544,6 +548,7 @@ func (i *Infra) deleteService(ctx context.Context, r ResourceRender) (err error) return i.Client.DeleteAllOf(ctx, svc, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) @@ -583,6 +588,7 @@ func (i *Infra) deleteHPA(ctx context.Context, r ResourceRender) (err error) { return i.Client.DeleteAllOf(ctx, hpa, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) @@ -622,6 +628,7 @@ func (i *Infra) deletePDB(ctx context.Context, r ResourceRender) (err error) { return i.Client.DeleteAllOf(ctx, pdb, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ + Namespace: ns, LabelSelector: r.LabelSelector(), }, }) diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index 6c261aff3f3d..300314d92e06 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -56,6 +56,13 @@ func (r *Runner) Start(ctx context.Context) (err error) { // Enable global ratelimit if it has been configured. if r.EnvoyGateway.RateLimit != nil { go r.enableRateLimitInfra(ctx) + } else { + // Delete the ratelimit infra if it exists. + go func() { + if err := r.mgr.DeleteRateLimitInfra(ctx); err != nil { + r.Logger.Error(err, "failed to delete ratelimit infra") + } + }() } r.Logger.Info("started") } diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 49d6fdbf0640..fdcace324f5f 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -848,7 +848,16 @@ type OIDC struct { CookieDomain *string `json:"cookieDomain,omitempty"` } +// OIDCProvider defines the schema for the OIDC Provider. +// +// +k8s:deepcopy-gen=true type OIDCProvider struct { + // Destination defines the destination for the OIDC Provider. + Destination *RouteDestination `json:"destination,omitempty"` + + // Traffic contains configuration for traffic features for the OIDC Provider + Traffic *TrafficFeatures `json:"traffic,omitempty"` + // The OIDC Provider's [authorization endpoint](https://openid.net/specs/openid-connect-core-1_0.html#AuthorizationEndpoint). AuthorizationEndpoint string `json:"authorizationEndpoint,omitempty"` diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 1a0185bbb9f1..3c0c1135f44b 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -1900,7 +1900,7 @@ func (in *Metrics) DeepCopy() *Metrics { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OIDC) DeepCopyInto(out *OIDC) { *out = *in - out.Provider = in.Provider + in.Provider.DeepCopyInto(&out.Provider) if in.ClientSecret != nil { in, out := &in.ClientSecret, &out.ClientSecret *out = make([]byte, len(*in)) @@ -1953,6 +1953,31 @@ func (in *OIDC) DeepCopy() *OIDC { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OIDCProvider) DeepCopyInto(out *OIDCProvider) { + *out = *in + if in.Destination != nil { + in, out := &in.Destination, &out.Destination + *out = new(RouteDestination) + (*in).DeepCopyInto(*out) + } + if in.Traffic != nil { + in, out := &in.Traffic, &out.Traffic + *out = new(TrafficFeatures) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OIDCProvider. +func (in *OIDCProvider) DeepCopy() *OIDCProvider { + if in == nil { + return nil + } + out := new(OIDCProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenTelemetryAccessLog) DeepCopyInto(out *OpenTelemetryAccessLog) { *out = *in diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 915e6e5acd8f..167e70746eca 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -8,6 +8,7 @@ package kubernetes import ( "context" "fmt" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -111,7 +112,10 @@ func newGatewayAPIController(mgr manager.Manager, cfg *config.Server, su Updater r.namespaceLabel = cfg.EnvoyGateway.Provider.Kubernetes.Watch.NamespaceSelector } - c, err := controller.New("gatewayapi", mgr, controller.Options{Reconciler: r, SkipNameValidation: skipNameValidation()}) + // controller-runtime doesn't allow run controller with same name for more than once + // see https://github.com/kubernetes-sigs/controller-runtime/blob/2b941650bce159006c88bd3ca0d132c7bc40e947/pkg/controller/name.go#L29 + name := fmt.Sprintf("gatewayapi-%d", time.Now().Unix()) + c, err := controller.New(name, mgr, controller.Options{Reconciler: r, SkipNameValidation: skipNameValidation()}) if err != nil { return fmt.Errorf("error creating controller: %w", err) } diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index cca90a24a177..4fdbc329dd09 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -112,7 +112,8 @@ func New(cfg *rest.Config, svr *ec.Server, resources *message.ProviderResources) // Emit elected & continue with envoyObjects of infra resources go func() { <-mgr.Elected() - close(svr.Elected) + // WARN: DO NOT CLOSE IT + svr.Elected <- struct{}{} }() return &Provider{ diff --git a/internal/wasm/httpserver.go b/internal/wasm/httpserver.go index 9b1d0b32c90a..14e70a8c6dcd 100644 --- a/internal/wasm/httpserver.go +++ b/internal/wasm/httpserver.go @@ -131,6 +131,12 @@ func (s *HTTPServer) Start(ctx context.Context) { return } }() + + go func() { + // waiting for shutdown + <-ctx.Done() + _ = s.server.Shutdown(context.Background()) + }() s.cache.Start(ctx) go s.resetFailedAttempts(ctx) } diff --git a/internal/xds/translator/oidc.go b/internal/xds/translator/oidc.go index 41228a1d2092..963b7c8046da 100644 --- a/internal/xds/translator/oidc.go +++ b/internal/xds/translator/oidc.go @@ -102,14 +102,24 @@ func oauth2FilterName(oidc *ir.OIDC) string { } func oauth2Config(oidc *ir.OIDC) (*oauth2v3.OAuth2, error) { - cluster, err := url2Cluster(oidc.Provider.TokenEndpoint) - if err != nil { - return nil, err - } - if cluster.endpointType == EndpointTypeStatic { - return nil, fmt.Errorf( - "static IP cluster is not allowed: %s", - oidc.Provider.TokenEndpoint) + var ( + tokenEndpointCluster string + err error + ) + + if oidc.Provider.Destination != nil && len(oidc.Provider.Destination.Settings) > 0 { + tokenEndpointCluster = oidc.Provider.Destination.Name + } else { + var cluster *urlCluster + if cluster, err = url2Cluster(oidc.Provider.TokenEndpoint); err != nil { + return nil, err + } + if cluster.endpointType == EndpointTypeStatic { + return nil, fmt.Errorf( + "static IP cluster is not allowed: %s", + oidc.Provider.TokenEndpoint) + } + tokenEndpointCluster = cluster.name } // Envoy OAuth2 filter deletes the HTTP authorization header by default, which surprises users. @@ -126,7 +136,7 @@ func oauth2Config(oidc *ir.OIDC) (*oauth2v3.OAuth2, error) { TokenEndpoint: &corev3.HttpUri{ Uri: oidc.Provider.TokenEndpoint, HttpUpstreamType: &corev3.HttpUri_Cluster{ - Cluster: cluster.name, + Cluster: tokenEndpointCluster, }, Timeout: &durationpb.Duration{ Seconds: defaultExtServiceRequestTimeout, @@ -210,9 +220,55 @@ func oauth2Config(oidc *ir.OIDC) (*oauth2v3.OAuth2, error) { oauth2.Config.Credentials.CookieDomain = *oidc.CookieDomain } + // Set the retry policy if it exists. + if oidc.Provider.Traffic != nil && oidc.Provider.Traffic.Retry != nil { + var rp *corev3.RetryPolicy + if rp, err = buildNonRouteRetryPolicy(oidc.Provider.Traffic.Retry); err != nil { + return nil, err + } + oauth2.Config.RetryPolicy = rp + } return oauth2, nil } +func buildNonRouteRetryPolicy(rr *ir.Retry) (*corev3.RetryPolicy, error) { + rp := &corev3.RetryPolicy{ + RetryOn: retryDefaultRetryOn, + } + + // These two fields in the RetryPolicy are just for route-level retries, they are not used for non-route retries. + // retry.PerRetry.Timeout + // retry.RetryOn.HTTPStatusCodes + + if rr.PerRetry != nil && rr.PerRetry.BackOff != nil { + rp.RetryBackOff = &corev3.BackoffStrategy{ + BaseInterval: &durationpb.Duration{ + Seconds: int64(rr.PerRetry.BackOff.BaseInterval.Seconds()), + }, + MaxInterval: &durationpb.Duration{ + Seconds: int64(rr.PerRetry.BackOff.MaxInterval.Seconds()), + }, + } + } + + if rr.NumRetries != nil { + rp.NumRetries = &wrappers.UInt32Value{ + Value: *rr.NumRetries, + } + } + + if rr.RetryOn != nil { + if len(rr.RetryOn.Triggers) > 0 { + if ro, err := buildRetryOn(rr.RetryOn.Triggers); err == nil { + rp.RetryOn = ro + } else { + return nil, err + } + } + } + return rp, nil +} + // routeContainsOIDC returns true if OIDC exists for the provided route. func routeContainsOIDC(irRoute *ir.HTTPRoute) bool { if irRoute != nil && @@ -226,7 +282,7 @@ func routeContainsOIDC(irRoute *ir.HTTPRoute) bool { func (*oidc) patchResources(tCtx *types.ResourceVersionTable, routes []*ir.HTTPRoute, ) error { - if err := createOAuth2TokenEndpointClusters(tCtx, routes); err != nil { + if err := createOAuthServerClusters(tCtx, routes); err != nil { return err } if err := createOAuth2Secrets(tCtx, routes); err != nil { @@ -235,9 +291,8 @@ func (*oidc) patchResources(tCtx *types.ResourceVersionTable, return nil } -// createOAuth2TokenEndpointClusters creates token endpoint clusters from the -// provided routes, if needed. -func createOAuth2TokenEndpointClusters(tCtx *types.ResourceVersionTable, +// createOAuthServerClusters creates clusters for the OAuth2 server. +func createOAuthServerClusters(tCtx *types.ResourceVersionTable, routes []*ir.HTTPRoute, ) error { if tCtx == nil || tCtx.XdsResources == nil { @@ -250,59 +305,78 @@ func createOAuth2TokenEndpointClusters(tCtx *types.ResourceVersionTable, continue } - var ( - cluster *urlCluster - ds *ir.DestinationSetting - tSocket *corev3.TransportSocket - err error - ) + oidc := route.Security.OIDC - cluster, err = url2Cluster(route.Security.OIDC.Provider.TokenEndpoint) - if err != nil { - errs = errors.Join(errs, err) - continue + // If the OIDC provider has a destination, use it. + if oidc.Provider.Destination != nil && len(oidc.Provider.Destination.Settings) > 0 { + if err := createExtServiceXDSCluster( + oidc.Provider.Destination, oidc.Provider.Traffic, tCtx); err != nil && !errors.Is( + err, ErrXdsClusterExists) { + errs = errors.Join(errs, err) + } + } else { + // Create a cluster with the token endpoint url. + if err := createOAuth2TokenEndpointCluster(tCtx, oidc.Provider.TokenEndpoint); err != nil { + errs = errors.Join(errs, err) + } } + } - // EG does not support static IP clusters for token endpoint clusters. - // This validation could be removed since it's already validated in the - // Gateway API translator. - if cluster.endpointType == EndpointTypeStatic { - errs = errors.Join(errs, fmt.Errorf( - "static IP cluster is not allowed: %s", - route.Security.OIDC.Provider.TokenEndpoint)) - continue - } + return errs +} - ds = &ir.DestinationSetting{ - Weight: ptr.To[uint32](1), - Endpoints: []*ir.DestinationEndpoint{ - ir.NewDestEndpoint( - cluster.hostname, - cluster.port), - }, - } +// createOAuth2TokenEndpointClusters creates token endpoint clusters from the +// provided routes, if needed. +func createOAuth2TokenEndpointCluster(tCtx *types.ResourceVersionTable, + tokenEndpoint string, +) error { + var ( + cluster *urlCluster + ds *ir.DestinationSetting + tSocket *corev3.TransportSocket + err error + ) + + if cluster, err = url2Cluster(tokenEndpoint); err != nil { + return err + } - clusterArgs := &xdsClusterArgs{ - name: cluster.name, - settings: []*ir.DestinationSetting{ds}, - tSocket: tSocket, - endpointType: cluster.endpointType, - } - if cluster.tls { - tSocket, err = buildXdsUpstreamTLSSocket(cluster.hostname) - if err != nil { - errs = errors.Join(errs, err) - continue - } - clusterArgs.tSocket = tSocket - } + // EG does not support static IP clusters for token endpoint clusters. + // This validation could be removed since it's already validated in the + // Gateway API translator. + if cluster.endpointType == EndpointTypeStatic { + return fmt.Errorf( + "static IP cluster is not allowed: %s", + tokenEndpoint) + } - if err = addXdsCluster(tCtx, clusterArgs); err != nil && !errors.Is(err, ErrXdsClusterExists) { - errs = errors.Join(errs, err) + ds = &ir.DestinationSetting{ + Weight: ptr.To[uint32](1), + Endpoints: []*ir.DestinationEndpoint{ + ir.NewDestEndpoint( + cluster.hostname, + cluster.port), + }, + } + + clusterArgs := &xdsClusterArgs{ + name: cluster.name, + settings: []*ir.DestinationSetting{ds}, + tSocket: tSocket, + endpointType: cluster.endpointType, + } + if cluster.tls { + if tSocket, err = buildXdsUpstreamTLSSocket(cluster.hostname); err != nil { + return err } + clusterArgs.tSocket = tSocket } - return errs + if err = addXdsCluster(tCtx, clusterArgs); err != nil && !errors.Is(err, ErrXdsClusterExists) { + return err + } + + return err } // createOAuth2Secrets creates OAuth2 client and HMAC secrets from the provided diff --git a/internal/xds/translator/testdata/in/xds-ir/oidc-backend-cluster-provider.yaml b/internal/xds/translator/testdata/in/xds-ir/oidc-backend-cluster-provider.yaml new file mode 100644 index 000000000000..6fc9a045bca8 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/oidc-backend-cluster-provider.yaml @@ -0,0 +1,60 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "first-route" + hostname: "*" + pathMatch: + exact: "baz" + destination: + name: "third-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + security: + oidc: + name: securitypolicy/envoy-gateway/policy-for-gateway + clientID: client1.apps.googleusercontent.com + clientSecret: Y2xpZW50MTpzZWNyZXQK + cookieSuffix: b0a1b740 + defaultRefreshTokenTTL: 24h0m0s + defaultTokenTTL: 30m0s + forwardAccessToken: true + hmacSecret: qrOYACHXoe7UEDI/raOjNSx+Z9ufXSc/22C3T6X/zPY= + logoutPath: /bar/logout + provider: + authorizationEndpoint: https://oauth.foo.com/oauth2/v2/auth + destination: + name: securitypolicy/envoy-gateway/policy-for-gateway/0 + settings: + - addressType: FQDN + endpoints: + - host: oauth.foo.com + port: 443 + protocol: HTTPS + weight: 1 + tokenEndpoint: https://oauth.foo.com/token + traffic: + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 1s + maxInterval: 5s + retryOn: + triggers: + - "5xx" + - gateway-error + - reset + redirectPath: /bar/oauth2/callback + redirectURL: https://www.example.com/bar/oauth2/callback + refreshToken: true + scopes: + - openid diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.clusters.yaml new file mode 100644 index 000000000000..863e761bf9a2 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.clusters.yaml @@ -0,0 +1,44 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: third-route-dest + lbPolicy: LEAST_REQUEST + name: third-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: securitypolicy/envoy-gateway/policy-for-gateway/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: oauth.foo.com + portValue: 443 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: securitypolicy/envoy-gateway/policy-for-gateway/0/backend/0 + name: securitypolicy/envoy-gateway/policy-for-gateway/0 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.endpoints.yaml new file mode 100644 index 000000000000..6c69841c2add --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: third-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: third-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.listeners.yaml new file mode 100644 index 000000000000..3addb294484e --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.listeners.yaml @@ -0,0 +1,83 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - disabled: true + name: envoy.filters.http.oauth2/securitypolicy/envoy-gateway/policy-for-gateway + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.oauth2.v3.OAuth2 + config: + authScopes: + - openid + authType: BASIC_AUTH + authorizationEndpoint: https://oauth.foo.com/oauth2/v2/auth + credentials: + clientId: client1.apps.googleusercontent.com + cookieNames: + bearerToken: AccessToken-b0a1b740 + idToken: IdToken-b0a1b740 + oauthExpires: OauthExpires-b0a1b740 + oauthHmac: OauthHMAC-b0a1b740 + oauthNonce: OauthNonce-b0a1b740 + refreshToken: RefreshToken-b0a1b740 + hmacSecret: + name: oauth2/hmac_secret/securitypolicy/envoy-gateway/policy-for-gateway + sdsConfig: + ads: {} + resourceApiVersion: V3 + tokenSecret: + name: oauth2/client_secret/securitypolicy/envoy-gateway/policy-for-gateway + sdsConfig: + ads: {} + resourceApiVersion: V3 + defaultExpiresIn: 1800s + defaultRefreshTokenExpiresIn: 86400s + forwardBearerToken: true + redirectPathMatcher: + path: + exact: /bar/oauth2/callback + redirectUri: https://www.example.com/bar/oauth2/callback + retryPolicy: + numRetries: 3 + retryBackOff: + baseInterval: 1s + maxInterval: 5s + retryOn: 5xx,gateway-error,reset + signoutPath: + path: + exact: /bar/logout + tokenEndpoint: + cluster: securitypolicy/envoy-gateway/policy-for-gateway/0 + timeout: 10s + uri: https://oauth.foo.com/token + useRefreshToken: true + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: first-listener + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.routes.yaml new file mode 100644 index 000000000000..b17df86476de --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.routes.yaml @@ -0,0 +1,18 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + path: baz + name: first-route + route: + cluster: third-route-dest + upgradeConfigs: + - upgradeType: websocket + typedPerFilterConfig: + envoy.filters.http.oauth2/securitypolicy/envoy-gateway/policy-for-gateway: + '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig + config: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.secrets.yaml new file mode 100644 index 000000000000..398ab6cef7b5 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backencluster-provider.secrets.yaml @@ -0,0 +1,8 @@ +- genericSecret: + secret: + inlineBytes: Y2xpZW50MTpzZWNyZXQK + name: oauth2/client_secret/securitypolicy/envoy-gateway/policy-for-gateway +- genericSecret: + secret: + inlineBytes: qrOYACHXoe7UEDI/raOjNSx+Z9ufXSc/22C3T6X/zPY= + name: oauth2/hmac_secret/securitypolicy/envoy-gateway/policy-for-gateway diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml new file mode 100644 index 000000000000..863e761bf9a2 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml @@ -0,0 +1,44 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: third-route-dest + lbPolicy: LEAST_REQUEST + name: third-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: securitypolicy/envoy-gateway/policy-for-gateway/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: oauth.foo.com + portValue: 443 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: securitypolicy/envoy-gateway/policy-for-gateway/0/backend/0 + name: securitypolicy/envoy-gateway/policy-for-gateway/0 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.endpoints.yaml new file mode 100644 index 000000000000..6c69841c2add --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: third-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: third-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.listeners.yaml new file mode 100644 index 000000000000..3addb294484e --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.listeners.yaml @@ -0,0 +1,83 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - disabled: true + name: envoy.filters.http.oauth2/securitypolicy/envoy-gateway/policy-for-gateway + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.oauth2.v3.OAuth2 + config: + authScopes: + - openid + authType: BASIC_AUTH + authorizationEndpoint: https://oauth.foo.com/oauth2/v2/auth + credentials: + clientId: client1.apps.googleusercontent.com + cookieNames: + bearerToken: AccessToken-b0a1b740 + idToken: IdToken-b0a1b740 + oauthExpires: OauthExpires-b0a1b740 + oauthHmac: OauthHMAC-b0a1b740 + oauthNonce: OauthNonce-b0a1b740 + refreshToken: RefreshToken-b0a1b740 + hmacSecret: + name: oauth2/hmac_secret/securitypolicy/envoy-gateway/policy-for-gateway + sdsConfig: + ads: {} + resourceApiVersion: V3 + tokenSecret: + name: oauth2/client_secret/securitypolicy/envoy-gateway/policy-for-gateway + sdsConfig: + ads: {} + resourceApiVersion: V3 + defaultExpiresIn: 1800s + defaultRefreshTokenExpiresIn: 86400s + forwardBearerToken: true + redirectPathMatcher: + path: + exact: /bar/oauth2/callback + redirectUri: https://www.example.com/bar/oauth2/callback + retryPolicy: + numRetries: 3 + retryBackOff: + baseInterval: 1s + maxInterval: 5s + retryOn: 5xx,gateway-error,reset + signoutPath: + path: + exact: /bar/logout + tokenEndpoint: + cluster: securitypolicy/envoy-gateway/policy-for-gateway/0 + timeout: 10s + uri: https://oauth.foo.com/token + useRefreshToken: true + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: first-listener + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.routes.yaml new file mode 100644 index 000000000000..b17df86476de --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.routes.yaml @@ -0,0 +1,18 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + path: baz + name: first-route + route: + cluster: third-route-dest + upgradeConfigs: + - upgradeType: websocket + typedPerFilterConfig: + envoy.filters.http.oauth2/securitypolicy/envoy-gateway/policy-for-gateway: + '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig + config: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.secrets.yaml new file mode 100644 index 000000000000..398ab6cef7b5 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.secrets.yaml @@ -0,0 +1,8 @@ +- genericSecret: + secret: + inlineBytes: Y2xpZW50MTpzZWNyZXQK + name: oauth2/client_secret/securitypolicy/envoy-gateway/policy-for-gateway +- genericSecret: + secret: + inlineBytes: qrOYACHXoe7UEDI/raOjNSx+Z9ufXSc/22C3T6X/zPY= + name: oauth2/hmac_secret/securitypolicy/envoy-gateway/policy-for-gateway diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index ddc32a2021e0..fe361099a842 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -384,6 +384,7 @@ _Appears in:_ | --- | --- | --- | --- | | `endpoints` | _[BackendEndpoint](#backendendpoint) array_ | true | Endpoints defines the endpoints to be used when connecting to the backend. | | `appProtocols` | _[AppProtocolType](#appprotocoltype) array_ | false | AppProtocols defines the application protocols to be supported when connecting to the backend. | +| `fallback` | _boolean_ | false | Fallback indicates whether the backend is designated as a fallback.
It is highly recommended to configure active or passive health checks to ensure that failover can be detected
when the active backends become unhealthy and to automatically readjust once the primary backends are healthy again.
The overprovisioning factor is set to 1.4, meaning the fallback backends will only start receiving traffic when
the health of the active backends falls below 72%. | #### BackendStatus @@ -865,7 +866,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `contentType` | _string_ | false | Content Type of the response. This will be set in the Content-Type header. | -| `body` | _[CustomResponseBody](#customresponsebody)_ | true | Body of the Custom Response | +| `body` | _[CustomResponseBody](#customresponsebody)_ | false | Body of the Custom Response | #### CustomResponseBody @@ -876,6 +877,7 @@ CustomResponseBody _Appears in:_ - [CustomResponse](#customresponse) +- [HTTPDirectResponseFilter](#httpdirectresponsefilter) | Field | Type | Required | Description | | --- | --- | --- | --- | @@ -1921,6 +1923,22 @@ _Appears in:_ | `idleTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.Duration)_ | false | IdleTimeout for an HTTP connection. Idle time is defined as a period in which there are no active requests in the connection.
Default: 1 hour. | +#### HTTPDirectResponseFilter + + + +HTTPDirectResponseFilter defines the configuration to return a fixed response. + +_Appears in:_ +- [HTTPRouteFilterSpec](#httproutefilterspec) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `contentType` | _string_ | false | Content Type of the response. This will be set in the Content-Type header. | +| `body` | _[CustomResponseBody](#customresponsebody)_ | false | Body of the Response | +| `statusCode` | _integer_ | false | Status Code of the HTTP response
If unset, defaults to 200. | + + #### HTTPExtAuthService @@ -2044,6 +2062,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `urlRewrite` | _[HTTPURLRewriteFilter](#httpurlrewritefilter)_ | false | | +| `directResponse` | _[HTTPDirectResponseFilter](#httpdirectresponsefilter)_ | false | | #### HTTPStatus diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index ddc32a2021e0..fe361099a842 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -384,6 +384,7 @@ _Appears in:_ | --- | --- | --- | --- | | `endpoints` | _[BackendEndpoint](#backendendpoint) array_ | true | Endpoints defines the endpoints to be used when connecting to the backend. | | `appProtocols` | _[AppProtocolType](#appprotocoltype) array_ | false | AppProtocols defines the application protocols to be supported when connecting to the backend. | +| `fallback` | _boolean_ | false | Fallback indicates whether the backend is designated as a fallback.
It is highly recommended to configure active or passive health checks to ensure that failover can be detected
when the active backends become unhealthy and to automatically readjust once the primary backends are healthy again.
The overprovisioning factor is set to 1.4, meaning the fallback backends will only start receiving traffic when
the health of the active backends falls below 72%. | #### BackendStatus @@ -865,7 +866,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `contentType` | _string_ | false | Content Type of the response. This will be set in the Content-Type header. | -| `body` | _[CustomResponseBody](#customresponsebody)_ | true | Body of the Custom Response | +| `body` | _[CustomResponseBody](#customresponsebody)_ | false | Body of the Custom Response | #### CustomResponseBody @@ -876,6 +877,7 @@ CustomResponseBody _Appears in:_ - [CustomResponse](#customresponse) +- [HTTPDirectResponseFilter](#httpdirectresponsefilter) | Field | Type | Required | Description | | --- | --- | --- | --- | @@ -1921,6 +1923,22 @@ _Appears in:_ | `idleTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.Duration)_ | false | IdleTimeout for an HTTP connection. Idle time is defined as a period in which there are no active requests in the connection.
Default: 1 hour. | +#### HTTPDirectResponseFilter + + + +HTTPDirectResponseFilter defines the configuration to return a fixed response. + +_Appears in:_ +- [HTTPRouteFilterSpec](#httproutefilterspec) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `contentType` | _string_ | false | Content Type of the response. This will be set in the Content-Type header. | +| `body` | _[CustomResponseBody](#customresponsebody)_ | false | Body of the Response | +| `statusCode` | _integer_ | false | Status Code of the HTTP response
If unset, defaults to 200. | + + #### HTTPExtAuthService @@ -2044,6 +2062,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `urlRewrite` | _[HTTPURLRewriteFilter](#httpurlrewritefilter)_ | false | | +| `directResponse` | _[HTTPDirectResponseFilter](#httpdirectresponsefilter)_ | false | | #### HTTPStatus diff --git a/test/cel-validation/securitypolicy_test.go b/test/cel-validation/securitypolicy_test.go index c4efa812d586..f00ee84260c6 100644 --- a/test/cel-validation/securitypolicy_test.go +++ b/test/cel-validation/securitypolicy_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" ) @@ -1121,6 +1122,103 @@ func TestSecurityPolicyTarget(t *testing.T) { }, wantErrors: []string{"at least one of claims or scopes must be specified"}, }, + { + desc: "oidc-retry", + mutate: func(sp *egv1a1.SecurityPolicy) { + sp.Spec = egv1a1.SecurityPolicySpec{ + PolicyTargetReferences: egv1a1.PolicyTargetReferences{ + TargetSelectors: []egv1a1.TargetSelector{ + { + Group: ptr.To(gwapiv1a2.Group("gateway.networking.k8s.io")), + Kind: "HTTPRoute", + MatchLabels: map[string]string{ + "eg/namespace": "reference-apps", + }, + }, + }, + }, + OIDC: &egv1a1.OIDC{ + Provider: egv1a1.OIDCProvider{ + BackendCluster: egv1a1.BackendCluster{ + BackendSettings: &egv1a1.ClusterSettings{ + Retry: &egv1a1.Retry{ + NumRetries: ptr.To(int32(3)), + PerRetry: &egv1a1.PerRetryPolicy{ + BackOff: &egv1a1.BackOffPolicy{ + BaseInterval: &metav1.Duration{ + Duration: time.Second * 1, + }, + MaxInterval: &metav1.Duration{ + Duration: time.Second * 10, + }, + }, + }, + RetryOn: &egv1a1.RetryOn{ + Triggers: []egv1a1.TriggerEnum{ + egv1a1.Error5XX, egv1a1.GatewayError, egv1a1.Reset, + }, + }, + }, + }, + }, + Issuer: "https://accounts.google.com", + AuthorizationEndpoint: ptr.To("https://accounts.google.com/o/oauth2/v2/auth"), + TokenEndpoint: ptr.To("https://oauth2.googleapis.com/token"), + }, + ClientID: "client-id", + ClientSecret: gwapiv1b1.SecretObjectReference{ + Name: "secret", + }, + }, + } + }, + wantErrors: []string{}, + }, + { + desc: "oidc-retry-unsupported-parameters", + mutate: func(sp *egv1a1.SecurityPolicy) { + sp.Spec = egv1a1.SecurityPolicySpec{ + PolicyTargetReferences: egv1a1.PolicyTargetReferences{ + TargetSelectors: []egv1a1.TargetSelector{ + { + Group: ptr.To(gwapiv1a2.Group("gateway.networking.k8s.io")), + Kind: "HTTPRoute", + MatchLabels: map[string]string{ + "eg/namespace": "reference-apps", + }, + }, + }, + }, + OIDC: &egv1a1.OIDC{ + Provider: egv1a1.OIDCProvider{ + BackendCluster: egv1a1.BackendCluster{ + BackendSettings: &egv1a1.ClusterSettings{ + Retry: &egv1a1.Retry{ + NumRetries: ptr.To(int32(3)), + PerRetry: &egv1a1.PerRetryPolicy{ + Timeout: &metav1.Duration{ + Duration: time.Second * 10, + }, + }, + RetryOn: &egv1a1.RetryOn{ + HTTPStatusCodes: []egv1a1.HTTPStatus{500}, + }, + }, + }, + }, + Issuer: "https://accounts.google.com", + AuthorizationEndpoint: ptr.To("https://accounts.google.com/o/oauth2/v2/auth"), + TokenEndpoint: ptr.To("https://oauth2.googleapis.com/token"), + }, + ClientID: "client-id", + ClientSecret: gwapiv1b1.SecretObjectReference{ + Name: "secret", + }, + }, + } + }, + wantErrors: []string{"Retry timeout is not supported", "HTTPStatusCodes is not supported"}, + }, } for _, tc := range cases { diff --git a/test/e2e/testdata/oidc-securitypolicy-backendcluster.yaml b/test/e2e/testdata/oidc-securitypolicy-backendcluster.yaml new file mode 100644 index 000000000000..2e1a86791ec3 --- /dev/null +++ b/test/e2e/testdata/oidc-securitypolicy-backendcluster.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-with-oidc + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + hostnames: ["www.example.com"] + rules: + - matches: + - path: + type: PathPrefix + value: /myapp # This is the path that will be protected by OIDC + backendRefs: + - name: infra-backend-v1 + port: 8080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-keycloak + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + hostnames: ["keycloak.gateway-conformance-infra"] + rules: + - backendRefs: + - name: keycloak + port: 80 +--- +apiVersion: v1 +kind: Secret +metadata: + namespace: gateway-conformance-infra + name: oidctest-secret +data: + client-secret: b2lkY3Rlc3QtY2xpZW50LXNlY3JldA== # base64 encoding of "oidctest-client-secret" +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: oidc-test # Test OIDC Provider represented by a backend cluster + namespace: gateway-conformance-infra +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: http-with-oidc + oidc: + provider: + backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-keycloak + port: 80 + backendSettings: + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 1s + maxInterval: 5s + retryOn: + triggers: ["5xx", "gateway-error", "reset"] + issuer: "http://keycloak.gateway-conformance-infra/realms/master" + authorizationEndpoint: "http://keycloak.gateway-conformance-infra/realms/master/protocol/openid-connect/auth" + tokenEndpoint: "http://keycloak.gateway-conformance-infra/realms/master/protocol/openid-connect/token" + clientID: "oidctest" + clientSecret: + name: "oidctest-secret" + redirectURL: "http://www.example.com/myapp/oauth2/callback" + logoutPath: "/myapp/logout" +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: backend-keycloak + namespace: gateway-conformance-infra +spec: + endpoints: + - fqdn: + hostname: 'keycloak.gateway-conformance-infra' + port: 80 diff --git a/test/e2e/tests/oidc-backendcluster.go b/test/e2e/tests/oidc-backendcluster.go new file mode 100644 index 000000000000..b2bcc93cecb0 --- /dev/null +++ b/test/e2e/tests/oidc-backendcluster.go @@ -0,0 +1,37 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e +// +build e2e + +package tests + +import ( + "testing" + + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, OIDCBackendClusterTest) +} + +// OIDCTest tests OIDC authentication for an http route with OIDC configured. +// The http route points to an application to verify that OIDC authentication works on application/http path level. +var OIDCBackendClusterTest = suite.ConformanceTest{ + ShortName: "OIDC with BackendCluster", + Description: "Test OIDC authentication", + Manifests: []string{"testdata/oidc-keycloak.yaml", "testdata/oidc-securitypolicy-backendcluster.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + t.Run("oidc provider represented by a BackendCluster", func(t *testing.T) { + // Add a function to dump current cluster status + t.Cleanup(func() { + CollectAndDump(t, suite.RestConfig) + }) + + testOIDC(t, suite) + }) + }, +} diff --git a/test/e2e/tests/oidc.go b/test/e2e/tests/oidc.go index 27fb79be9b30..f03512c1e274 100644 --- a/test/e2e/tests/oidc.go +++ b/test/e2e/tests/oidc.go @@ -31,8 +31,6 @@ import ( ) const ( - testURL = "http://www.example.com/myapp" - logoutURL = "http://www.example.com/myapp/logout" keyCloakLoginFormID = "kc-form-login" username = "oidcuser" password = "oidcpassword" @@ -49,100 +47,13 @@ var OIDCTest = suite.ConformanceTest{ Description: "Test OIDC authentication", Manifests: []string{"testdata/oidc-keycloak.yaml", "testdata/oidc-securitypolicy.yaml"}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { - t.Run("http route with oidc authentication", func(t *testing.T) { + t.Run("oidc provider represented by a URL", func(t *testing.T) { // Add a function to dump current cluster status t.Cleanup(func() { CollectAndDump(t, suite.RestConfig) }) - ns := "gateway-conformance-infra" - routeNN := types.NamespacedName{Name: "http-with-oidc", Namespace: ns} - gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} - gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) - - ancestorRef := gwapiv1a2.ParentReference{ - Group: gatewayapi.GroupPtr(gwapiv1.GroupName), - Kind: gatewayapi.KindPtr(resource.KindGateway), - Namespace: gatewayapi.NamespacePtr(gwNN.Namespace), - Name: gwapiv1.ObjectName(gwNN.Name), - } - SecurityPolicyMustBeAccepted(t, suite.Client, types.NamespacedName{Name: "oidc-test", Namespace: ns}, suite.ControllerName, ancestorRef) - podInitialized := corev1.PodCondition{Type: corev1.PodInitialized, Status: corev1.ConditionTrue} - - // Wait for the keycloak pod to be configured with the test user and client - WaitForPods(t, suite.Client, ns, map[string]string{"job-name": "setup-keycloak"}, corev1.PodSucceeded, podInitialized) - - // Initialize the test OIDC client that will keep track of the state of the OIDC login process - client, err := NewOIDCTestClient( - WithLoggingOptions(t.Log, true), - // Map the application and keycloak cluster DNS name to the gateway address - WithCustomAddressMappings(map[string]string{ - "www.example.com:80": gwAddr, - "keycloak.gateway-conformance-infra:80": gwAddr, - }), - ) - require.NoError(t, err) - - if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, 5*time.Minute, true, - func(_ context.Context) (done bool, err error) { - tlog.Logf(t, "sending request to %s", testURL) - - // Send a request to the http route with OIDC configured. - // It will be redirected to the keycloak login page - res, err := client.Get(testURL, true) - require.NoError(t, err, "Failed to get the login page") - require.Equal(t, 200, res.StatusCode, "Expected 200 OK") - - // Parse the response body to get the URL where the login page would post the user-entered credentials - if err := client.ParseLoginForm(res.Body, keyCloakLoginFormID); err != nil { - tlog.Logf(t, "failed to parse login form: %v", err) - return false, nil - } - - t.Log("successfully parsed login form") - return true, nil - }); err != nil { - t.Errorf("failed to parse login form: %v", err) - } - - // Submit the login form to the IdP. - // This will authenticate and redirect back to the application - res, err := client.Login(map[string]string{"username": username, "password": password, "credentialId": ""}) - require.NoError(t, err, "Failed to login to the IdP") - - // Verify that we get the expected response from the application - body, err := io.ReadAll(res.Body) - require.NoError(t, err) - require.Equal(t, http.StatusOK, res.StatusCode) - require.Contains(t, string(body), "infra-backend-v1", "Expected response from the application") - - // Verify that we can access the application without logging in again - res, err = client.Get(testURL, false) - require.NoError(t, err) - require.Equal(t, http.StatusOK, res.StatusCode) - require.Contains(t, string(body), "infra-backend-v1", "Expected response from the application") - - // Verify that we can logout - // Note: OAuth2 filter just clears its cookies and does not log out from the IdP. - res, err = client.Get(logoutURL, false) - require.NoError(t, err) - require.Equal(t, http.StatusFound, res.StatusCode) - - // After logout, OAuth2 filter will redirect back to the root of the host, e.g, "www.example.com". - // Ideally, this should redirect to the application's root, e.g, "www.example.com/myapp", - // but Envoy OAuth2 filter does not support this yet. - require.Equal(t, "http://www.example.com/", res.Header.Get("Location"), "Expected redirect to the root of the host") - - // Verify that the oauth2 cookies have been deleted - var cookieDeleted bool - deletedCookies := res.Header.Values("Set-Cookie") - regx := regexp.MustCompile("^IdToken-.+=deleted.+") - for _, cookie := range deletedCookies { - if regx.Match([]byte(cookie)) { - cookieDeleted = true - } - } - require.True(t, cookieDeleted, "IdToken cookie not deleted") + testOIDC(t, suite) }) t.Run("http route without oidc authentication", func(t *testing.T) { @@ -185,3 +96,102 @@ var OIDCTest = suite.ConformanceTest{ }) }, } + +func testOIDC(t *testing.T, suite *suite.ConformanceTestSuite) { + var ( + testURL = "http://www.example.com/myapp" + logoutURL = "http://www.example.com/myapp/logout" + route = "http-with-oidc" + sp = "oidc-test" + ns = "gateway-conformance-infra" + ) + + routeNN := types.NamespacedName{Name: route, Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + + ancestorRef := gwapiv1a2.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGateway), + Namespace: gatewayapi.NamespacePtr(gwNN.Namespace), + Name: gwapiv1.ObjectName(gwNN.Name), + } + SecurityPolicyMustBeAccepted(t, suite.Client, types.NamespacedName{Name: sp, Namespace: ns}, suite.ControllerName, ancestorRef) + + podInitialized := corev1.PodCondition{Type: corev1.PodInitialized, Status: corev1.ConditionTrue} + + // Wait for the keycloak pod to be configured with the test user and client + WaitForPods(t, suite.Client, ns, map[string]string{"job-name": "setup-keycloak"}, corev1.PodSucceeded, podInitialized) + + // Initialize the test OIDC client that will keep track of the state of the OIDC login process + client, err := NewOIDCTestClient( + WithLoggingOptions(t.Log, true), + // Map the application and keycloak cluster DNS name to the gateway address + WithCustomAddressMappings(map[string]string{ + "www.example.com:80": gwAddr, + "keycloak.gateway-conformance-infra:80": gwAddr, + }), + ) + require.NoError(t, err) + + if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, 5*time.Minute, true, + func(_ context.Context) (done bool, err error) { + tlog.Logf(t, "sending request to %s", testURL) + + // Send a request to the http route with OIDC configured. + // It will be redirected to the keycloak login page + res, err := client.Get(testURL, true) + require.NoError(t, err, "Failed to get the login page") + require.Equal(t, 200, res.StatusCode, "Expected 200 OK") + + // Parse the response body to get the URL where the login page would post the user-entered credentials + if err := client.ParseLoginForm(res.Body, keyCloakLoginFormID); err != nil { + tlog.Logf(t, "failed to parse login form: %v", err) + return false, nil + } + + t.Log("successfully parsed login form") + return true, nil + }); err != nil { + t.Errorf("failed to parse login form: %v", err) + } + + // Submit the login form to the IdP. + // This will authenticate and redirect back to the application + res, err := client.Login(map[string]string{"username": username, "password": password, "credentialId": ""}) + require.NoError(t, err, "Failed to login to the IdP") + + // Verify that we get the expected response from the application + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + require.Contains(t, string(body), "infra-backend-v1", "Expected response from the application") + + // Verify that we can access the application without logging in again + res, err = client.Get(testURL, false) + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + require.Contains(t, string(body), "infra-backend-v1", "Expected response from the application") + + // Verify that we can logout + // Note: OAuth2 filter just clears its cookies and does not log out from the IdP. + res, err = client.Get(logoutURL, false) + require.NoError(t, err) + require.Equal(t, http.StatusFound, res.StatusCode) + + // After logout, OAuth2 filter will redirect back to the root of the host, e.g, "www.example.com". + // Ideally, this should redirect to the application's root, e.g, "www.example.com/myapp", + // but Envoy OAuth2 filter does not support this yet. + require.Equal(t, "http://www.example.com/", res.Header.Get("Location"), "Expected redirect to the root of the host") + + // Verify that the oauth2 cookies have been deleted + var cookieDeleted bool + deletedCookies := res.Header.Values("Set-Cookie") + regx := regexp.MustCompile("^IdToken-.+=deleted.+") + for _, cookie := range deletedCookies { + if regx.Match([]byte(cookie)) { + cookieDeleted = true + } + } + require.True(t, cookieDeleted, "IdToken cookie not deleted") +} diff --git a/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml b/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml index 8a1513469dea..3746f12a18d4 100644 --- a/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml +++ b/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml @@ -204,10 +204,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -218,6 +220,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -229,6 +232,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/control-plane-with-pdb.out.yaml b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml index a71e46fe7bdc..582fab0e8b90 100644 --- a/test/helm/gateway-helm/control-plane-with-pdb.out.yaml +++ b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml @@ -219,10 +219,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -233,6 +235,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -244,6 +247,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/default-config.out.yaml b/test/helm/gateway-helm/default-config.out.yaml index 140d271c6e71..b4aac21b12c4 100644 --- a/test/helm/gateway-helm/default-config.out.yaml +++ b/test/helm/gateway-helm/default-config.out.yaml @@ -204,10 +204,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -218,6 +220,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -229,6 +232,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/deployment-custom-topology.out.yaml b/test/helm/gateway-helm/deployment-custom-topology.out.yaml index 586b64b55849..37f58022beb1 100644 --- a/test/helm/gateway-helm/deployment-custom-topology.out.yaml +++ b/test/helm/gateway-helm/deployment-custom-topology.out.yaml @@ -204,10 +204,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -218,6 +220,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -229,6 +232,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/deployment-images-config.out.yaml b/test/helm/gateway-helm/deployment-images-config.out.yaml index 10f849e1d77c..8506b87b0ee9 100644 --- a/test/helm/gateway-helm/deployment-images-config.out.yaml +++ b/test/helm/gateway-helm/deployment-images-config.out.yaml @@ -204,10 +204,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -218,6 +220,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -229,6 +232,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/deployment-priorityclass.out.yaml b/test/helm/gateway-helm/deployment-priorityclass.out.yaml index 4f735c420957..3f82746416be 100644 --- a/test/helm/gateway-helm/deployment-priorityclass.out.yaml +++ b/test/helm/gateway-helm/deployment-priorityclass.out.yaml @@ -204,10 +204,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -218,6 +220,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -229,6 +232,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/envoy-gateway-config.out.yaml b/test/helm/gateway-helm/envoy-gateway-config.out.yaml index 04159958265a..9a3f5b4846b3 100644 --- a/test/helm/gateway-helm/envoy-gateway-config.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-config.out.yaml @@ -206,10 +206,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -220,6 +222,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -231,6 +234,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/global-images-config.out.yaml b/test/helm/gateway-helm/global-images-config.out.yaml index f280fc9f218b..38be258c7a33 100644 --- a/test/helm/gateway-helm/global-images-config.out.yaml +++ b/test/helm/gateway-helm/global-images-config.out.yaml @@ -208,10 +208,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -222,6 +224,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -233,6 +236,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/test/helm/gateway-helm/service-annotations.out.yaml b/test/helm/gateway-helm/service-annotations.out.yaml index ec50a16e30de..532988e919e0 100644 --- a/test/helm/gateway-helm/service-annotations.out.yaml +++ b/test/helm/gateway-helm/service-annotations.out.yaml @@ -204,10 +204,12 @@ rules: resources: - serviceaccounts - services + - configmaps verbs: - create - get - delete + - deletecollection - patch - apiGroups: - apps @@ -218,6 +220,7 @@ rules: - create - get - delete + - deletecollection - patch - apiGroups: - autoscaling @@ -229,6 +232,7 @@ rules: - create - get - delete + - deletecollection - patch --- # Source: gateway-helm/templates/leader-election-rbac.yaml diff --git a/tools/hack/deployment-exists.sh b/tools/hack/deployment-exists.sh index cc50e1c26389..d5081b424bce 100755 --- a/tools/hack/deployment-exists.sh +++ b/tools/hack/deployment-exists.sh @@ -3,9 +3,8 @@ DEPLOYMENT_LABEL_SELECTOR=$1 DEPLOYMENT_NAMESPACE=$2 - # Timeout for deployment to exist (in seconds) -exist_timeout=25 +exist_timeout=300 end=$((SECONDS+exist_timeout)) while true; do diff --git a/tools/make/kube.mk b/tools/make/kube.mk index c659e2347878..91c1580ab2d5 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -138,9 +138,6 @@ e2e: create-cluster kube-install-image kube-deploy install-ratelimit install-e2e install-ratelimit: @$(LOG_TARGET) kubectl apply -f examples/redis/redis.yaml - kubectl rollout restart deployment envoy-gateway -n envoy-gateway-system - kubectl rollout status --watch --timeout=5m -n envoy-gateway-system deployment/envoy-gateway - kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available tools/hack/deployment-exists.sh "app.kubernetes.io/name=envoy-ratelimit" "envoy-gateway-system" kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-ratelimit --for=condition=Available