diff --git a/api/v1alpha1/backendtrafficpolicy_types.go b/api/v1alpha1/backendtrafficpolicy_types.go index 77d5d4c0416..ce8e6b93037 100644 --- a/api/v1alpha1/backendtrafficpolicy_types.go +++ b/api/v1alpha1/backendtrafficpolicy_types.go @@ -66,10 +66,16 @@ type BackendTrafficPolicySpec struct { // +optional TCPKeepalive *TCPKeepalive `json:"tcpKeepalive,omitempty"` + // HealthCheck allows gateway to perform active health checking on backends. + // + // +optional + HealthCheck *HealthCheck `json:"healthCheck,omitempty"` + // FaultInjection defines the fault injection policy to be applied. This configuration can be used to // inject delays and abort requests to mimic failure scenarios such as service failures and overloads // +optional FaultInjection *FaultInjection `json:"faultInjection,omitempty"` + // Circuit Breaker settings for the upstream connections and requests. // If not set, circuit breakers will be enabled with the default thresholds // diff --git a/api/v1alpha1/healthcheck_types.go b/api/v1alpha1/healthcheck_types.go new file mode 100644 index 00000000000..a308c7d5719 --- /dev/null +++ b/api/v1alpha1/healthcheck_types.go @@ -0,0 +1,133 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// HealthCheck defines the health check configuration. +// EG supports various types of health checking including HTTP, TCP. +// +union +// +// +kubebuilder:validation:XValidation:rule="self.type == 'HTTP' ? has(self.http) : !has(self.http)",message="If Health Checker type is HTTP, http field needs to be set." +// +kubebuilder:validation:XValidation:rule="self.type == 'TCP' ? has(self.tcp) : !has(self.tcp)",message="If Health Checker type is TCP, tcp field needs to be set." +type HealthCheck struct { + // Timeout defines the time to wait for a health check response. + // + // +kubebuilder:validation:Format=duration + // +kubebuilder:default="1s" + // +optional + Timeout *metav1.Duration `json:"timeout"` + + // Interval defines the time between health checks. + // + // +kubebuilder:validation:Format=duration + // +kubebuilder:default="3s" + // +optional + Interval *metav1.Duration `json:"interval"` + + // UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + // + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=3 + // +optional + UnhealthyThreshold *uint32 `json:"unhealthyThreshold"` + + // HealthyThreshold defines the number of healthy health checks required before a backend host is marked healthy. + // + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=1 + // +optional + HealthyThreshold *uint32 `json:"healthyThreshold"` + + // Type defines the type of health checker. + // +kubebuilder:validation:Enum=HTTP;TCP + // +unionDiscriminator + Type HealthCheckerType `json:"type" yaml:"type"` + + // HTTP defines the configuration of http health checker. + // It's required while the health checker type is HTTP. + // +optional + HTTP *HTTPHealthChecker `json:"http,omitempty" yaml:"http,omitempty"` + + // TCP defines the configuration of tcp health checker. + // It's required while the health checker type is TCP. + // +optional + TCP *TCPHealthChecker `json:"tcp,omitempty" yaml:"tcp,omitempty"` +} + +// HealthCheckerType is the type of health checker. +// +kubebuilder:validation:Enum=HTTP;TCP +type HealthCheckerType string + +const ( + // HealthCheckerTypeHTTP defines the HTTP type of health checking. + HealthCheckerTypeHTTP HealthCheckerType = "HTTP" + // HealthCheckerTypeTCP defines the TCP type of health checking. + HealthCheckerTypeTCP HealthCheckerType = "TCP" +) + +// HTTPHealthChecker defines the settings of http health check. +type HTTPHealthChecker struct { + // Path defines the HTTP path that will be requested during health checking. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=1024 + Path string `json:"path" yaml:"path"` + // Method defines the HTTP method used for health checking. + // Defaults to GET + // +optional + Method *string `json:"method,omitempty" yaml:"method,omitempty"` + // ExpectedStatuses defines a list of HTTP response statuses considered healthy. + // Defaults to 200 only + // +optional + ExpectedStatuses []HTTPStatus `json:"expectedStatuses,omitempty" yaml:"expectedStatuses,omitempty"` + // ExpectedResponse defines a list of HTTP expected responses to match. + // +optional + ExpectedResponse *HealthCheckPayload `json:"expectedResponse,omitempty" yaml:"expectedResponse,omitempty"` +} + +// TCPHealthChecker defines the settings of tcp health check. +type TCPHealthChecker struct { + // Send defines the request payload. + // +optional + Send *HealthCheckPayload `json:"send,omitempty" yaml:"send,omitempty"` + // Receive defines the expected response payload. + // +optional + Receive *HealthCheckPayload `json:"receive,omitempty" yaml:"receive,omitempty"` +} + +// HTTPStatus defines the http status code. +// +kubebuilder:validation:Minimum=100 +// +kubebuilder:validation:Maximum=600 +// +kubebuilder:validation:ExclusiveMaximum=true +type HTTPStatus int + +// HealthCheckPayloadType is the type of the payload. +// +kubebuilder:validation:Enum=Text;Binary +type HealthCheckPayloadType string + +const ( + // HealthCheckPayloadTypeText defines the Text type payload. + HealthCheckPayloadTypeText HealthCheckPayloadType = "Text" + // HealthCheckPayloadTypeBinary defines the Binary type payload. + HealthCheckPayloadTypeBinary HealthCheckPayloadType = "Binary" +) + +// HealthCheckPayload defines the encoding of the payload bytes in the payload. +// +union +// +kubebuilder:validation:XValidation:rule="self.type == 'Text' ? has(self.text) : !has(self.text)",message="If payload type is Text, text field needs to be set." +// +kubebuilder:validation:XValidation:rule="self.type == 'Binary' ? has(self.binary) : !has(self.binary)",message="If payload type is Binary, binary field needs to be set." +type HealthCheckPayload struct { + // Type defines the type of the payload. + // +kubebuilder:validation:Enum=Text;Binary + // +unionDiscriminator + Type HealthCheckPayloadType `json:"type" yaml:"type"` + // Text payload in plain text. + // +optional + Text *string `json:"text,omitempty" yaml:"text,omitempty"` + // Binary payload base64 encoded. + // +optional + Binary []byte `json:"binary,omitempty" yaml:"binary,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 57b16a3ed10..42aa61b0dec 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -101,6 +101,11 @@ func (in *BackendTrafficPolicySpec) DeepCopyInto(out *BackendTrafficPolicySpec) *out = new(TCPKeepalive) (*in).DeepCopyInto(*out) } + if in.HealthCheck != nil { + in, out := &in.HealthCheck, &out.HealthCheck + *out = new(HealthCheck) + (*in).DeepCopyInto(*out) + } if in.FaultInjection != nil { in, out := &in.FaultInjection, &out.FaultInjection *out = new(FaultInjection) @@ -1377,6 +1382,36 @@ func (in *HTTP3Settings) DeepCopy() *HTTP3Settings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPHealthChecker) DeepCopyInto(out *HTTPHealthChecker) { + *out = *in + if in.Method != nil { + in, out := &in.Method, &out.Method + *out = new(string) + **out = **in + } + if in.ExpectedStatuses != nil { + in, out := &in.ExpectedStatuses, &out.ExpectedStatuses + *out = make([]HTTPStatus, len(*in)) + copy(*out, *in) + } + if in.ExpectedResponse != nil { + in, out := &in.ExpectedResponse, &out.ExpectedResponse + *out = new(HealthCheckPayload) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPHealthChecker. +func (in *HTTPHealthChecker) DeepCopy() *HTTPHealthChecker { + if in == nil { + return nil + } + out := new(HTTPHealthChecker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HeaderMatch) DeepCopyInto(out *HeaderMatch) { *out = *in @@ -1402,6 +1437,76 @@ func (in *HeaderMatch) DeepCopy() *HeaderMatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthCheck) DeepCopyInto(out *HealthCheck) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(v1.Duration) + **out = **in + } + if in.Interval != nil { + in, out := &in.Interval, &out.Interval + *out = new(v1.Duration) + **out = **in + } + if in.UnhealthyThreshold != nil { + in, out := &in.UnhealthyThreshold, &out.UnhealthyThreshold + *out = new(uint32) + **out = **in + } + if in.HealthyThreshold != nil { + in, out := &in.HealthyThreshold, &out.HealthyThreshold + *out = new(uint32) + **out = **in + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPHealthChecker) + (*in).DeepCopyInto(*out) + } + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = new(TCPHealthChecker) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthCheck. +func (in *HealthCheck) DeepCopy() *HealthCheck { + if in == nil { + return nil + } + out := new(HealthCheck) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthCheckPayload) DeepCopyInto(out *HealthCheckPayload) { + *out = *in + if in.Text != nil { + in, out := &in.Text, &out.Text + *out = new(string) + **out = **in + } + if in.Binary != nil { + in, out := &in.Binary, &out.Binary + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthCheckPayload. +func (in *HealthCheckPayload) DeepCopy() *HealthCheckPayload { + if in == nil { + return nil + } + out := new(HealthCheckPayload) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *JSONPatchOperation) DeepCopyInto(out *JSONPatchOperation) { *out = *in @@ -2603,6 +2708,31 @@ func (in *StringMatch) DeepCopy() *StringMatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TCPHealthChecker) DeepCopyInto(out *TCPHealthChecker) { + *out = *in + if in.Send != nil { + in, out := &in.Send, &out.Send + *out = new(HealthCheckPayload) + (*in).DeepCopyInto(*out) + } + if in.Receive != nil { + in, out := &in.Receive, &out.Receive + *out = new(HealthCheckPayload) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPHealthChecker. +func (in *TCPHealthChecker) DeepCopy() *TCPHealthChecker { + if in == nil { + return nil + } + out := new(TCPHealthChecker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TCPKeepalive) DeepCopyInto(out *TCPKeepalive) { *out = *in diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml index 6373da4e516..36e51bcf023 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backendtrafficpolicies.yaml @@ -131,6 +131,177 @@ spec: x-kubernetes-validations: - message: Delay and abort faults are set at least one. rule: ' has(self.delay) || has(self.abort) ' + healthCheck: + description: HealthCheck allows gateway to perform active health checking + on backends. + properties: + healthyThreshold: + default: 1 + description: HealthyThreshold defines the number of healthy health + checks required before a backend host is marked healthy. + format: int32 + minimum: 1 + type: integer + http: + description: HTTP defines the configuration of http health checker. + It's required while the health checker type is HTTP. + properties: + expectedResponse: + description: ExpectedResponse defines a list of HTTP expected + responses to match. + properties: + binary: + description: Binary payload base64 encoded. + format: byte + type: string + text: + description: Text payload in plain text. + type: string + type: + allOf: + - enum: + - Text + - Binary + - enum: + - Text + - Binary + description: Type defines the type of the payload. + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: If payload type is Text, text field needs to be + set. + rule: 'self.type == ''Text'' ? has(self.text) : !has(self.text)' + - message: If payload type is Binary, binary field needs to + be set. + rule: 'self.type == ''Binary'' ? has(self.binary) : !has(self.binary)' + expectedStatuses: + description: ExpectedStatuses defines a list of HTTP response + statuses considered healthy. Defaults to 200 only + items: + description: HTTPStatus defines the http status code. + exclusiveMaximum: true + maximum: 600 + minimum: 100 + type: integer + type: array + method: + description: Method defines the HTTP method used for health + checking. Defaults to GET + type: string + path: + description: Path defines the HTTP path that will be requested + during health checking. + maxLength: 1024 + minLength: 1 + type: string + required: + - path + type: object + interval: + default: 3s + description: Interval defines the time between health checks. + format: duration + type: string + tcp: + description: TCP defines the configuration of tcp health checker. + It's required while the health checker type is TCP. + properties: + receive: + description: Receive defines the expected response payload. + properties: + binary: + description: Binary payload base64 encoded. + format: byte + type: string + text: + description: Text payload in plain text. + type: string + type: + allOf: + - enum: + - Text + - Binary + - enum: + - Text + - Binary + description: Type defines the type of the payload. + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: If payload type is Text, text field needs to be + set. + rule: 'self.type == ''Text'' ? has(self.text) : !has(self.text)' + - message: If payload type is Binary, binary field needs to + be set. + rule: 'self.type == ''Binary'' ? has(self.binary) : !has(self.binary)' + send: + description: Send defines the request payload. + properties: + binary: + description: Binary payload base64 encoded. + format: byte + type: string + text: + description: Text payload in plain text. + type: string + type: + allOf: + - enum: + - Text + - Binary + - enum: + - Text + - Binary + description: Type defines the type of the payload. + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: If payload type is Text, text field needs to be + set. + rule: 'self.type == ''Text'' ? has(self.text) : !has(self.text)' + - message: If payload type is Binary, binary field needs to + be set. + rule: 'self.type == ''Binary'' ? has(self.binary) : !has(self.binary)' + type: object + timeout: + default: 1s + description: Timeout defines the time to wait for a health check + response. + format: duration + type: string + type: + allOf: + - enum: + - HTTP + - TCP + - enum: + - HTTP + - TCP + description: Type defines the type of health checker. + type: string + unhealthyThreshold: + default: 3 + description: UnhealthyThreshold defines the number of unhealthy + health checks required before a backend host is marked unhealthy. + format: int32 + minimum: 1 + type: integer + required: + - type + type: object + x-kubernetes-validations: + - message: If Health Checker type is HTTP, http field needs to be + set. + rule: 'self.type == ''HTTP'' ? has(self.http) : !has(self.http)' + - message: If Health Checker type is TCP, tcp field needs to be set. + rule: 'self.type == ''TCP'' ? has(self.tcp) : !has(self.tcp)' loadBalancer: description: LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go index 8377b827022..e7975c308ec 100644 --- a/internal/gatewayapi/backendtrafficpolicy.go +++ b/internal/gatewayapi/backendtrafficpolicy.go @@ -245,6 +245,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen rl *ir.RateLimit lb *ir.LoadBalancer pp *ir.ProxyProtocol + hc *ir.HealthCheck cb *ir.CircuitBreaker fi *ir.FaultInjection ) @@ -259,6 +260,9 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen if policy.Spec.ProxyProtocol != nil { pp = t.buildProxyProtocol(policy) } + if policy.Spec.HealthCheck != nil { + hc = t.buildHealthCheck(policy) + } if policy.Spec.CircuitBreaker != nil { cb = t.buildCircuitBreaker(policy) } @@ -276,6 +280,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen r.RateLimit = rl r.LoadBalancer = lb r.ProxyProtocol = pp + r.HealthCheck = hc r.CircuitBreaker = cb r.FaultInjection = fi } @@ -290,6 +295,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back rl *ir.RateLimit lb *ir.LoadBalancer pp *ir.ProxyProtocol + hc *ir.HealthCheck cb *ir.CircuitBreaker fi *ir.FaultInjection ) @@ -304,6 +310,9 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back if policy.Spec.ProxyProtocol != nil { pp = t.buildProxyProtocol(policy) } + if policy.Spec.HealthCheck != nil { + hc = t.buildHealthCheck(policy) + } if policy.Spec.CircuitBreaker != nil { cb = t.buildCircuitBreaker(policy) } @@ -330,6 +339,9 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back if r.ProxyProtocol == nil { r.ProxyProtocol = pp } + if r.HealthCheck == nil { + r.HealthCheck = hc + } if r.CircuitBreaker == nil { r.CircuitBreaker = cb } @@ -658,6 +670,85 @@ func (t *Translator) buildProxyProtocol(policy *egv1a1.BackendTrafficPolicy) *ir return pp } +func (t *Translator) buildHealthCheck(policy *egv1a1.BackendTrafficPolicy) *ir.HealthCheck { + if policy.Spec.HealthCheck == nil { + return nil + } + + hc := policy.Spec.HealthCheck + irHC := &ir.HealthCheck{ + Timeout: hc.Timeout, + Interval: hc.Interval, + UnhealthyThreshold: hc.UnhealthyThreshold, + HealthyThreshold: hc.HealthyThreshold, + } + switch hc.Type { + case egv1a1.HealthCheckerTypeHTTP: + irHC.HTTP = t.buildHTTPHealthChecker(hc.HTTP) + case egv1a1.HealthCheckerTypeTCP: + irHC.TCP = t.buildTCPHealthChecker(hc.TCP) + } + + return irHC +} + +func (t *Translator) buildHTTPHealthChecker(h *egv1a1.HTTPHealthChecker) *ir.HTTPHealthChecker { + if h == nil { + return nil + } + + irHTTP := &ir.HTTPHealthChecker{ + Path: h.Path, + Method: h.Method, + } + if irHTTP.Method != nil { + *irHTTP.Method = strings.ToUpper(*irHTTP.Method) + } + + var irStatuses []ir.HTTPStatus + // deduplicate http statuses + statusSet := make(map[egv1a1.HTTPStatus]bool, len(h.ExpectedStatuses)) + for _, r := range h.ExpectedStatuses { + if _, ok := statusSet[r]; !ok { + statusSet[r] = true + irStatuses = append(irStatuses, ir.HTTPStatus(r)) + } + } + irHTTP.ExpectedStatuses = irStatuses + + irHTTP.ExpectedResponse = translateHealthCheckPayload(h.ExpectedResponse) + return irHTTP +} + +func (t *Translator) buildTCPHealthChecker(h *egv1a1.TCPHealthChecker) *ir.TCPHealthChecker { + if h == nil { + return nil + } + + irTCP := &ir.TCPHealthChecker{ + Send: translateHealthCheckPayload(h.Send), + Receive: translateHealthCheckPayload(h.Receive), + } + return irTCP +} + +func translateHealthCheckPayload(p *egv1a1.HealthCheckPayload) *ir.HealthCheckPayload { + if p == nil { + return nil + } + + irPayload := &ir.HealthCheckPayload{} + switch p.Type { + case egv1a1.HealthCheckPayloadTypeText: + irPayload.Text = p.Text + case egv1a1.HealthCheckPayloadTypeBinary: + irPayload.Binary = make([]byte, len(p.Binary)) + copy(irPayload.Binary, p.Binary) + } + + return irPayload +} + func ratelimitUnitToDuration(unit egv1a1.RateLimitUnit) int64 { var seconds int64 diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-healthcheck.in.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-healthcheck.in.yaml new file mode 100644 index 00000000000..31a494a527d --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-healthcheck.in.yaml @@ -0,0 +1,203 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-2 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-2 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/v2" + backendRefs: + - name: service-2 + port: 8080 +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-3 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/v3" + backendRefs: + - name: service-3 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + healthCheck: + timeout: "500ms" + interval: "3s" + unhealthyThreshold: 3 + healthyThreshold: 1 + type: HTTP + http: + path: "/healthz" + method: "GET" + expectedStatuses: + - 200 + - 300 + expectedResponse: + type: Binary + binary: RXZlcnl0aGluZyBPSw== +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route-1 + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + healthCheck: + timeout: "1s" + interval: "5s" + unhealthyThreshold: 3 + healthyThreshold: 3 + type: HTTP + http: + path: "/healthz" + method: "GET" + expectedStatuses: + - 200 + - 201 + expectedResponse: + type: Text + text: pong +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route-2 + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-2 + namespace: default + healthCheck: + timeout: "1s" + interval: "5s" + unhealthyThreshold: 3 + healthyThreshold: 3 + type: TCP + tcp: + send: + type: Text + text: ping + receive: + type: Text + text: pong +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route-3 + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-3 + namespace: default + healthCheck: + timeout: 1s + interval: 3s + unhealthyThreshold: 3 + healthyThreshold: 1 + type: TCP + tcp: + send: + type: Binary + binary: cGluZw== + receive: + type: Binary + binary: RXZlcnl0aGluZyBPSw== diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-healthcheck.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-healthcheck.out.yaml new file mode 100755 index 00000000000..b8b9b641bb3 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-healthcheck.out.yaml @@ -0,0 +1,533 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route-1 + namespace: default + spec: + healthCheck: + healthyThreshold: 3 + http: + expectedResponse: + text: pong + type: Text + expectedStatuses: + - 200 + - 201 + method: GET + path: /healthz + interval: 5s + timeout: 1s + type: HTTP + unhealthyThreshold: 3 + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route-2 + namespace: default + spec: + healthCheck: + healthyThreshold: 3 + interval: 5s + tcp: + receive: + text: pong + type: Text + send: + text: ping + type: Text + timeout: 1s + type: TCP + unhealthyThreshold: 3 + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-2 + namespace: default + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route-3 + namespace: default + spec: + healthCheck: + healthyThreshold: 1 + interval: 3s + tcp: + receive: + binary: RXZlcnl0aGluZyBPSw== + type: Binary + send: + binary: cGluZw== + type: Binary + timeout: 1s + type: TCP + unhealthyThreshold: 3 + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-3 + namespace: default + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + healthCheck: + healthyThreshold: 1 + http: + expectedResponse: + binary: RXZlcnl0aGluZyBPSw== + type: Binary + expectedStatuses: + - 200 + - 300 + method: GET + path: /healthz + interval: 3s + timeout: 500ms + type: HTTP + unhealthyThreshold: 3 + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-2 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 3 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-2 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-2 + port: 8080 + matches: + - path: + value: /v2 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-3 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-3 + port: 8080 + matches: + - path: + value: /v3 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 + envoy-gateway/gateway-2: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-2/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-2 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-2 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + name: envoy-gateway/gateway-1/http + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: GRPC + weight: 1 + healthCheck: + healthyThreshold: 1 + http: + expectedResponse: + binary: RXZlcnl0aGluZyBPSw== + expectedStatuses: + - 200 + - 300 + method: GET + path: /healthz + interval: 3s + timeout: 500ms + unhealthyThreshold: 3 + hostname: '*' + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* + envoy-gateway/gateway-2: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-2/http + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: httproute/default/httproute-2/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + healthCheck: + healthyThreshold: 3 + interval: 5s + tcp: + receive: + text: pong + send: + text: ping + timeout: 1s + unhealthyThreshold: 3 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-2/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: /v2 + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: httproute/default/httproute-3/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + healthCheck: + healthyThreshold: 1 + interval: 3s + tcp: + receive: + binary: RXZlcnl0aGluZyBPSw== + send: + binary: cGluZw== + timeout: 1s + unhealthyThreshold: 3 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-3/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: /v3 + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + healthCheck: + healthyThreshold: 3 + http: + expectedResponse: + text: pong + expectedStatuses: + - 200 + - 201 + method: GET + path: /healthz + interval: 5s + timeout: 1s + unhealthyThreshold: 3 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / diff --git a/internal/ir/xds.go b/internal/ir/xds.go index ceda60007f8..b63cf65ded3 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -9,6 +9,7 @@ import ( "cmp" "errors" "net" + "net/http" "reflect" "github.com/tetratelabs/multierror" @@ -24,29 +25,39 @@ import ( ) var ( - ErrListenerNameEmpty = errors.New("field Name must be specified") - ErrListenerAddressInvalid = errors.New("field Address must be a valid IP address") - ErrListenerPortInvalid = errors.New("field Port specified is invalid") - ErrHTTPListenerHostnamesEmpty = errors.New("field Hostnames must be specified with at least a single hostname entry") - ErrTCPListenerSNIsEmpty = errors.New("field SNIs must be specified with at least a single server name entry") - ErrTLSServerCertEmpty = errors.New("field ServerCertificate must be specified") - ErrTLSPrivateKey = errors.New("field PrivateKey must be specified") - ErrHTTPRouteNameEmpty = errors.New("field Name must be specified") - ErrHTTPRouteHostnameEmpty = errors.New("field Hostname must be specified") - ErrDestinationNameEmpty = errors.New("field Name must be specified") - ErrDestEndpointHostInvalid = errors.New("field Address must be a valid IP or FQDN address") - ErrDestEndpointPortInvalid = errors.New("field Port specified is invalid") - ErrStringMatchConditionInvalid = errors.New("only one of the Exact, Prefix, SafeRegex or Distinct fields must be set") - ErrStringMatchNameIsEmpty = errors.New("field Name must be specified") - ErrDirectResponseStatusInvalid = errors.New("only HTTP status codes 100 - 599 are supported for DirectResponse") - ErrRedirectUnsupportedStatus = errors.New("only HTTP status codes 301 and 302 are supported for redirect filters") - ErrRedirectUnsupportedScheme = errors.New("only http and https are supported for the scheme in redirect filters") - ErrHTTPPathModifierDoubleReplace = errors.New("redirect filter cannot have a path modifier that supplies both fullPathReplace and prefixMatchReplace") - ErrHTTPPathModifierNoReplace = errors.New("redirect filter cannot have a path modifier that does not supply either fullPathReplace or prefixMatchReplace") - ErrAddHeaderEmptyName = errors.New("header modifier filter cannot configure a header without a name to be added") - ErrAddHeaderDuplicate = errors.New("header modifier filter attempts to add the same header more than once (case insensitive)") - ErrRemoveHeaderDuplicate = errors.New("header modifier filter attempts to remove the same header more than once (case insensitive)") - ErrLoadBalancerInvalid = errors.New("loadBalancer setting is invalid, only one setting can be set") + ErrListenerNameEmpty = errors.New("field Name must be specified") + ErrListenerAddressInvalid = errors.New("field Address must be a valid IP address") + ErrListenerPortInvalid = errors.New("field Port specified is invalid") + ErrHTTPListenerHostnamesEmpty = errors.New("field Hostnames must be specified with at least a single hostname entry") + ErrTCPListenerSNIsEmpty = errors.New("field SNIs must be specified with at least a single server name entry") + ErrTLSServerCertEmpty = errors.New("field ServerCertificate must be specified") + ErrTLSPrivateKey = errors.New("field PrivateKey must be specified") + ErrHTTPRouteNameEmpty = errors.New("field Name must be specified") + ErrHTTPRouteHostnameEmpty = errors.New("field Hostname must be specified") + ErrDestinationNameEmpty = errors.New("field Name must be specified") + ErrDestEndpointHostInvalid = errors.New("field Address must be a valid IP or FQDN address") + ErrDestEndpointPortInvalid = errors.New("field Port specified is invalid") + ErrStringMatchConditionInvalid = errors.New("only one of the Exact, Prefix, SafeRegex or Distinct fields must be set") + ErrStringMatchNameIsEmpty = errors.New("field Name must be specified") + ErrDirectResponseStatusInvalid = errors.New("only HTTP status codes 100 - 599 are supported for DirectResponse") + ErrRedirectUnsupportedStatus = errors.New("only HTTP status codes 301 and 302 are supported for redirect filters") + ErrRedirectUnsupportedScheme = errors.New("only http and https are supported for the scheme in redirect filters") + ErrHTTPPathModifierDoubleReplace = errors.New("redirect filter cannot have a path modifier that supplies both fullPathReplace and prefixMatchReplace") + ErrHTTPPathModifierNoReplace = errors.New("redirect filter cannot have a path modifier that does not supply either fullPathReplace or prefixMatchReplace") + ErrAddHeaderEmptyName = errors.New("header modifier filter cannot configure a header without a name to be added") + ErrAddHeaderDuplicate = errors.New("header modifier filter attempts to add the same header more than once (case insensitive)") + ErrRemoveHeaderDuplicate = errors.New("header modifier filter attempts to remove the same header more than once (case insensitive)") + ErrLoadBalancerInvalid = errors.New("loadBalancer setting is invalid, only one setting can be set") + ErrHealthCheckTimeoutInvalid = errors.New("field HealthCheck.Timeout must be specified") + ErrHealthCheckIntervalInvalid = errors.New("field HealthCheck.Interval must be specified") + ErrHealthCheckUnhealthyThresholdInvalid = errors.New("field HealthCheck.UnhealthyThreshold should be greater than 0") + ErrHealthCheckHealthyThresholdInvalid = errors.New("field HealthCheck.HealthyThreshold should be greater than 0") + ErrHealthCheckerInvalid = errors.New("health checker setting is invalid, only one health checker can be set") + ErrHCHTTPPathInvalid = errors.New("field HTTPHealthChecker.Path should be specified") + ErrHCHTTPMethodInvalid = errors.New("only one of the GET, HEAD, POST, DELETE, OPTIONS, TRACE, PATCH of HTTPHealthChecker.Method could be set") + ErrHCHTTPExpectedStatusesInvalid = errors.New("field HTTPHealthChecker.ExpectedStatuses should be specified") + ErrHealthCheckPayloadInvalid = errors.New("one of Text, Binary fields must be set in payload") + ErrHTTPStatusInvalid = errors.New("HTTPStatus should be in [200,600)") ) // Xds holds the intermediate representation of a Gateway and is @@ -307,6 +318,8 @@ type HTTPRoute struct { ProxyProtocol *ProxyProtocol `json:"proxyProtocol,omitempty" yaml:"proxyProtocol,omitempty"` // BasicAuth defines the schema for the HTTP Basic Authentication. BasicAuth *BasicAuth `json:"basicAuth,omitempty" yaml:"basicAuth,omitempty"` + // HealthCheck defines the configuration for active health checking on the upstream. + HealthCheck *HealthCheck `json:"healthCheck,omitempty" yaml:"healthCheck,omitempty"` // FaultInjection defines the schema for injecting faults into HTTP requests. FaultInjection *FaultInjection `json:"faultInjection,omitempty" yaml:"faultInjection,omitempty"` // ExtensionRefs holds unstructured resources that were introduced by an extension and used on the HTTPRoute as extensionRef filters @@ -534,6 +547,11 @@ func (h HTTPRoute) Validate() error { errs = multierror.Append(errs, err) } } + if h.HealthCheck != nil { + if err := h.HealthCheck.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } return errs } @@ -1198,3 +1216,170 @@ type CircuitBreaker struct { // The maximum number of parallel requests that Envoy will make. MaxParallelRequests *uint32 `json:"maxParallelRequests,omitempty" yaml:"maxParallelRequests,omitempty"` } + +// HealthCheck defines health check settings +// +k8s:deepcopy-gen=true +type HealthCheck struct { + // Timeout defines the time to wait for a health check response. + Timeout *metav1.Duration `json:"timeout"` + // Interval defines the time between health checks. + Interval *metav1.Duration `json:"interval"` + // UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. + UnhealthyThreshold *uint32 `json:"unhealthyThreshold"` + // HealthyThreshold defines the number of healthy health checks required before a backend host is marked healthy. + HealthyThreshold *uint32 `json:"healthyThreshold"` + // HTTP defines the configuration of http health checker. + HTTP *HTTPHealthChecker `json:"http,omitempty" yaml:"http,omitempty"` + // TCP defines the configuration of tcp health checker. + TCP *TCPHealthChecker `json:"tcp,omitempty" yaml:"tcp,omitempty"` +} + +// Validate the fields within the HealthCheck structure. +func (h *HealthCheck) Validate() error { + var errs error + + if h.Timeout != nil && h.Timeout.Duration == 0 { + errs = multierror.Append(errs, ErrHealthCheckTimeoutInvalid) + } + if h.Interval != nil && h.Interval.Duration == 0 { + errs = multierror.Append(errs, ErrHealthCheckIntervalInvalid) + } + if h.UnhealthyThreshold != nil && *h.UnhealthyThreshold == 0 { + errs = multierror.Append(errs, ErrHealthCheckUnhealthyThresholdInvalid) + } + if h.HealthyThreshold != nil && *h.HealthyThreshold == 0 { + errs = multierror.Append(errs, ErrHealthCheckHealthyThresholdInvalid) + } + + matchCount := 0 + if h.HTTP != nil { + matchCount++ + } + if h.TCP != nil { + matchCount++ + } + if matchCount != 1 { + errs = multierror.Append(errs, ErrHealthCheckerInvalid) + } + + if h.HTTP != nil { + if err := h.HTTP.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + if h.TCP != nil { + if err := h.TCP.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + + return errs +} + +// HTTPHealthChecker defines the settings of http health check. +// +k8s:deepcopy-gen=true +type HTTPHealthChecker struct { + // Path defines the HTTP path that will be requested during health checking. + Path string `json:"path" yaml:"path"` + // Method defines the HTTP method used for health checking. + Method *string `json:"method,omitempty" yaml:"method,omitempty"` + // ExpectedStatuses defines a list of HTTP response statuses considered healthy. + ExpectedStatuses []HTTPStatus `json:"expectedStatuses,omitempty" yaml:"expectedStatuses,omitempty"` + // ExpectedResponse defines a list of HTTP expected responses to match. + ExpectedResponse *HealthCheckPayload `json:"expectedResponse,omitempty" yaml:"expectedResponses,omitempty"` +} + +// Validate the fields within the HTTPHealthChecker structure. +func (c *HTTPHealthChecker) Validate() error { + var errs error + if c.Path == "" { + errs = multierror.Append(errs, ErrHCHTTPPathInvalid) + } + if c.Method != nil { + switch *c.Method { + case http.MethodGet: + case http.MethodHead: + case http.MethodPost: + case http.MethodPut: + case http.MethodDelete: + case http.MethodOptions: + case http.MethodTrace: + case http.MethodPatch: + case "": + default: + errs = multierror.Append(errs, ErrHCHTTPMethodInvalid) + } + } + if len(c.ExpectedStatuses) == 0 { + errs = multierror.Append(errs, ErrHCHTTPExpectedStatusesInvalid) + } + for _, r := range c.ExpectedStatuses { + if err := r.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + if c.ExpectedResponse != nil { + if err := c.ExpectedResponse.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs +} + +// HTTPStatus represents http status code. +type HTTPStatus int + +func (h HTTPStatus) Validate() error { + if h < 100 || h >= 600 { + return ErrHTTPStatusInvalid + } + return nil +} + +// TCPHealthChecker defines the settings of tcp health check. +// +k8s:deepcopy-gen=true +type TCPHealthChecker struct { + Send *HealthCheckPayload `json:"send,omitempty" yaml:"send,omitempty"` + Receive *HealthCheckPayload `json:"receive,omitempty" yaml:"receive,omitempty"` +} + +// Validate the fields within the TCPHealthChecker structure. +func (c *TCPHealthChecker) Validate() error { + var errs error + if c.Send != nil { + if err := c.Send.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + if c.Receive != nil { + if err := c.Receive.Validate(); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs +} + +// HealthCheckPayload defines the encoding of the payload bytes in the payload. +// +k8s:deepcopy-gen=true +type HealthCheckPayload struct { + // Text payload in plain text. + Text *string `json:"text,omitempty" yaml:"text,omitempty"` + // Binary payload base64 encoded + Binary []byte `json:"binary,omitempty" yaml:"binary,omitempty"` +} + +// Validate the fields in the HealthCheckPayload. +func (p *HealthCheckPayload) Validate() error { + var errs error + matchCount := 0 + if p.Text != nil && *p.Text != "" { + matchCount++ + } + if len(p.Binary) > 0 { + matchCount++ + } + if matchCount != 1 { + errs = multierror.Append(errs, ErrHealthCheckPayloadInvalid) + } + return errs +} diff --git a/internal/ir/xds_test.go b/internal/ir/xds_test.go index cca7ef581ae..0fc54c2eb39 100644 --- a/internal/ir/xds_test.go +++ b/internal/ir/xds_test.go @@ -6,11 +6,14 @@ package ir import ( + "net/http" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -1219,3 +1222,198 @@ func TestPrintable(t *testing.T) { }) } } + +func TestValidateHealthCheck(t *testing.T) { + tests := []struct { + name string + input HealthCheck + want error + }{ + { + name: "invalid timeout", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Duration(0)}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To[uint32](3), + HealthyThreshold: ptr.To[uint32](3), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + ExpectedStatuses: []HTTPStatus{200, 400}, + }, + }, + want: ErrHealthCheckTimeoutInvalid, + }, + { + name: "invalid interval", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Duration(0)}, + UnhealthyThreshold: ptr.To[uint32](3), + HealthyThreshold: ptr.To[uint32](3), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodGet), + ExpectedStatuses: []HTTPStatus{200, 400}, + }, + }, + want: ErrHealthCheckIntervalInvalid, + }, + { + name: "invalid unhealthy threshold", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To[uint32](0), + HealthyThreshold: ptr.To[uint32](3), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodPatch), + ExpectedStatuses: []HTTPStatus{200, 400}, + }, + }, + want: ErrHealthCheckUnhealthyThresholdInvalid, + }, + { + name: "invalid healthy threshold", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To[uint32](3), + HealthyThreshold: ptr.To[uint32](0), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodPost), + ExpectedStatuses: []HTTPStatus{200, 400}, + }, + }, + want: ErrHealthCheckHealthyThresholdInvalid, + }, + { + name: "http-health-check: invalid path", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To[uint32](3), + HealthyThreshold: ptr.To[uint32](3), + HTTP: &HTTPHealthChecker{ + Path: "", + Method: ptr.To(http.MethodPut), + ExpectedStatuses: []HTTPStatus{200, 400}, + }, + }, + want: ErrHCHTTPPathInvalid, + }, + { + name: "http-health-check: invalid method", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To(uint32(3)), + HealthyThreshold: ptr.To(uint32(3)), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodConnect), + ExpectedStatuses: []HTTPStatus{200, 400}, + }, + }, + want: ErrHCHTTPMethodInvalid, + }, + { + name: "http-health-check: invalid expected-statuses", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To(uint32(3)), + HealthyThreshold: ptr.To(uint32(3)), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodDelete), + ExpectedStatuses: []HTTPStatus{}, + }, + }, + want: ErrHCHTTPExpectedStatusesInvalid, + }, + { + name: "http-health-check: invalid range", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To(uint32(3)), + HealthyThreshold: ptr.To(uint32(3)), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodHead), + ExpectedStatuses: []HTTPStatus{100, 600}, + }, + }, + want: ErrHTTPStatusInvalid, + }, + { + name: "http-health-check: invalid expected-responses", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To(uint32(3)), + HealthyThreshold: ptr.To(uint32(3)), + HTTP: &HTTPHealthChecker{ + Path: "/healthz", + Method: ptr.To(http.MethodOptions), + ExpectedStatuses: []HTTPStatus{200, 300}, + ExpectedResponse: &HealthCheckPayload{ + Text: ptr.To("foo"), + Binary: []byte{'f', 'o', 'o'}, + }, + }, + }, + want: ErrHealthCheckPayloadInvalid, + }, + { + name: "tcp-health-check: invalid send payload", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To(uint32(3)), + HealthyThreshold: ptr.To(uint32(3)), + TCP: &TCPHealthChecker{ + Send: &HealthCheckPayload{ + Text: ptr.To("foo"), + Binary: []byte{'f', 'o', 'o'}, + }, + Receive: &HealthCheckPayload{ + Text: ptr.To("foo"), + }, + }, + }, + want: ErrHealthCheckPayloadInvalid, + }, + { + name: "tcp-health-check: invalid receive payload", + input: HealthCheck{ + Timeout: &metav1.Duration{Duration: time.Second}, + Interval: &metav1.Duration{Duration: time.Second}, + UnhealthyThreshold: ptr.To(uint32(3)), + HealthyThreshold: ptr.To(uint32(3)), + TCP: &TCPHealthChecker{ + Send: &HealthCheckPayload{ + Text: ptr.To("foo"), + }, + Receive: &HealthCheckPayload{ + Text: ptr.To("foo"), + Binary: []byte{'f', 'o', 'o'}, + }, + }, + }, + want: ErrHealthCheckPayloadInvalid, + }, + } + for i := range tests { + test := tests[i] + t.Run(test.name, func(t *testing.T) { + if test.want == nil { + require.NoError(t, test.input.Validate()) + } else { + require.EqualError(t, test.input.Validate(), test.want.Error()) + } + }) + } +} diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index cc36162185e..5d8715ae2a1 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -417,6 +417,36 @@ func (in *GlobalRateLimit) DeepCopy() *GlobalRateLimit { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPHealthChecker) DeepCopyInto(out *HTTPHealthChecker) { + *out = *in + if in.Method != nil { + in, out := &in.Method, &out.Method + *out = new(string) + **out = **in + } + if in.ExpectedStatuses != nil { + in, out := &in.ExpectedStatuses, &out.ExpectedStatuses + *out = make([]HTTPStatus, len(*in)) + copy(*out, *in) + } + if in.ExpectedResponse != nil { + in, out := &in.ExpectedResponse, &out.ExpectedResponse + *out = new(HealthCheckPayload) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPHealthChecker. +func (in *HTTPHealthChecker) DeepCopy() *HTTPHealthChecker { + if in == nil { + return nil + } + out := new(HTTPHealthChecker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPListener) DeepCopyInto(out *HTTPListener) { *out = *in @@ -616,6 +646,11 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) { *out = new(BasicAuth) (*in).DeepCopyInto(*out) } + if in.HealthCheck != nil { + in, out := &in.HealthCheck, &out.HealthCheck + *out = new(HealthCheck) + (*in).DeepCopyInto(*out) + } if in.FaultInjection != nil { in, out := &in.FaultInjection, &out.FaultInjection *out = new(FaultInjection) @@ -649,6 +684,76 @@ func (in *HTTPRoute) DeepCopy() *HTTPRoute { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthCheck) DeepCopyInto(out *HealthCheck) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(v1.Duration) + **out = **in + } + if in.Interval != nil { + in, out := &in.Interval, &out.Interval + *out = new(v1.Duration) + **out = **in + } + if in.UnhealthyThreshold != nil { + in, out := &in.UnhealthyThreshold, &out.UnhealthyThreshold + *out = new(uint32) + **out = **in + } + if in.HealthyThreshold != nil { + in, out := &in.HealthyThreshold, &out.HealthyThreshold + *out = new(uint32) + **out = **in + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPHealthChecker) + (*in).DeepCopyInto(*out) + } + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = new(TCPHealthChecker) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthCheck. +func (in *HealthCheck) DeepCopy() *HealthCheck { + if in == nil { + return nil + } + out := new(HealthCheck) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthCheckPayload) DeepCopyInto(out *HealthCheckPayload) { + *out = *in + if in.Text != nil { + in, out := &in.Text, &out.Text + *out = new(string) + **out = **in + } + if in.Binary != nil { + in, out := &in.Binary, &out.Binary + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthCheckPayload. +func (in *HealthCheckPayload) DeepCopy() *HealthCheckPayload { + if in == nil { + return nil + } + out := new(HealthCheckPayload) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Infra) DeepCopyInto(out *Infra) { *out = *in @@ -1260,6 +1365,31 @@ func (in *StringMatch) DeepCopy() *StringMatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TCPHealthChecker) DeepCopyInto(out *TCPHealthChecker) { + *out = *in + if in.Send != nil { + in, out := &in.Send, &out.Send + *out = new(HealthCheckPayload) + (*in).DeepCopyInto(*out) + } + if in.Receive != nil { + in, out := &in.Receive, &out.Receive + *out = new(HealthCheckPayload) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPHealthChecker. +func (in *TCPHealthChecker) DeepCopy() *TCPHealthChecker { + if in == nil { + return nil + } + out := new(TCPHealthChecker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TCPKeepalive) DeepCopyInto(out *TCPKeepalive) { *out = *in diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index 91acdc04a66..58e6641c667 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -6,7 +6,9 @@ package translator import ( + "encoding/hex" "fmt" + "sort" "time" clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" @@ -15,6 +17,7 @@ import ( proxyprotocolv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/proxy_protocol/v3" rawbufferv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/raw_buffer/v3" httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" + xdstype "github.com/envoyproxy/go-control-plane/envoy/type/v3" "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" "google.golang.org/protobuf/types/known/anypb" @@ -38,6 +41,7 @@ type xdsClusterArgs struct { loadBalancer *ir.LoadBalancer proxyProtocol *ir.ProxyProtocol circuitBreaker *ir.CircuitBreaker + healthCheck *ir.HealthCheck } type EndpointType int @@ -131,6 +135,45 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster { cluster.LbPolicy = clusterv3.Cluster_MAGLEV } + if args.healthCheck != nil { + hc := &corev3.HealthCheck{ + Timeout: durationpb.New(args.healthCheck.Timeout.Duration), + Interval: durationpb.New(args.healthCheck.Interval.Duration), + } + if args.healthCheck.UnhealthyThreshold != nil { + hc.UnhealthyThreshold = wrapperspb.UInt32(*args.healthCheck.UnhealthyThreshold) + } + if args.healthCheck.HealthyThreshold != nil { + hc.HealthyThreshold = wrapperspb.UInt32(*args.healthCheck.HealthyThreshold) + } + if args.healthCheck.HTTP != nil { + httpChecker := &corev3.HealthCheck_HttpHealthCheck{ + Path: args.healthCheck.HTTP.Path, + } + if args.healthCheck.HTTP.Method != nil { + httpChecker.Method = corev3.RequestMethod(corev3.RequestMethod_value[*args.healthCheck.HTTP.Method]) + } + httpChecker.ExpectedStatuses = buildHTTPStatusRange(args.healthCheck.HTTP.ExpectedStatuses) + if receive := buildHealthCheckPayload(args.healthCheck.HTTP.ExpectedResponse); receive != nil { + httpChecker.Receive = append(httpChecker.Receive, receive) + } + hc.HealthChecker = &corev3.HealthCheck_HttpHealthCheck_{ + HttpHealthCheck: httpChecker, + } + } + if args.healthCheck.TCP != nil { + tcpChecker := &corev3.HealthCheck_TcpHealthCheck{ + Send: buildHealthCheckPayload(args.healthCheck.TCP.Send), + } + if receive := buildHealthCheckPayload(args.healthCheck.TCP.Receive); receive != nil { + tcpChecker.Receive = append(tcpChecker.Receive, receive) + } + hc.HealthChecker = &corev3.HealthCheck_TcpHealthCheck_{ + TcpHealthCheck: tcpChecker, + } + } + cluster.HealthChecks = []*corev3.HealthCheck{hc} + } if args.circuitBreaker != nil { cluster.CircuitBreakers = buildXdsClusterCircuitBreaker(args.circuitBreaker) } @@ -138,6 +181,56 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster { return cluster } +// buildHTTPStatusRange converts an array of http status to an array of the range of http status. +func buildHTTPStatusRange(irStatuses []ir.HTTPStatus) []*xdstype.Int64Range { + if len(irStatuses) == 0 { + return nil + } + ranges := []*xdstype.Int64Range{} + sort.Slice(irStatuses, func(i int, j int) bool { + return irStatuses[i] < irStatuses[j] + }) + var start, end int64 + for i := 0; i < len(irStatuses); i++ { + switch { + case start == 0: + start = int64(irStatuses[i]) + end = int64(irStatuses[i] + 1) + case int64(irStatuses[i]) == end: + end++ + default: + ranges = append(ranges, &xdstype.Int64Range{Start: start, End: end}) + start = int64(irStatuses[i]) + end = int64(irStatuses[i] + 1) + } + } + if start != 0 { + ranges = append(ranges, &xdstype.Int64Range{Start: start, End: end}) + } + return ranges +} + +func buildHealthCheckPayload(irLoad *ir.HealthCheckPayload) *corev3.HealthCheck_Payload { + if irLoad == nil { + return nil + } + + var hcp corev3.HealthCheck_Payload + if irLoad.Text != nil && *irLoad.Text != "" { + hcp.Payload = &corev3.HealthCheck_Payload_Text{ + Text: hex.EncodeToString([]byte(*irLoad.Text)), + } + } + if len(irLoad.Binary) > 0 { + binPayload := &corev3.HealthCheck_Payload_Binary{ + Binary: make([]byte, len(irLoad.Binary)), + } + copy(binPayload.Binary, irLoad.Binary) + hcp.Payload = binPayload + } + return &hcp +} + func buildXdsClusterCircuitBreaker(circuitBreaker *ir.CircuitBreaker) *clusterv3.CircuitBreakers { cbt := &clusterv3.CircuitBreakers_Thresholds{ Priority: corev3.RoutingPriority_DEFAULT, diff --git a/internal/xds/translator/testdata/in/xds-ir/health-check.yaml b/internal/xds/translator/testdata/in/xds-ir/health-check.yaml new file mode 100644 index 00000000000..6fe80abc608 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/health-check.yaml @@ -0,0 +1,83 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + routes: + - name: "first-route" + hostname: "*" + healthCheck: + timeout: "500ms" + interval: "3s" + unhealthyThreshold: 3 + healthyThreshold: 1 + http: + path: "/healthz" + expectedResponse: + text: "ok" + expectedStatuses: + - 200 + - 300 + destination: + name: "first-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + - name: "second-route" + hostname: "*" + healthCheck: + timeout: "1s" + interval: "5s" + unhealthyThreshold: 3 + healthyThreshold: 3 + http: + path: "/healthz" + expectedResponse: + binary: "cG9uZw==" + expectedStatuses: + - 200 + - 201 + destination: + name: "second-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + - name: "third-route" + hostname: "*" + healthCheck: + timeout: "1s" + interval: "5s" + unhealthyThreshold: 3 + healthyThreshold: 3 + tcp: + send: + text: "ping" + receive: + text: "pong" + destination: + name: "third-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + - name: "fourth-route" + hostname: "*" + healthCheck: + timeout: "1s" + interval: "5s" + unhealthyThreshold: 3 + healthyThreshold: 3 + tcp: + send: + binary: "cGluZw==" + receive: + binary: "cG9uZw==" + destination: + name: "fourth-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/health-check.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/health-check.clusters.yaml new file mode 100644 index 00000000000..628302cce16 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/health-check.clusters.yaml @@ -0,0 +1,102 @@ +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest + healthChecks: + - healthyThreshold: 1 + httpHealthCheck: + expectedStatuses: + - end: "201" + start: "200" + - end: "301" + start: "300" + path: /healthz + receive: + - text: 6f6b + interval: 3s + timeout: 0.500s + unhealthyThreshold: 3 + lbPolicy: LEAST_REQUEST + name: first-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: second-route-dest + healthChecks: + - healthyThreshold: 3 + httpHealthCheck: + expectedStatuses: + - end: "202" + start: "200" + path: /healthz + receive: + - binary: cG9uZw== + interval: 5s + timeout: 1s + unhealthyThreshold: 3 + lbPolicy: LEAST_REQUEST + name: second-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: third-route-dest + healthChecks: + - healthyThreshold: 3 + interval: 5s + tcpHealthCheck: + receive: + - text: 706f6e67 + send: + text: "70696e67" + timeout: 1s + unhealthyThreshold: 3 + lbPolicy: LEAST_REQUEST + name: third-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: fourth-route-dest + healthChecks: + - healthyThreshold: 3 + interval: 5s + tcpHealthCheck: + receive: + - binary: cG9uZw== + send: + binary: cGluZw== + timeout: 1s + unhealthyThreshold: 3 + lbPolicy: LEAST_REQUEST + name: fourth-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/health-check.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/health-check.endpoints.yaml new file mode 100644 index 00000000000..f185af17da7 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/health-check.endpoints.yaml @@ -0,0 +1,48 @@ +- clusterName: first-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: first-route-dest/backend/0 +- clusterName: second-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: second-route-dest/backend/0 +- clusterName: third-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: third-route-dest/backend/0 +- clusterName: fourth-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: fourth-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/health-check.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/health-check.listeners.yaml new file mode 100644 index 00000000000..73ee1b42ef6 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/health-check.listeners.yaml @@ -0,0 +1,33 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + statPrefix: http + upgradeConfigs: + - upgradeType: websocket + useRemoteAddress: true + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/health-check.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/health-check.routes.yaml new file mode 100644 index 00000000000..79c66f88204 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/health-check.routes.yaml @@ -0,0 +1,27 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: first-route + route: + cluster: first-route-dest + - match: + prefix: / + name: second-route + route: + cluster: second-route-dest + - match: + prefix: / + name: third-route + route: + cluster: third-route-dest + - match: + prefix: / + name: fourth-route + route: + cluster: fourth-route-dest diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index f73f3a0b7f3..80cdbf21ce0 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -505,6 +505,7 @@ func processXdsCluster(tCtx *types.ResourceVersionTable, httpRoute *ir.HTTPRoute loadBalancer: httpRoute.LoadBalancer, proxyProtocol: httpRoute.ProxyProtocol, circuitBreaker: httpRoute.CircuitBreaker, + healthCheck: httpRoute.HealthCheck, }); err != nil && !errors.Is(err, ErrXdsClusterExists) { return err } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index fd895945a65..c81ffd7a488 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -215,6 +215,9 @@ func TestTranslateXds(t *testing.T) { { name: "basic-auth", }, + { + name: "health-check", + }, { name: "local-ratelimit", }, diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index b4654de51db..e4346847774 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -87,6 +87,7 @@ _Appears in:_ | `loadBalancer` _[LoadBalancer](#loadbalancer)_ | LoadBalancer policy to apply when routing traffic from the gateway to the backend endpoints | | `proxyProtocol` _[ProxyProtocol](#proxyprotocol)_ | ProxyProtocol enables the Proxy Protocol when communicating with the backend. | | `tcpKeepalive` _[TCPKeepalive](#tcpkeepalive)_ | TcpKeepalive settings associated with the upstream client connection. Disabled by default. | +| `healthCheck` _[HealthCheck](#healthcheck)_ | HealthCheck allows gateway to perform active health checking on backends. | | `faultInjection` _[FaultInjection](#faultinjection)_ | FaultInjection defines the fault injection policy to be applied. This configuration can be used to inject delays and abort requests to mimic failure scenarios such as service failures and overloads | | `circuitBreaker` _[CircuitBreaker](#circuitbreaker)_ | Circuit Breaker settings for the upstream connections and requests. If not set, circuit breakers will be enabled with the default thresholds | @@ -924,6 +925,34 @@ _Appears in:_ +#### HTTPHealthChecker + + + +HTTPHealthChecker defines the settings of http health check. + +_Appears in:_ +- [HealthCheck](#healthcheck) + +| Field | Description | +| --- | --- | +| `path` _string_ | Path defines the HTTP path that will be requested during health checking. | +| `method` _string_ | Method defines the HTTP method used for health checking. Defaults to GET | +| `expectedStatuses` _[HTTPStatus](#httpstatus) array_ | ExpectedStatuses defines a list of HTTP response statuses considered healthy. Defaults to 200 only | +| `expectedResponse` _[HealthCheckPayload](#healthcheckpayload)_ | ExpectedResponse defines a list of HTTP expected responses to match. | + + +#### HTTPStatus + +_Underlying type:_ `integer` + +HTTPStatus defines the http status code. + +_Appears in:_ +- [HTTPHealthChecker](#httphealthchecker) + + + #### HeaderMatch @@ -935,6 +964,65 @@ _Appears in:_ +#### HealthCheck + + + +HealthCheck defines the health check configuration. EG supports various types of health checking including HTTP, TCP. + +_Appears in:_ +- [BackendTrafficPolicySpec](#backendtrafficpolicyspec) + +| Field | Description | +| --- | --- | +| `timeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#duration-v1-meta)_ | Timeout defines the time to wait for a health check response. | +| `interval` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#duration-v1-meta)_ | Interval defines the time between health checks. | +| `unhealthyThreshold` _integer_ | UnhealthyThreshold defines the number of unhealthy health checks required before a backend host is marked unhealthy. | +| `healthyThreshold` _integer_ | HealthyThreshold defines the number of healthy health checks required before a backend host is marked healthy. | +| `type` _[HealthCheckerType](#healthcheckertype)_ | Type defines the type of health checker. | +| `http` _[HTTPHealthChecker](#httphealthchecker)_ | HTTP defines the configuration of http health checker. It's required while the health checker type is HTTP. | +| `tcp` _[TCPHealthChecker](#tcphealthchecker)_ | TCP defines the configuration of tcp health checker. It's required while the health checker type is TCP. | + + +#### HealthCheckPayload + + + +HealthCheckPayload defines the encoding of the payload bytes in the payload. + +_Appears in:_ +- [HTTPHealthChecker](#httphealthchecker) +- [TCPHealthChecker](#tcphealthchecker) + +| Field | Description | +| --- | --- | +| `type` _[HealthCheckPayloadType](#healthcheckpayloadtype)_ | Type defines the type of the payload. | +| `text` _string_ | Text payload in plain text. | +| `binary` _integer array_ | Binary payload base64 encoded. | + + +#### HealthCheckPayloadType + +_Underlying type:_ `string` + +HealthCheckPayloadType is the type of the payload. + +_Appears in:_ +- [HealthCheckPayload](#healthcheckpayload) + + + +#### HealthCheckerType + +_Underlying type:_ `string` + +HealthCheckerType is the type of health checker. + +_Appears in:_ +- [HealthCheck](#healthcheck) + + + #### InfrastructureProviderType _Underlying type:_ `string` @@ -1879,6 +1967,21 @@ _Appears in:_ +#### TCPHealthChecker + + + +TCPHealthChecker defines the settings of tcp health check. + +_Appears in:_ +- [HealthCheck](#healthcheck) + +| Field | Description | +| --- | --- | +| `send` _[HealthCheckPayload](#healthcheckpayload)_ | Send defines the request payload. | +| `receive` _[HealthCheckPayload](#healthcheckpayload)_ | Receive defines the expected response payload. | + + #### TCPKeepalive diff --git a/test/cel-validation/backendtrafficpolicy_test.go b/test/cel-validation/backendtrafficpolicy_test.go index 02af065f6b2..ba7d077107e 100644 --- a/test/cel-validation/backendtrafficpolicy_test.go +++ b/test/cel-validation/backendtrafficpolicy_test.go @@ -15,12 +15,12 @@ import ( "testing" "time" - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" "k8s.io/utils/ptr" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" ) func TestBackendTrafficPolicyTarget(t *testing.T) { @@ -479,6 +479,282 @@ func TestBackendTrafficPolicyTarget(t *testing.T) { "spec.circuitBreaker.maxConnections: Invalid value: 4294967296: spec.circuitBreaker.maxConnections in body should be less than or equal to 4294967295", }, }, + { + desc: "invalid path of http health checker", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "", + }, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck.http.path: Invalid value: "": spec.healthCheck.http.path in body should be at least 1 chars long`, + }, + }, + { + desc: "invalid unhealthy threshold", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + UnhealthyThreshold: ptr.To[uint32](0), + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + }, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck.unhealthyThreshold: Invalid value: 0: spec.healthCheck.unhealthyThreshold in body should be greater than or equal to 1`, + }, + }, + { + desc: "invalid healthy threshold", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + HealthyThreshold: ptr.To[uint32](0), + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + }, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck.healthyThreshold: Invalid value: 0: spec.healthCheck.healthyThreshold in body should be greater than or equal to 1`, + }, + }, + { + desc: "invalid health checker type", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + TCP: &egv1a1.TCPHealthChecker{}, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck: Invalid value: "object": If Health Checker type is HTTP, http field needs to be set., spec.healthCheck: Invalid value: "object": If Health Checker type is TCP, tcp field needs to be set`, + }, + }, + { + desc: "invalid http expected statuses", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + ExpectedStatuses: []egv1a1.HTTPStatus{99, 200}, + }, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck.http.expectedStatuses[0]: Invalid value: 99: spec.healthCheck.http.expectedStatuses[0] in body should be greater than or equal to 100`, + }, + }, + { + desc: "valid http expected statuses", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + ExpectedStatuses: []egv1a1.HTTPStatus{100, 200, 201}, + }, + }, + } + }, + wantErrors: []string{}, + }, + { + desc: "invalid http expected statuses - out of range", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + ExpectedStatuses: []egv1a1.HTTPStatus{200, 300, 601}, + }, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck.http.expectedStatuses[2]: Invalid value: 601: spec.healthCheck.http.expectedStatuses[2] in body should be less than 600`, + }, + }, + { + desc: "http expected responses - invalid text payload", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + ExpectedResponse: &egv1a1.HealthCheckPayload{ + Type: egv1a1.HealthCheckPayloadTypeText, + Binary: []byte{'f', 'o', 'o'}, + }, + }, + }, + } + }, + wantErrors: []string{ + `[spec.healthCheck.http.expectedResponse: Invalid value: "object": If payload type is Text, text field needs to be set., spec.healthCheck.http.expectedResponse: Invalid value: "object": If payload type is Binary, binary field needs to be set.]`, + }, + }, + { + desc: "http expected responses - invalid binary payload", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeHTTP, + HTTP: &egv1a1.HTTPHealthChecker{ + Path: "/healthz", + ExpectedResponse: &egv1a1.HealthCheckPayload{ + Type: egv1a1.HealthCheckPayloadTypeBinary, + Text: ptr.To("foo"), + }, + }, + }, + } + }, + wantErrors: []string{ + `[spec.healthCheck.http.expectedResponse: Invalid value: "object": If payload type is Text, text field needs to be set., spec.healthCheck.http.expectedResponse: Invalid value: "object": If payload type is Binary, binary field needs to be set.]`, + }, + }, + { + desc: "invalid tcp send", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeTCP, + TCP: &egv1a1.TCPHealthChecker{ + Send: &egv1a1.HealthCheckPayload{ + Type: egv1a1.HealthCheckPayloadTypeText, + Binary: []byte{'f', 'o', 'o'}, + }, + Receive: &egv1a1.HealthCheckPayload{ + Type: egv1a1.HealthCheckPayloadTypeText, + Text: ptr.To("foo"), + }, + }, + }, + } + }, + wantErrors: []string{ + `spec.healthCheck.tcp.send: Invalid value: "object": If payload type is Text, text field needs to be set., spec.healthCheck.tcp.send: Invalid value: "object": If payload type is Binary, binary field needs to be set.`, + }, + }, + { + desc: "invalid tcp receive", + mutate: func(btp *egv1a1.BackendTrafficPolicy) { + btp.Spec = egv1a1.BackendTrafficPolicySpec{ + TargetRef: gwapiv1a2.PolicyTargetReferenceWithSectionName{ + PolicyTargetReference: gwapiv1a2.PolicyTargetReference{ + Group: gwapiv1a2.Group("gateway.networking.k8s.io"), + Kind: gwapiv1a2.Kind("Gateway"), + Name: gwapiv1a2.ObjectName("eg"), + }, + }, + HealthCheck: &egv1a1.HealthCheck{ + Type: egv1a1.HealthCheckerTypeTCP, + TCP: &egv1a1.TCPHealthChecker{ + Send: &egv1a1.HealthCheckPayload{ + Type: egv1a1.HealthCheckPayloadTypeText, + Text: ptr.To("foo"), + }, + Receive: &egv1a1.HealthCheckPayload{ + Type: egv1a1.HealthCheckPayloadTypeText, + Binary: []byte{'f', 'o', 'o'}, + }, + }, + }, + } + }, + wantErrors: []string{ + `[spec.healthCheck.tcp.receive: Invalid value: "object": If payload type is Text, text field needs to be set., spec.healthCheck.tcp.receive: Invalid value: "object": If payload type is Binary, binary field needs to be set.]`, + }, + }, } for _, tc := range cases {