diff --git a/.github/workflows/conformance-k8s-kind.yaml b/.github/workflows/conformance-k8s-kind.yaml index 496d8dda474e8..b8554989140a2 100644 --- a/.github/workflows/conformance-k8s-kind.yaml +++ b/.github/workflows/conformance-k8s-kind.yaml @@ -196,7 +196,6 @@ jobs: # LoadBalancer|GCE|ExternalIP : require a cloud provider, some of them are GCE specifics # Netpol|NetworkPolicy : network policies, demand significant resources and use to be slow, better to run in a different job # Aggregator : Flaky, https://github.com/cilium/cilium/issues/24622. - # same.port.number.but.different.protocols|HostPort|should.serve.endpoints.on.same.port.and.different.protocols : #9207 # rejected : Kubernetes expect Services without endpoints associated to REJECT the connection to notify the client, Cilium silently drops the packet # externalTrafficPolicy : needs investigation @@ -205,7 +204,7 @@ jobs: export E2E_REPORT_DIR=${PWD}/_artifacts /usr/local/bin/ginkgo --nodes=25 \ --focus="\[Conformance\]|\[sig-network\]" \ - --skip="Feature|Federation|PerformanceDNS|DualStack|Disruptive|Serial|KubeProxy|kube-proxy|ExternalIP|LoadBalancer|GCE|Netpol|NetworkPolicy|Aggregator|rejected|externalTrafficPolicy|HostPort|same.port.number.but.different.protocols|should.serve.endpoints.on.same.port.and.different.protocols" \ + --skip="Feature|Federation|PerformanceDNS|DualStack|Disruptive|Serial|KubeProxy|kube-proxy|ExternalIP|LoadBalancer|GCE|Netpol|NetworkPolicy|Aggregator|rejected|externalTrafficPolicy" \ /usr/local/bin/e2e.test \ -- \ --kubeconfig=${PWD}/_artifacts/kubeconfig.conf \ diff --git a/Documentation/cmdref/cilium-agent.md b/Documentation/cmdref/cilium-agent.md index a7de508b78e34..c30827ec879d9 100644 --- a/Documentation/cmdref/cilium-agent.md +++ b/Documentation/cmdref/cilium-agent.md @@ -47,6 +47,7 @@ cilium-agent [flags] --bpf-lb-maglev-table-size uint Maglev per service backend table size (parameter M) (default 16381) --bpf-lb-map-max int Maximum number of entries in Cilium BPF lbmap (default 65536) --bpf-lb-mode string BPF load balancing mode ("snat", "dsr", "hybrid") (default "snat") + --bpf-lb-proto-diff Enable support for service protocol differentiation (TCP, UDP, SCTP) (default true) --bpf-lb-rss-ipv4-src-cidr string BPF load balancing RSS outer source IPv4 CIDR prefix for IPIP --bpf-lb-rss-ipv6-src-cidr string BPF load balancing RSS outer source IPv6 CIDR prefix for IPIP --bpf-lb-sock Enable socket-based LB for E/W traffic diff --git a/Documentation/cmdref/cilium_service_update.md b/Documentation/cmdref/cilium_service_update.md index 4a114be137102..fddc5f4fec33e 100644 --- a/Documentation/cmdref/cilium_service_update.md +++ b/Documentation/cmdref/cilium_service_update.md @@ -24,6 +24,7 @@ cilium service update [flags] --k8s-load-balancer Set service as a k8s LoadBalancer --k8s-node-port Set service as a k8s NodePort --local-redirect Set service as Local Redirect + --protocol string Protocol for service (e.g. TCP, UDP) (default "tcp") --states strings Backend state(s) as {active(default),terminating,quarantined,maintenance} ``` diff --git a/Documentation/network/kubernetes/kubeproxy-free.rst b/Documentation/network/kubernetes/kubeproxy-free.rst index 210c8ae78830a..30d305b09e5dd 100644 --- a/Documentation/network/kubernetes/kubeproxy-free.rst +++ b/Documentation/network/kubernetes/kubeproxy-free.rst @@ -242,14 +242,14 @@ In this example, services with port ``31940`` were created (one for each of devi $ kubectl -n kube-system exec ds/cilium -- cilium service list ID Frontend Service Type Backend [...] - 4 10.104.239.135:80 ClusterIP 1 => 10.217.0.107:80 - 2 => 10.217.0.149:80 - 5 0.0.0.0:31940 NodePort 1 => 10.217.0.107:80 - 2 => 10.217.0.149:80 - 6 192.168.178.29:31940 NodePort 1 => 10.217.0.107:80 - 2 => 10.217.0.149:80 - 7 172.16.0.29:31940 NodePort 1 => 10.217.0.107:80 - 2 => 10.217.0.149:80 + 4 10.104.239.135:80/TCP ClusterIP 1 => 10.217.0.107:80/TCP + 2 => 10.217.0.149:80/TCP + 5 0.0.0.0:31940/TCP NodePort 1 => 10.217.0.107:80/TCP + 2 => 10.217.0.149:80/TCP + 6 192.168.178.29:31940/TCP NodePort 1 => 10.217.0.107:80/TCP + 2 => 10.217.0.149:80/TCP + 7 172.16.0.29:31940/TCP NodePort 1 => 10.217.0.107:80/TCP + 2 => 10.217.0.149:80/TCP Create a variable with the node port for testing: diff --git a/api/v1/models/backend_address.go b/api/v1/models/backend_address.go index 3f17b81f49175..f02578a08fe64 100644 --- a/api/v1/models/backend_address.go +++ b/api/v1/models/backend_address.go @@ -37,6 +37,9 @@ type BackendAddress struct { // on related annotation of global service. Applicable for active state only. Preferred bool `json:"preferred,omitempty"` + // Layer 4 protocol (TCP, UDP, etc) + Protocol string `json:"protocol,omitempty"` + // State of the backend for load-balancing service traffic // Enum: [active terminating quarantined maintenance] State string `json:"state,omitempty"` diff --git a/api/v1/openapi.yaml b/api/v1/openapi.yaml index ead85313b2089..1c1a47e91d948 100644 --- a/api/v1/openapi.yaml +++ b/api/v1/openapi.yaml @@ -2860,6 +2860,9 @@ definitions: ip: description: Layer 3 address type: string + protocol: + description: Layer 4 protocol (TCP, UDP, etc) + type: string port: description: Layer 4 port number type: integer diff --git a/api/v1/server/embedded_spec.go b/api/v1/server/embedded_spec.go index 2c74438ef57c7..d419924b22ed8 100644 --- a/api/v1/server/embedded_spec.go +++ b/api/v1/server/embedded_spec.go @@ -1874,6 +1874,10 @@ func init() { "description": "Indicator if this backend is preferred in the context of clustermesh service affinity. The value is set based\non related annotation of global service. Applicable for active state only.", "type": "boolean" }, + "protocol": { + "description": "Layer 4 protocol (TCP, UDP, etc)", + "type": "string" + }, "state": { "description": "State of the backend for load-balancing service traffic", "type": "string", @@ -7250,6 +7254,10 @@ func init() { "description": "Indicator if this backend is preferred in the context of clustermesh service affinity. The value is set based\non related annotation of global service. Applicable for active state only.", "type": "boolean" }, + "protocol": { + "description": "Layer 4 protocol (TCP, UDP, etc)", + "type": "string" + }, "state": { "description": "State of the backend for load-balancing service traffic", "type": "string", diff --git a/bpf/bpf_sock.c b/bpf/bpf_sock.c index 0cf21bd9e8078..e84e28307bd3b 100644 --- a/bpf/bpf_sock.c +++ b/bpf/bpf_sock.c @@ -39,6 +39,9 @@ static __always_inline __maybe_unused bool is_v6_loopback(const union v6addr *da return ipv6_addr_equals(&loopback, daddr); } +/* Hack due to missing narrow ctx access. */ +#define ctx_protocol(__ctx) ((__u8)(volatile __u32)(__ctx)->protocol) + /* Hack due to missing narrow ctx access. */ static __always_inline __maybe_unused __be16 ctx_dst_port(const struct bpf_sock_addr *ctx) @@ -104,12 +107,12 @@ bool sock_is_health_check(struct bpf_sock_addr *ctx __maybe_unused) static __always_inline __maybe_unused __u64 sock_select_slot(struct bpf_sock_addr *ctx) { - return ctx->protocol == IPPROTO_TCP ? + return ctx_protocol(ctx) == IPPROTO_TCP ? get_prandom_u32() : sock_local_cookie(ctx); } static __always_inline __maybe_unused -bool sock_proto_enabled(__u32 proto) +bool sock_proto_enabled(__u8 proto) { switch (proto) { case IPPROTO_TCP: @@ -294,10 +297,14 @@ static __always_inline int __sock4_xlate_fwd(struct bpf_sock_addr *ctx, struct lb4_backend *backend; struct lb4_service *svc; __u16 dst_port = ctx_dst_port(ctx); + __u8 protocol = ctx_protocol(ctx); __u32 dst_ip = ctx->user_ip4; struct lb4_key key = { .address = dst_ip, .dport = dst_port, +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + .proto = protocol, +#endif }, orig_key = key; struct lb4_service *backend_slot; bool backend_from_affinity = false; @@ -309,7 +316,7 @@ static __always_inline int __sock4_xlate_fwd(struct bpf_sock_addr *ctx, if (is_defined(ENABLE_SOCKET_LB_HOST_ONLY) && !in_hostns) return -ENXIO; - if (!udp_only && !sock_proto_enabled(ctx->protocol)) + if (!udp_only && !sock_proto_enabled(protocol)) return -ENOTSUP; /* In case a direct match fails, we try to look-up surrogate @@ -317,8 +324,13 @@ static __always_inline int __sock4_xlate_fwd(struct bpf_sock_addr *ctx, * HostPort services. */ svc = lb4_lookup_service(&key, true, false); - if (!svc) + if (!svc) { + /* Restore the original key's protocol as lb4_lookup_service + * has overwritten it. + */ + lb4_key_set_protocol(&key, protocol); svc = sock4_wildcard_lookup_full(&key, in_hostns); + } if (!svc) return -ENXIO; if (svc->count == 0 && !lb4_svc_is_l7loadbalancer(svc)) @@ -477,23 +489,32 @@ int cil_sock4_connect(struct bpf_sock_addr *ctx) static __always_inline int __sock4_post_bind(struct bpf_sock *ctx, struct bpf_sock *ctx_full) { + __u8 protocol = ctx_protocol(ctx); struct lb4_service *svc; struct lb4_key key = { .address = ctx->src_ip4, .dport = ctx_src_port(ctx), +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + .proto = protocol, +#endif }; - if (!sock_proto_enabled(ctx->protocol) || + if (!sock_proto_enabled(protocol) || !ctx_in_hostns(ctx_full, NULL)) return 0; svc = lb4_lookup_service(&key, true, false); - if (!svc) + if (!svc) { /* Perform a wildcard lookup for the case where the caller * tries to bind to loopback or an address with host identity * (without remote hosts). + * + * Restore the original key's protocol as lb4_lookup_service + * has overwritten it. */ + lb4_key_set_protocol(&key, protocol); svc = sock4_wildcard_lookup(&key, false, false, true); + } /* If the sockaddr of this socket overlaps with a NodePort, * LoadBalancer or ExternalIP service. We must reject this @@ -540,7 +561,7 @@ static __always_inline int __sock4_pre_bind(struct bpf_sock_addr *ctx, .peer = { .address = ctx->user_ip4, .port = ctx_dst_port(ctx), - .proto = (__u8)ctx->protocol, + .proto = ctx_protocol(ctx), }, }; int ret; @@ -556,7 +577,7 @@ int cil_sock4_pre_bind(struct bpf_sock_addr *ctx) { int ret = SYS_PROCEED; - if (!sock_proto_enabled(ctx->protocol) || + if (!sock_proto_enabled(ctx_protocol(ctx)) || !ctx_in_hostns(ctx, NULL)) return ret; if (sock_is_health_check(ctx) && @@ -573,6 +594,7 @@ static __always_inline int __sock4_xlate_rev(struct bpf_sock_addr *ctx, { struct ipv4_revnat_entry *val; __u16 dst_port = ctx_dst_port(ctx); + __u8 protocol = ctx_protocol(ctx); __u32 dst_ip = ctx->user_ip4; struct ipv4_revnat_tuple key = { .cookie = sock_local_cookie(ctx_full), @@ -588,12 +610,20 @@ static __always_inline int __sock4_xlate_rev(struct bpf_sock_addr *ctx, struct lb4_key svc_key = { .address = val->address, .dport = val->port, +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + .proto = protocol, +#endif }; svc = lb4_lookup_service(&svc_key, true, false); - if (!svc) + if (!svc) { + /* Restore the original key's protocol as lb4_lookup_service + * has overwritten it. + */ + lb4_key_set_protocol(&svc_key, protocol); svc = sock4_wildcard_lookup_full(&svc_key, ctx_in_hostns(ctx_full, NULL)); + } if (!svc || svc->rev_nat_index != val->rev_nat_index || (svc->count == 0 && !lb4_svc_is_l7loadbalancer(svc))) { map_delete_elem(&LB4_REVERSE_NAT_SK_MAP, &key); @@ -810,7 +840,7 @@ int sock6_xlate_v4_in_v6(struct bpf_sock_addr *ctx __maybe_unused, return -ENXIO; memset(&fake_ctx, 0, sizeof(fake_ctx)); - fake_ctx.protocol = ctx->protocol; + fake_ctx.protocol = ctx_protocol(ctx); fake_ctx.user_ip4 = addr6.p4; fake_ctx.user_port = ctx_dst_port(ctx); @@ -840,7 +870,7 @@ sock6_post_bind_v4_in_v6(struct bpf_sock *ctx __maybe_unused) return 0; memset(&fake_ctx, 0, sizeof(fake_ctx)); - fake_ctx.protocol = ctx->protocol; + fake_ctx.protocol = ctx_protocol(ctx); fake_ctx.src_ip4 = addr6.p4; fake_ctx.src_port = ctx->src_port; @@ -851,12 +881,16 @@ sock6_post_bind_v4_in_v6(struct bpf_sock *ctx __maybe_unused) static __always_inline int __sock6_post_bind(struct bpf_sock *ctx) { + __u8 protocol = ctx_protocol(ctx); struct lb6_service *svc; struct lb6_key key = { .dport = ctx_src_port(ctx), +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + .proto = protocol, +#endif }; - if (!sock_proto_enabled(ctx->protocol) || + if (!sock_proto_enabled(protocol) || !ctx_in_hostns(ctx, NULL)) return 0; @@ -864,6 +898,10 @@ static __always_inline int __sock6_post_bind(struct bpf_sock *ctx) svc = lb6_lookup_service(&key, true, false); if (!svc) { + /* Restore the original key's protocol as lb6_lookup_service + * has overwritten it. + */ + lb6_key_set_protocol(&key, protocol); svc = sock6_wildcard_lookup(&key, false, false, true); if (!svc) return sock6_post_bind_v4_in_v6(ctx); @@ -904,7 +942,7 @@ sock6_pre_bind_v4_in_v6(struct bpf_sock_addr *ctx __maybe_unused) ctx_get_v6_address(ctx, &addr6); memset(&fake_ctx, 0, sizeof(fake_ctx)); - fake_ctx.protocol = ctx->protocol; + fake_ctx.protocol = ctx_protocol(ctx); fake_ctx.user_ip4 = addr6.p4; fake_ctx.user_port = ctx_dst_port(ctx); @@ -935,7 +973,7 @@ static __always_inline int __sock6_pre_bind(struct bpf_sock_addr *ctx) struct lb6_health val = { .peer = { .port = ctx_dst_port(ctx), - .proto = (__u8)ctx->protocol, + .proto = ctx_protocol(ctx), }, }; int ret = 0; @@ -957,7 +995,7 @@ int cil_sock6_pre_bind(struct bpf_sock_addr *ctx) { int ret = SYS_PROCEED; - if (!sock_proto_enabled(ctx->protocol) || + if (!sock_proto_enabled(ctx_protocol(ctx)) || !ctx_in_hostns(ctx, NULL)) return ret; if (sock_is_health_check(ctx) && @@ -978,8 +1016,12 @@ static __always_inline int __sock6_xlate_fwd(struct bpf_sock_addr *ctx, struct lb6_backend *backend; struct lb6_service *svc; __u16 dst_port = ctx_dst_port(ctx); + __u8 protocol = ctx_protocol(ctx); struct lb6_key key = { .dport = dst_port, +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + .proto = protocol, +#endif }, orig_key; struct lb6_service *backend_slot; bool backend_from_affinity = false; @@ -991,15 +1033,20 @@ static __always_inline int __sock6_xlate_fwd(struct bpf_sock_addr *ctx, if (is_defined(ENABLE_SOCKET_LB_HOST_ONLY) && !in_hostns) return -ENXIO; - if (!udp_only && !sock_proto_enabled(ctx->protocol)) + if (!udp_only && !sock_proto_enabled(protocol)) return -ENOTSUP; ctx_get_v6_address(ctx, &key.address); memcpy(&orig_key, &key, sizeof(key)); svc = lb6_lookup_service(&key, true, false); - if (!svc) + if (!svc) { + /* Restore the original key's protocol as lb6_lookup_service + * has overwritten it. + */ + lb6_key_set_protocol(&key, protocol); svc = sock6_wildcard_lookup_full(&key, in_hostns); + } if (!svc) return sock6_xlate_v4_in_v6(ctx, udp_only); if (svc->count == 0 && !lb6_svc_is_l7loadbalancer(svc)) @@ -1144,7 +1191,7 @@ sock6_xlate_rev_v4_in_v6(struct bpf_sock_addr *ctx __maybe_unused) return -ENXIO; memset(&fake_ctx, 0, sizeof(fake_ctx)); - fake_ctx.protocol = ctx->protocol; + fake_ctx.protocol = ctx_protocol(ctx); fake_ctx.user_ip4 = addr6.p4; fake_ctx.user_port = ctx_dst_port(ctx); @@ -1167,6 +1214,7 @@ static __always_inline int __sock6_xlate_rev(struct bpf_sock_addr *ctx) struct ipv6_revnat_tuple key = {}; struct ipv6_revnat_entry *val; __u16 dst_port = ctx_dst_port(ctx); + __u8 protocol = ctx_protocol(ctx); key.cookie = sock_local_cookie(ctx); key.port = dst_port; @@ -1181,12 +1229,20 @@ static __always_inline int __sock6_xlate_rev(struct bpf_sock_addr *ctx) struct lb6_key svc_key = { .address = val->address, .dport = val->port, +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + .proto = protocol, +#endif }; svc = lb6_lookup_service(&svc_key, true, false); - if (!svc) + if (!svc) { + /* Restore the original key's protocol as lb6_lookup_service + * has overwritten it. + */ + lb6_key_set_protocol(&svc_key, protocol); svc = sock6_wildcard_lookup_full(&svc_key, ctx_in_hostns(ctx, NULL)); + } if (!svc || svc->rev_nat_index != val->rev_nat_index || (svc->count == 0 && !lb6_svc_is_l7loadbalancer(svc))) { map_delete_elem(&LB6_REVERSE_NAT_SK_MAP, &key); diff --git a/bpf/lib/common.h b/bpf/lib/common.h index e0f1482f0d141..b6406ecd78733 100644 --- a/bpf/lib/common.h +++ b/bpf/lib/common.h @@ -945,7 +945,7 @@ struct lb6_key { union v6addr address; /* Service virtual IPv6 address */ __be16 dport; /* L4 port filter, if unset, all ports apply */ __u16 backend_slot; /* Backend iterator, 0 indicates the svc frontend */ - __u8 proto; /* L4 protocol, currently not used (set to 0) */ + __u8 proto; /* L4 protocol, 0 indicates any protocol */ __u8 scope; /* LB_LOOKUP_SCOPE_* for externalTrafficPolicy=Local */ __u8 pad[2]; }; @@ -1003,7 +1003,7 @@ struct lb4_key { __be32 address; /* Service virtual IPv4 address */ __be16 dport; /* L4 port filter, if unset, all ports apply */ __u16 backend_slot; /* Backend iterator, 0 indicates the svc frontend */ - __u8 proto; /* L4 protocol, currently not used (set to 0) */ + __u8 proto; /* L4 protocol, 0 indicates any protocol */ __u8 scope; /* LB_LOOKUP_SCOPE_* for externalTrafficPolicy=Local */ __u8 pad[2]; }; diff --git a/bpf/lib/lb.h b/bpf/lib/lb.h index 6eb5619ad7b63..5256aca903268 100644 --- a/bpf/lib/lb.h +++ b/bpf/lib/lb.h @@ -487,11 +487,19 @@ static __always_inline int lb6_rev_nat(struct __ctx_buff *ctx, int l4_off, return __lb6_rev_nat(ctx, l4_off, tuple, flags, nat); } +static __always_inline void +lb6_key_set_protocol(struct lb6_key *key __maybe_unused, + __u8 protocol __maybe_unused) +{ +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + key->proto = protocol; +#endif +} + static __always_inline void lb6_fill_key(struct lb6_key *key, struct ipv6_ct_tuple *tuple) { - /* FIXME: set after adding support for different L4 protocols in LB */ - key->proto = 0; + lb6_key_set_protocol(key, tuple->nexthdr); ipv6_addr_copy(&key->address, &tuple->daddr); key->dport = tuple->sport; } @@ -586,6 +594,15 @@ struct lb6_service *lb6_lookup_service(struct lb6_key *key, key->scope = LB_LOOKUP_SCOPE_EXT; key->backend_slot = 0; svc = map_lookup_elem(&LB6_SERVICES_MAP_V2, key); + +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + /* If there are no elements for a specific protocol, check for ANY entries. */ + if (!svc && key->proto != 0) { + key->proto = 0; + svc = map_lookup_elem(&LB6_SERVICES_MAP_V2, key); + } +#endif + if (svc) { if (!scope_switch || !lb6_svc_is_two_scopes(svc)) /* Packets for L7 LB are redirected even when there are no backends. */ @@ -1014,6 +1031,12 @@ struct lb6_service *lb6_lookup_service(struct lb6_key *key __maybe_unused, return NULL; } +static __always_inline void +lb6_key_set_protocol(struct lb6_key *key __maybe_unused, + __u8 protocol __maybe_unused) +{ +} + static __always_inline struct lb6_service *__lb6_lookup_backend_slot(struct lb6_key *key __maybe_unused) { @@ -1137,11 +1160,19 @@ static __always_inline int lb4_rev_nat(struct __ctx_buff *ctx, int l3_off, int l loopback, has_l4_header); } +static __always_inline void +lb4_key_set_protocol(struct lb4_key *key __maybe_unused, + __u8 protocol __maybe_unused) +{ +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + key->proto = protocol; +#endif +} + static __always_inline void lb4_fill_key(struct lb4_key *key, const struct ipv4_ct_tuple *tuple) { - /* FIXME: set after adding support for different L4 protocols in LB */ - key->proto = 0; + lb4_key_set_protocol(key, tuple->nexthdr); key->address = tuple->daddr; /* CT tuple has ports in reverse order: */ key->dport = tuple->sport; @@ -1240,6 +1271,15 @@ struct lb4_service *lb4_lookup_service(struct lb4_key *key, key->scope = LB_LOOKUP_SCOPE_EXT; key->backend_slot = 0; svc = map_lookup_elem(&LB4_SERVICES_MAP_V2, key); + +#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION) + /* If there are no elements for a specific protocol, check for ANY entries. */ + if (!svc && key->proto != 0) { + key->proto = 0; + svc = map_lookup_elem(&LB4_SERVICES_MAP_V2, key); + } +#endif + if (svc) { if (!scope_switch || !lb4_svc_is_two_scopes(svc)) /* Packets for L7 LB are redirected even when there are no backends. */ diff --git a/cilium/cmd/bpf_lb_list.go b/cilium/cmd/bpf_lb_list.go index e5f2522635904..b3a4b1f43284e 100644 --- a/cilium/cmd/bpf_lb_list.go +++ b/cilium/cmd/bpf_lb_list.go @@ -14,6 +14,7 @@ import ( "github.com/cilium/cilium/pkg/common" "github.com/cilium/cilium/pkg/loadbalancer" "github.com/cilium/cilium/pkg/maps/lbmap" + "github.com/cilium/cilium/pkg/u8proto" ) const ( @@ -105,12 +106,12 @@ func dumpSVC(serviceList map[string][]string) { } else if backend, found := backendMap[backendID]; !found { entry = fmt.Sprintf("backend %d not found", backendID) } else { - fmtStr := "%s:%d (%d) (%d)" + fmtStr := "%s:%d/%s (%d) (%d)" if svcKey.IsIPv6() { - fmtStr = "[%s]:%d (%d) (%d)" + fmtStr = "[%s]:%d/%s (%d) (%d)" } entry = fmt.Sprintf(fmtStr, backend.GetAddress(), - backend.GetPort(), revNATID, backendSlot) + backend.GetPort(), u8proto.U8proto(backend.GetProtocol()).String(), revNATID, backendSlot) } serviceList[svc] = append(serviceList[svc], entry) diff --git a/cilium/cmd/service_update.go b/cilium/cmd/service_update.go index bc0a2d76b5ca1..758be4c6a5582 100644 --- a/cilium/cmd/service_update.go +++ b/cilium/cmd/service_update.go @@ -29,6 +29,7 @@ var ( localRedirect bool idU uint64 frontend string + protocol string backends []string backendStates []string backendWeights []uint @@ -59,13 +60,38 @@ func init() { serviceUpdateCmd.Flags().StringVarP(&k8sIntTrafficPolicy, "k8s-int-traffic-policy", "", "Cluster", "Set service with k8s internalTrafficPolicy as {Local,Cluster}") serviceUpdateCmd.Flags().BoolVarP(&k8sClusterInternal, "k8s-cluster-internal", "", false, "Set service as cluster-internal for externalTrafficPolicy=Local xor internalTrafficPolicy=Local") serviceUpdateCmd.Flags().StringVarP(&frontend, "frontend", "", "", "Frontend address") + serviceUpdateCmd.Flags().StringVarP(&protocol, "protocol", "", "tcp", "Protocol for service (e.g. TCP, UDP)") serviceUpdateCmd.Flags().StringSliceVarP(&backends, "backends", "", []string{}, "Backend address or addresses ()") serviceUpdateCmd.Flags().StringSliceVarP(&backendStates, "states", "", []string{}, "Backend state(s) as {active(default),terminating,quarantined,maintenance}") serviceUpdateCmd.Flags().UintSliceVarP(&backendWeights, "backend-weights", "", []uint{}, "Backend weights (100 default, 0 means maintenance state, only for maglev mode)") } -func parseFrontendAddress(address string) *models.FrontendAddress { - frontend, err := net.ResolveTCPAddr("tcp", address) +func parseAddress(l4Protocol, address string) (ip net.IP, port int, proto string, err error) { + switch proto = strings.ToLower(l4Protocol); proto { + case "tcp": + var tcpAddr *net.TCPAddr + tcpAddr, err = net.ResolveTCPAddr(proto, address) + if err != nil { + return + } + ip = tcpAddr.IP + port = tcpAddr.Port + case "udp": + var udpAddr *net.UDPAddr + udpAddr, err = net.ResolveUDPAddr(proto, address) + if err != nil { + return + } + ip = udpAddr.IP + port = udpAddr.Port + default: + err = fmt.Errorf("unrecognized protocol %q", l4Protocol) + } + return +} + +func parseFrontendAddress(l4Protocol, address string) *models.FrontendAddress { + ip, port, proto, err := parseAddress(l4Protocol, address) if err != nil { Fatalf("Unable to parse frontend address: %s\n", err) } @@ -75,11 +101,10 @@ func parseFrontendAddress(address string) *models.FrontendAddress { scope = models.FrontendAddressScopeInternal } - // FIXME support more than TCP return &models.FrontendAddress{ - IP: frontend.IP.String(), - Port: uint16(frontend.Port), - Protocol: models.FrontendAddressProtocolTCP, + IP: ip.String(), + Port: uint16(port), + Protocol: proto, Scope: scope, } } @@ -95,7 +120,7 @@ func updateService(cmd *cobra.Command, args []string) { warnIdTypeDeprecation() id := int64(idU) - fa := parseFrontendAddress(frontend) + fa := parseFrontendAddress(protocol, frontend) skipFrontendCheck := false var spec *models.ServiceSpec @@ -203,17 +228,17 @@ func updateService(cmd *cobra.Command, args []string) { default: Fatalf("Invalid number of backend states (%v) for backends (%v)", backendStates, backends) } + for i, backend := range backends { - beAddr, err := net.ResolveTCPAddr("tcp", backend) + ip, port, proto, err := parseAddress(protocol, backend) if err != nil { - Fatalf("Cannot parse backend address \"%s\": %s", backend, err) + Fatalf("Cannot parse backend address %q: %s", backend, err) } - // Backend ID will be set by the daemon - be := loadbalancer.NewBackend(0, loadbalancer.TCP, cmtypes.MustAddrClusterFromIP(beAddr.IP), uint16(beAddr.Port)) + be := loadbalancer.NewBackend(0, loadbalancer.L4Type(strings.ToUpper(proto)), cmtypes.MustAddrClusterFromIP(ip), uint16(port)) - if !skipFrontendCheck && fa.Port == 0 && beAddr.Port != 0 { - Fatalf("L4 backend found (%v) with L3 frontend", beAddr) + if !skipFrontendCheck && fa.Port == 0 && port != 0 { + Fatalf("L4 backend found (%s:%d) with L3 frontend", ip, port) } ba := be.GetBackendModel() diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go index f16493c3e5c1e..ec336c0e37144 100644 --- a/daemon/cmd/daemon_main.go +++ b/daemon/cmd/daemon_main.go @@ -609,6 +609,9 @@ func initializeFlags() { option.NodePortAccelerationNative, option.NodePortAccelerationDisabled)) option.BindEnv(Vp, option.LoadBalancerAcceleration) + flags.Bool(option.LoadBalancerProtocolDifferentiation, true, "Enable support for service protocol differentiation (TCP, UDP, SCTP)") + option.BindEnv(Vp, option.LoadBalancerProtocolDifferentiation) + flags.Uint(option.MaglevTableSize, maglev.DefaultTableSize, "Maglev per service backend table size (parameter M)") option.BindEnv(Vp, option.MaglevTableSize) diff --git a/install/kubernetes/cilium/templates/cilium-configmap.yaml b/install/kubernetes/cilium/templates/cilium-configmap.yaml index ca3d945c4aa4c..c19c5b051590c 100644 --- a/install/kubernetes/cilium/templates/cilium-configmap.yaml +++ b/install/kubernetes/cilium/templates/cilium-configmap.yaml @@ -724,6 +724,9 @@ data: {{- if hasKey .Values.loadBalancer "serviceTopology" }} enable-service-topology: {{ .Values.loadBalancer.serviceTopology | quote }} {{- end }} +{{- if hasKey .Values.loadBalancer "protocolDifferentiation" }} + bpf-lb-proto-diff: {{ .Values.loadBalancer.protocolDifferentiation.enabled | quote }} +{{- end }} {{- end }} {{- if hasKey .Values.maglev "tableSize" }} diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go index 02c3af3b15a7f..a0c70c198e8db 100644 --- a/pkg/datapath/linux/config/config.go +++ b/pkg/datapath/linux/config/config.go @@ -464,6 +464,7 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC cDefinesMap["IPV6_RSS_PREFIX_BITS"] = "128" } } + if option.Config.NodePortAcceleration != option.NodePortAccelerationDisabled { cDefinesMap["ENABLE_NODEPORT_ACCELERATION"] = "1" } @@ -699,6 +700,10 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC } cDefinesMap["EPHEMERAL_MIN"] = fmt.Sprintf("%d", ephemeralMin) + if option.Config.LoadBalancerProtocolDifferentiation { + cDefinesMap["ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION"] = "1" + } + // Since golang maps are unordered, we sort the keys in the map // to get a consistent written format to the writer. This maintains // the consistency when we try to calculate hash for a datapath after diff --git a/pkg/datapath/types/lbmap.go b/pkg/datapath/types/lbmap.go index 44ba2bbe6b417..47cd80c214957 100644 --- a/pkg/datapath/types/lbmap.go +++ b/pkg/datapath/types/lbmap.go @@ -30,9 +30,10 @@ type LBMap interface { } type UpsertServiceParams struct { - ID uint16 - IP net.IP - Port uint16 + ID uint16 + IP net.IP + Port uint16 + Protocol uint8 // PreferredBackends is a subset of ActiveBackends // Note: this is only used in clustermesh with service affinity annotation. diff --git a/pkg/k8s/service.go b/pkg/k8s/service.go index 4ec47f42b9d7d..8bd202671dbd5 100644 --- a/pkg/k8s/service.go +++ b/pkg/k8s/service.go @@ -533,13 +533,10 @@ func NewService(ips []net.IP, externalIPs, loadBalancerIPs, loadBalancerSourceRa } // UniquePorts returns a map of all unique ports configured in the service -func (s *Service) UniquePorts() map[uint16]bool { - // We are not discriminating the different L4 protocols on the same L4 - // port so we create the number of unique sets of service IP + service - // port. - uniqPorts := map[uint16]bool{} +func (s *Service) UniquePorts() map[string]bool { + uniqPorts := map[string]bool{} for _, p := range s.Ports { - uniqPorts[p.Port] = true + uniqPorts[p.String()] = true } return uniqPorts } diff --git a/pkg/k8s/service_test.go b/pkg/k8s/service_test.go index f585f7e186eed..ae6b3b7d0d034 100644 --- a/pkg/k8s/service_test.go +++ b/pkg/k8s/service_test.go @@ -325,13 +325,13 @@ func (s *K8sSuite) TestIsK8ServiceExternal(c *check.C) { func (s *K8sSuite) TestServiceUniquePorts(c *check.C) { type testMatrix struct { input Service - expected map[uint16]bool + expected map[string]bool } matrix := []testMatrix{ { input: Service{}, - expected: map[uint16]bool{}, + expected: map[string]bool{}, }, { input: Service{ @@ -346,10 +346,11 @@ func (s *K8sSuite) TestServiceUniquePorts(c *check.C) { }, }, }, - expected: map[uint16]bool{ - 1: true, - 2: true, - }}, + expected: map[string]bool{ + "1/NONE": true, + "2/NONE": true, + }, + }, } for _, m := range matrix { diff --git a/pkg/k8s/slim/k8s/api/core/v1/generated.pb.go b/pkg/k8s/slim/k8s/api/core/v1/generated.pb.go index 63de1bc6a424f..f3807d01e251b 100644 --- a/pkg/k8s/slim/k8s/api/core/v1/generated.pb.go +++ b/pkg/k8s/slim/k8s/api/core/v1/generated.pb.go @@ -1175,172 +1175,181 @@ func init() { } var fileDescriptor_871504499faea14d = []byte{ - // 2636 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5a, 0xcf, 0x6f, 0x23, 0x49, - 0xf5, 0x9f, 0xb6, 0xe3, 0xc4, 0x7e, 0x4e, 0x9c, 0x4d, 0xcd, 0x66, 0xd5, 0x93, 0xef, 0x6c, 0x92, - 0x6f, 0x0f, 0x42, 0xc3, 0x2f, 0x5b, 0x33, 0x10, 0x98, 0x9d, 0xd9, 0x19, 0x26, 0x76, 0x32, 0x3b, - 0xde, 0x4d, 0x66, 0x9b, 0x72, 0xb4, 0x20, 0x38, 0xb0, 0x9d, 0xee, 0x8a, 0xdd, 0x1b, 0xbb, 0xdb, - 0x74, 0x95, 0x33, 0x6b, 0x09, 0xc1, 0x22, 0x04, 0x5a, 0x60, 0x24, 0xf8, 0x07, 0xb8, 0x70, 0xe3, - 0x0c, 0x17, 0xc4, 0x19, 0x31, 0xdc, 0x96, 0xdb, 0x08, 0xa1, 0xb0, 0x13, 0x24, 0x6e, 0x1c, 0x10, - 0x27, 0x82, 0x84, 0x50, 0x55, 0x57, 0x57, 0x77, 0x3b, 0xf6, 0x4c, 0xc6, 0x8e, 0x14, 0x71, 0x72, - 0xf7, 0x7b, 0xaf, 0xde, 0xa7, 0x5e, 0xd5, 0xab, 0xf7, 0xa3, 0xdc, 0xb0, 0xd9, 0x74, 0x59, 0xab, - 0xb7, 0x5b, 0xb6, 0xfd, 0x4e, 0xc5, 0x76, 0xdb, 0x6e, 0x4f, 0xfd, 0x74, 0xf7, 0x9b, 0x95, 0xfd, - 0x1b, 0xb4, 0x42, 0xdb, 0x6e, 0x47, 0x3c, 0x58, 0x5d, 0xb7, 0x62, 0xfb, 0x01, 0xa9, 0x1c, 0x5c, - 0xab, 0x34, 0x89, 0x47, 0x02, 0x8b, 0x11, 0xa7, 0xdc, 0x0d, 0x7c, 0xe6, 0xa3, 0xb5, 0x58, 0x4d, - 0x39, 0x1c, 0x1f, 0xfd, 0x74, 0xf7, 0x9b, 0xe5, 0xfd, 0x1b, 0xb4, 0xcc, 0xd5, 0x88, 0x07, 0xab, - 0xeb, 0x96, 0xb9, 0x9a, 0xf2, 0xc1, 0xb5, 0xa5, 0x7b, 0x2f, 0x84, 0x4e, 0x2b, 0x1d, 0xc2, 0xac, - 0x21, 0xf0, 0x4b, 0x9f, 0x4b, 0xe8, 0x69, 0xfa, 0x4d, 0xbf, 0x22, 0xc8, 0xbb, 0xbd, 0x3d, 0xf1, - 0x26, 0x5e, 0xc4, 0x93, 0x14, 0xe7, 0x0a, 0xcb, 0xae, 0xcf, 0x75, 0x76, 0x2c, 0xbb, 0xe5, 0x7a, - 0x24, 0xe8, 0x0b, 0xc4, 0xa0, 0xe7, 0x31, 0xb7, 0x43, 0x4e, 0xe8, 0xff, 0xe2, 0xf3, 0x06, 0x50, - 0xbb, 0x45, 0x3a, 0xd6, 0xe0, 0x38, 0x63, 0x0b, 0x4a, 0xb5, 0xb6, 0x4b, 0x3c, 0x56, 0x37, 0x6b, - 0xbe, 0xb7, 0xe7, 0x36, 0xd1, 0x4d, 0x28, 0xf1, 0x01, 0x7e, 0x8f, 0x35, 0x88, 0xed, 0x7b, 0x0e, - 0xd5, 0xb5, 0x55, 0xed, 0x6a, 0xae, 0x8a, 0x8e, 0x0e, 0x57, 0x4a, 0x3b, 0x29, 0x0e, 0x1e, 0x90, - 0x34, 0x7e, 0x9b, 0x81, 0x42, 0xcd, 0xf7, 0x98, 0xc5, 0xf1, 0xd1, 0x2a, 0x4c, 0x79, 0x56, 0x87, - 0x88, 0xf1, 0x85, 0xea, 0xec, 0xe3, 0xc3, 0x95, 0x0b, 0x47, 0x87, 0x2b, 0x53, 0x0f, 0xac, 0x0e, - 0xc1, 0x82, 0x83, 0xae, 0x40, 0xce, 0xed, 0x58, 0x4d, 0xa2, 0x67, 0x84, 0xc8, 0x9c, 0x14, 0xc9, - 0xd5, 0x39, 0x11, 0x87, 0x3c, 0xe4, 0x42, 0xae, 0xeb, 0x07, 0x8c, 0xea, 0xd3, 0xab, 0xd9, 0xab, - 0xc5, 0xeb, 0x1b, 0xe5, 0xb1, 0x76, 0xb2, 0xac, 0xe6, 0x65, 0xfa, 0x01, 0x8b, 0xa1, 0xf8, 0x1b, - 0xc5, 0x21, 0x02, 0xfa, 0x36, 0xcc, 0x1e, 0xf8, 0xed, 0x5e, 0x87, 0x6c, 0xfb, 0x3d, 0x8f, 0x51, - 0xbd, 0x20, 0x10, 0xab, 0x63, 0x22, 0xbe, 0x13, 0xab, 0xaa, 0xbe, 0x2c, 0xf1, 0x66, 0x13, 0x44, - 0x8a, 0x53, 0x68, 0xc6, 0x7f, 0x34, 0x98, 0x4b, 0xcd, 0xf2, 0x14, 0x2b, 0xf8, 0x59, 0xc8, 0xb7, - 0x7c, 0xca, 0xb8, 0xb4, 0x58, 0xc4, 0x5c, 0xf5, 0x25, 0x29, 0x95, 0xbf, 0x2f, 0xe9, 0x58, 0x49, - 0xa0, 0x5b, 0x30, 0x67, 0x27, 0x01, 0xf4, 0xac, 0x18, 0xb2, 0x28, 0x87, 0xa4, 0xd1, 0x71, 0x5a, - 0x16, 0xdd, 0x80, 0xbc, 0xf0, 0x19, 0xdb, 0x6f, 0xeb, 0x53, 0x62, 0x42, 0x97, 0x23, 0x28, 0x53, - 0xd2, 0x8f, 0x13, 0xcf, 0x58, 0x49, 0xa3, 0x4f, 0xc2, 0x34, 0x9f, 0x42, 0xdd, 0xd4, 0x73, 0x62, - 0x5c, 0x49, 0x8e, 0x9b, 0xbe, 0x2f, 0xa8, 0x58, 0x72, 0x8d, 0x1f, 0x68, 0x50, 0x52, 0x53, 0x68, - 0x30, 0x8b, 0x11, 0x44, 0x61, 0x26, 0xe8, 0x79, 0x9e, 0xeb, 0x35, 0x85, 0x79, 0xc5, 0xeb, 0x5b, - 0x93, 0x6e, 0xbf, 0xd0, 0x8b, 0x43, 0x9d, 0xd5, 0xe2, 0xd1, 0xe1, 0xca, 0x8c, 0x7c, 0xc1, 0x11, - 0x92, 0xf1, 0x43, 0x0d, 0x16, 0x87, 0xca, 0xa3, 0x0e, 0x14, 0x28, 0xb3, 0x02, 0x46, 0x9c, 0x75, - 0x26, 0x76, 0xa5, 0x78, 0xfd, 0xf5, 0x17, 0x9b, 0x10, 0x2d, 0xf3, 0x10, 0xc1, 0x67, 0xc4, 0xcf, - 0x52, 0x75, 0x41, 0x2e, 0x45, 0xa1, 0x11, 0xa9, 0xc5, 0x31, 0x82, 0xf1, 0x6b, 0x0d, 0xe6, 0x53, - 0x13, 0xe9, 0x51, 0xf4, 0x1e, 0xe4, 0x28, 0x9f, 0x92, 0x5c, 0x8f, 0xcd, 0x33, 0x59, 0x8f, 0xf8, - 0x3c, 0x84, 0xe6, 0x86, 0x10, 0x68, 0x0d, 0x8a, 0xca, 0x07, 0xea, 0x1b, 0x7a, 0x5e, 0xec, 0xde, - 0x45, 0x29, 0x5a, 0xac, 0xc5, 0x2c, 0x9c, 0x94, 0x33, 0xbe, 0x0a, 0xf3, 0x9b, 0x9e, 0xd3, 0xf5, - 0x5d, 0x8f, 0xad, 0x3b, 0x4e, 0x40, 0x28, 0x45, 0x4b, 0x90, 0x71, 0xbb, 0xd2, 0x8f, 0x41, 0x2a, - 0xc8, 0xd4, 0x4d, 0x9c, 0x71, 0xbb, 0xe8, 0x2a, 0xe4, 0x3d, 0xdf, 0x21, 0xdc, 0xab, 0xa5, 0x63, - 0xcd, 0x72, 0xa7, 0x7a, 0x20, 0x69, 0x58, 0x71, 0x8d, 0x47, 0x1a, 0xcc, 0x46, 0x9a, 0x4f, 0x79, - 0x40, 0x56, 0x61, 0xaa, 0x1b, 0x1f, 0x0e, 0x25, 0x21, 0x1c, 0x5c, 0x70, 0x52, 0x7e, 0x9d, 0x7d, - 0x11, 0xbf, 0x36, 0xfe, 0xa9, 0x41, 0x29, 0x9a, 0x4e, 0xa3, 0xb7, 0x4b, 0x09, 0x43, 0x0f, 0xa1, - 0x60, 0x85, 0x26, 0x13, 0x1e, 0x38, 0x79, 0xf8, 0xb8, 0x37, 0xe6, 0x0e, 0x0d, 0x2c, 0x61, 0xec, - 0x2a, 0xeb, 0x11, 0x00, 0x8e, 0xb1, 0x50, 0x2b, 0x8a, 0x92, 0x59, 0x01, 0x5a, 0x9b, 0x10, 0x74, - 0x74, 0x90, 0x34, 0xfe, 0xa1, 0x41, 0x21, 0x12, 0xa3, 0x28, 0x80, 0x3c, 0x77, 0x68, 0xc7, 0x62, - 0x96, 0x3c, 0x10, 0xd5, 0x71, 0x0f, 0xc4, 0xdb, 0xbb, 0xef, 0x11, 0x9b, 0x6d, 0x13, 0x66, 0x55, - 0x91, 0x44, 0x86, 0x98, 0x86, 0x15, 0x0e, 0xea, 0xc2, 0x0c, 0x15, 0xcb, 0x4d, 0xf5, 0x8c, 0xb0, - 0x76, 0x73, 0x42, 0x6b, 0xc3, 0xcd, 0xab, 0xce, 0x4b, 0xd4, 0x99, 0xf0, 0x9d, 0xe2, 0x08, 0xc6, - 0xf8, 0x9b, 0x06, 0x73, 0xca, 0xe6, 0x2d, 0x97, 0x32, 0xe4, 0x9d, 0xb0, 0xfb, 0xee, 0xb8, 0x76, - 0x73, 0x7d, 0xc2, 0x6a, 0x15, 0xba, 0x23, 0x4a, 0xc2, 0x66, 0x02, 0x39, 0x97, 0x91, 0x4e, 0x64, - 0xf1, 0xdd, 0x09, 0x2d, 0xa6, 0x89, 0x64, 0xcb, 0xd5, 0xe2, 0x50, 0xbb, 0xf1, 0x7b, 0x0d, 0x2e, - 0x6e, 0xf9, 0x96, 0x53, 0xb5, 0xda, 0x96, 0x67, 0x93, 0xa0, 0xee, 0x35, 0x9f, 0x7b, 0x7e, 0x65, - 0x0e, 0x12, 0x07, 0x31, 0x4c, 0xe4, 0xa9, 0x1c, 0xe4, 0x89, 0x33, 0x1c, 0x49, 0xa0, 0xbd, 0xc8, - 0x51, 0xa7, 0x84, 0x21, 0xeb, 0x63, 0x1a, 0xc2, 0x5d, 0x32, 0x8c, 0x88, 0x23, 0xdc, 0xf4, 0x27, - 0x1a, 0xa0, 0xa4, 0x25, 0x32, 0x7c, 0xf6, 0x60, 0xc6, 0x0d, 0x6d, 0x92, 0xc7, 0xf3, 0xcd, 0x31, - 0x27, 0x30, 0x64, 0x95, 0x62, 0x07, 0x92, 0x04, 0x1c, 0x61, 0x19, 0xdf, 0x85, 0x02, 0x0f, 0x4a, - 0xb4, 0x6b, 0xd9, 0xe4, 0x3c, 0xce, 0x8c, 0xf0, 0x60, 0x35, 0x83, 0xff, 0x65, 0x0f, 0x56, 0x46, - 0x8c, 0xf0, 0xe0, 0xc7, 0x19, 0x98, 0xe2, 0xa9, 0xe3, 0x5c, 0x22, 0x93, 0x05, 0x53, 0xb4, 0x4b, - 0x6c, 0x99, 0x9b, 0xbf, 0x3c, 0xae, 0x89, 0xbe, 0x43, 0x1a, 0x5d, 0x62, 0xc7, 0xe9, 0x8a, 0xbf, - 0x61, 0xa1, 0x1a, 0xb9, 0x30, 0x4d, 0x85, 0x2b, 0x8b, 0x64, 0x35, 0xfe, 0x01, 0x12, 0x20, 0xe1, - 0x01, 0x52, 0xf5, 0x58, 0xf8, 0x8e, 0x25, 0x80, 0xd1, 0x81, 0x22, 0x97, 0x8a, 0x72, 0xf8, 0xe7, - 0x61, 0x8a, 0xf5, 0xbb, 0x51, 0xb2, 0x5d, 0x89, 0xe6, 0xb6, 0xd3, 0xef, 0x92, 0xe3, 0xc3, 0x95, - 0xf9, 0x84, 0x28, 0x27, 0x61, 0x21, 0x8c, 0x3e, 0x05, 0x33, 0x32, 0x49, 0xc9, 0xd8, 0xa0, 0xce, - 0x88, 0x94, 0xc5, 0x11, 0xdf, 0xf8, 0x25, 0x77, 0x51, 0xdf, 0x21, 0x35, 0xdf, 0x73, 0x5c, 0xe6, - 0xfa, 0x1e, 0x5a, 0x4b, 0x21, 0xfe, 0xff, 0x00, 0xe2, 0x42, 0x4a, 0x38, 0x81, 0xf9, 0x9a, 0x5a, - 0xa2, 0x4c, 0x6a, 0xa0, 0xb4, 0x8f, 0x4f, 0x56, 0x0d, 0x4b, 0x9b, 0xcc, 0x4b, 0xd5, 0x80, 0x58, - 0xd4, 0xf7, 0x06, 0x4b, 0x55, 0x2c, 0xa8, 0x58, 0x72, 0x8d, 0x3f, 0x6b, 0x20, 0x0a, 0x94, 0x73, - 0x39, 0x49, 0xef, 0xa6, 0x4f, 0xd2, 0xad, 0x09, 0x3c, 0x60, 0xc4, 0x21, 0xfa, 0x97, 0x34, 0x8f, - 0xfb, 0x1d, 0xdf, 0xc2, 0xae, 0xef, 0xd4, 0xea, 0x1b, 0x58, 0x6e, 0x84, 0xda, 0x42, 0x33, 0x24, - 0xe3, 0x88, 0xcf, 0x4b, 0x39, 0xf9, 0x48, 0xf5, 0x99, 0xd5, 0x6c, 0x54, 0xca, 0x49, 0x39, 0x8a, - 0x15, 0x17, 0x5d, 0x07, 0xe8, 0x06, 0xfe, 0x81, 0xeb, 0x88, 0xca, 0x32, 0xac, 0xbb, 0xd4, 0xd9, - 0x32, 0x15, 0x07, 0x27, 0xa4, 0x90, 0x03, 0xd3, 0xbc, 0xc6, 0x64, 0x54, 0xcf, 0x09, 0xc3, 0x5f, - 0x1f, 0xd3, 0xf0, 0x1d, 0xae, 0x24, 0xde, 0x5a, 0xf1, 0x4a, 0xb1, 0xd4, 0x6d, 0xfc, 0x5b, 0x03, - 0x88, 0x0f, 0x07, 0x7a, 0x1f, 0xc0, 0x8e, 0x9c, 0x25, 0x4a, 0x5a, 0x1b, 0x13, 0xac, 0xb8, 0xf2, - 0xbc, 0xd8, 0x5c, 0x45, 0xa2, 0x38, 0x81, 0x85, 0x68, 0xb2, 0x96, 0xcc, 0x4d, 0xd4, 0x8a, 0x26, - 0xce, 0xe6, 0xb3, 0xeb, 0x48, 0xe3, 0x77, 0x19, 0xc8, 0x9a, 0xbe, 0x73, 0x2e, 0xd1, 0xf3, 0xdd, - 0x54, 0xf4, 0xbc, 0x33, 0x76, 0x65, 0xe0, 0x8c, 0x0c, 0x9e, 0xad, 0x81, 0xe0, 0x79, 0x77, 0x02, - 0x8c, 0x67, 0xc7, 0xce, 0x27, 0x59, 0x98, 0xe5, 0x6e, 0xaf, 0x62, 0xd9, 0x17, 0x52, 0xb1, 0x6c, - 0x75, 0x20, 0x96, 0xbd, 0x94, 0x94, 0x3d, 0x9b, 0x50, 0xd6, 0x87, 0xb9, 0xb6, 0x45, 0x99, 0x19, - 0xf8, 0xbb, 0x84, 0xf7, 0x9a, 0xd2, 0xe4, 0xc9, 0xfa, 0x55, 0x75, 0x55, 0xb0, 0x95, 0x54, 0x8d, - 0xd3, 0x48, 0xe8, 0x43, 0x0d, 0x10, 0xa7, 0xec, 0x04, 0x96, 0x47, 0x43, 0x93, 0x5c, 0xd9, 0xdc, - 0x4d, 0x3a, 0x81, 0x25, 0x39, 0x01, 0xb4, 0x75, 0x42, 0x3f, 0x1e, 0x82, 0x79, 0xda, 0x80, 0xce, - 0x83, 0x5c, 0x87, 0x50, 0x6a, 0x35, 0x89, 0x3e, 0x9d, 0x0e, 0x72, 0xdb, 0x21, 0x19, 0x47, 0x7c, - 0xe3, 0x0a, 0xe4, 0x4c, 0xdf, 0xa9, 0x9b, 0xcf, 0x2a, 0x8a, 0x8d, 0x3f, 0x69, 0xc0, 0xc3, 0xe3, - 0xb9, 0xe4, 0x87, 0x6f, 0xa6, 0xf3, 0xc3, 0xcd, 0xf1, 0x9d, 0x7c, 0x44, 0x7a, 0xf8, 0x55, 0x56, - 0x18, 0x27, 0xb2, 0xc3, 0x07, 0x1a, 0x94, 0x5c, 0xcf, 0x65, 0xea, 0x36, 0x80, 0xea, 0x2f, 0x4f, - 0x54, 0xe0, 0x29, 0x45, 0xd5, 0x57, 0x24, 0x78, 0xa9, 0x9e, 0xd2, 0x8f, 0x07, 0xf0, 0x10, 0x13, - 0x21, 0x3a, 0x42, 0xcf, 0x9c, 0x11, 0x7a, 0x32, 0x3c, 0x47, 0xc8, 0x09, 0x1c, 0xf4, 0x26, 0x20, - 0x4a, 0x82, 0x03, 0xd7, 0x26, 0xeb, 0xb6, 0xed, 0xf7, 0x3c, 0x26, 0x2e, 0x30, 0xc2, 0x3b, 0x12, - 0xe5, 0xa5, 0x8d, 0x13, 0x12, 0x78, 0xc8, 0x28, 0xde, 0x42, 0xa9, 0x2b, 0x10, 0x48, 0xb7, 0x50, - 0x27, 0xaf, 0x41, 0xd0, 0x1a, 0x14, 0x79, 0x3b, 0xf5, 0x80, 0xb0, 0x87, 0x7e, 0xb0, 0xaf, 0x17, - 0x57, 0xb5, 0xab, 0xf9, 0xf8, 0x5a, 0xe6, 0x7e, 0xcc, 0xc2, 0x49, 0x39, 0xe3, 0x17, 0x39, 0x28, - 0xa8, 0xc0, 0x85, 0x2a, 0x90, 0xeb, 0xb6, 0x2c, 0x1a, 0x05, 0xa4, 0x4b, 0xaa, 0x89, 0xe2, 0xc4, - 0xe3, 0x30, 0x69, 0x8b, 0x67, 0x1c, 0xca, 0xa1, 0x87, 0xa9, 0x44, 0x98, 0x99, 0xe8, 0x9a, 0x21, - 0x19, 0xed, 0x9e, 0x9b, 0x07, 0x4f, 0x79, 0x7d, 0x88, 0xae, 0xf0, 0xce, 0xd2, 0xa9, 0x9b, 0xf2, - 0x00, 0x27, 0xda, 0x42, 0xa7, 0x6e, 0xe2, 0x90, 0xc7, 0x6b, 0x08, 0xf1, 0x40, 0xf5, 0xd9, 0x89, - 0x6a, 0x08, 0xa1, 0x34, 0x9e, 0x8a, 0x78, 0xa5, 0x58, 0xea, 0x46, 0xae, 0xbc, 0x27, 0x14, 0x61, - 0x6f, 0xe6, 0x0c, 0xc2, 0xde, 0x9c, 0xba, 0x23, 0x14, 0x91, 0x2e, 0xd6, 0x8e, 0x7e, 0xaa, 0xc1, - 0x82, 0x9d, 0xbe, 0x23, 0x24, 0x54, 0xcf, 0x4f, 0x74, 0xf5, 0x34, 0x70, 0xe7, 0xa8, 0x9c, 0x63, - 0xa1, 0x36, 0x08, 0x84, 0x4f, 0x62, 0xa3, 0x5b, 0x90, 0xff, 0x96, 0x4f, 0x6b, 0x6d, 0x8b, 0x52, - 0xbd, 0x90, 0xea, 0x15, 0xf2, 0x5f, 0x79, 0xbb, 0x21, 0xe8, 0xc7, 0x87, 0x2b, 0x45, 0xd3, 0x77, - 0xa2, 0x57, 0xac, 0x06, 0x18, 0x3f, 0xd2, 0x00, 0xe2, 0xde, 0x5e, 0x5d, 0xdf, 0x69, 0xa7, 0xba, - 0xbe, 0xcb, 0xbc, 0xd0, 0xb5, 0xf4, 0x0a, 0xe4, 0x48, 0x10, 0xf8, 0x81, 0xac, 0x3e, 0x0b, 0xdc, - 0x57, 0x36, 0x39, 0x01, 0x87, 0x74, 0xe3, 0x0f, 0x53, 0x30, 0xdd, 0x20, 0x76, 0x40, 0xd8, 0xb9, - 0x94, 0x43, 0x9f, 0x81, 0x82, 0xdb, 0xe9, 0xf4, 0x98, 0xb5, 0xdb, 0x26, 0xc2, 0xf5, 0xf3, 0xa1, - 0x1b, 0xd4, 0x23, 0x22, 0x8e, 0xf9, 0x28, 0x80, 0x29, 0x31, 0xb9, 0xf0, 0x5c, 0xbe, 0x31, 0xe6, - 0xc6, 0x87, 0xd6, 0x96, 0x37, 0x2c, 0x66, 0x6d, 0x7a, 0x2c, 0xe8, 0xab, 0x7c, 0x3f, 0xc5, 0x49, - 0x3f, 0xfe, 0xcb, 0x4a, 0xae, 0xda, 0x67, 0x84, 0x62, 0x81, 0x85, 0xbe, 0xa7, 0x01, 0x50, 0x16, - 0xb8, 0x5e, 0x93, 0x73, 0x65, 0x6d, 0xbc, 0x3d, 0x19, 0x74, 0x43, 0xe9, 0x0b, 0x27, 0xa0, 0x96, - 0x28, 0x66, 0xe0, 0x04, 0x28, 0x2a, 0xcb, 0xb2, 0x2a, 0x9b, 0x8a, 0xbb, 0x51, 0x59, 0x05, 0xa1, - 0xd6, 0xb8, 0xa0, 0x5a, 0xfa, 0x12, 0x14, 0x94, 0x72, 0xf4, 0x12, 0x64, 0xf7, 0x49, 0x3f, 0x8c, - 0x80, 0x98, 0x3f, 0xa2, 0x97, 0x21, 0x77, 0x60, 0xb5, 0x7b, 0xe1, 0x45, 0xd6, 0x2c, 0x0e, 0x5f, - 0x6e, 0x66, 0x6e, 0x68, 0x4b, 0xb7, 0x61, 0x7e, 0x60, 0x6e, 0xcf, 0x1b, 0x5e, 0x48, 0x0c, 0x37, - 0x3e, 0xd6, 0x40, 0x4e, 0xe6, 0x5c, 0x4a, 0x82, 0xdd, 0x74, 0x49, 0x70, 0x7b, 0xa2, 0x4d, 0x1a, - 0x51, 0x15, 0xfc, 0x31, 0x03, 0x33, 0x32, 0xdf, 0x9d, 0xcb, 0x79, 0x71, 0x52, 0xed, 0x43, 0x75, - 0x6c, 0x13, 0x85, 0x05, 0x23, 0x5b, 0x88, 0xf6, 0x40, 0x0b, 0xb1, 0x31, 0x21, 0xce, 0xb3, 0xdb, - 0x88, 0x23, 0x0d, 0x8a, 0x52, 0xf2, 0x5c, 0xfc, 0xc6, 0x4e, 0xfb, 0xcd, 0x9d, 0xc9, 0x8c, 0x1d, - 0xe1, 0x38, 0xbf, 0x89, 0x8d, 0x3c, 0xe5, 0xbf, 0x3a, 0xe3, 0x07, 0xfd, 0x28, 0xa1, 0x64, 0x47, - 0x26, 0x14, 0x59, 0x8b, 0x89, 0xff, 0x47, 0x73, 0xe9, 0xbf, 0x54, 0x1f, 0x48, 0x3a, 0x56, 0x12, - 0xc6, 0xa3, 0xa2, 0x9a, 0xbb, 0x28, 0x87, 0x9b, 0xd1, 0xf5, 0xb6, 0x36, 0x51, 0xc3, 0x9e, 0x58, - 0x8e, 0x11, 0xff, 0x55, 0x7f, 0x07, 0xf2, 0x94, 0xb4, 0x89, 0xcd, 0xfc, 0x40, 0x6e, 0x8e, 0x39, - 0xb9, 0xc7, 0x97, 0x1b, 0x52, 0x65, 0x18, 0x7c, 0x95, 0xe1, 0x11, 0x19, 0x2b, 0x4c, 0x54, 0x81, - 0x82, 0xdd, 0xee, 0x51, 0x46, 0x82, 0xba, 0x29, 0xa3, 0xaf, 0xba, 0x59, 0xa8, 0x45, 0x0c, 0x1c, - 0xcb, 0xa0, 0x32, 0x80, 0x7a, 0xa1, 0x3a, 0x12, 0xb7, 0x43, 0x25, 0x51, 0xf6, 0x29, 0x2a, 0x4e, - 0x48, 0xa0, 0x8a, 0x8c, 0xec, 0xe1, 0x5f, 0x82, 0xff, 0x37, 0x10, 0xd9, 0xa3, 0x45, 0x4f, 0xf4, - 0xca, 0xd7, 0xa0, 0x48, 0xde, 0x67, 0x24, 0xf0, 0xac, 0x36, 0x47, 0xc8, 0x09, 0x84, 0x79, 0x5e, - 0x12, 0x6f, 0xc6, 0x64, 0x9c, 0x94, 0x41, 0x3b, 0x30, 0x4f, 0x09, 0xa5, 0xae, 0xef, 0xad, 0xef, - 0xed, 0xf1, 0xae, 0xa2, 0x2f, 0xaa, 0xb5, 0x42, 0xf5, 0xd3, 0x12, 0x6e, 0xbe, 0x91, 0x66, 0x1f, - 0x0b, 0x52, 0x58, 0xbf, 0x4b, 0x12, 0x1e, 0x54, 0x81, 0xee, 0x40, 0xa9, 0x9d, 0xfc, 0x77, 0xc0, - 0x94, 0x5d, 0x81, 0xea, 0x67, 0x52, 0xff, 0x1d, 0x98, 0x78, 0x40, 0x1a, 0x7d, 0x0d, 0xf4, 0x24, - 0xa5, 0xe1, 0xf7, 0x02, 0x9b, 0x60, 0xcb, 0x6b, 0x92, 0xf0, 0x93, 0x84, 0x42, 0xf5, 0xf2, 0xd1, - 0xe1, 0x8a, 0xbe, 0x35, 0x42, 0x06, 0x8f, 0x1c, 0x8d, 0x28, 0x2c, 0x46, 0xe6, 0xef, 0x04, 0xd6, - 0xde, 0x9e, 0x6b, 0x9b, 0x7e, 0xdb, 0xb5, 0xfb, 0xa2, 0x87, 0x28, 0x54, 0x6f, 0xcb, 0x09, 0x2e, - 0x6e, 0x0e, 0x13, 0x3a, 0x3e, 0x5c, 0xb9, 0x2c, 0x6d, 0x1f, 0xca, 0xc7, 0xc3, 0x75, 0xa3, 0x6d, - 0xb8, 0xd8, 0x22, 0x56, 0x9b, 0xb5, 0x6a, 0x2d, 0x62, 0xef, 0x47, 0x67, 0x48, 0x9f, 0x15, 0x67, - 0x2b, 0xda, 0xd7, 0x8b, 0xf7, 0x4f, 0x8a, 0xe0, 0x61, 0xe3, 0xd0, 0xcf, 0x35, 0x58, 0x1c, 0x58, - 0xf1, 0xf0, 0xd3, 0x15, 0xbd, 0x34, 0xd1, 0x17, 0x02, 0x8d, 0x61, 0x3a, 0xab, 0x97, 0xf8, 0x72, - 0x0c, 0x65, 0xe1, 0xe1, 0xb3, 0x40, 0x37, 0x01, 0xdc, 0xee, 0x3d, 0xab, 0xe3, 0xb6, 0x5d, 0x42, - 0xf5, 0x8b, 0x62, 0xbf, 0x96, 0xb8, 0x9f, 0xd7, 0xcd, 0x88, 0xca, 0x63, 0x93, 0x7c, 0xeb, 0xe3, - 0x84, 0x34, 0xda, 0x82, 0x92, 0x7c, 0xeb, 0xcb, 0x8d, 0x59, 0x10, 0x1b, 0xf3, 0x09, 0xd1, 0x05, - 0x9b, 0x49, 0xce, 0xf1, 0x09, 0x0a, 0x1e, 0x18, 0x8b, 0x6a, 0xb0, 0x90, 0xf4, 0x84, 0xb0, 0x22, - 0x5f, 0x14, 0x0a, 0x17, 0x79, 0x35, 0xbf, 0x35, 0xc8, 0xc4, 0x27, 0xe5, 0x91, 0x0f, 0x8b, 0xae, - 0x37, 0xcc, 0x65, 0x5e, 0x11, 0x8a, 0x5e, 0xe3, 0xeb, 0x53, 0xf7, 0x9e, 0xed, 0x2e, 0x43, 0xf9, - 0x78, 0xb8, 0xde, 0xa5, 0x5b, 0x30, 0x97, 0x8a, 0x42, 0x2f, 0x54, 0x66, 0x3d, 0xca, 0xf0, 0xd1, - 0x89, 0xcc, 0x8a, 0xbe, 0xaf, 0xc1, 0x6c, 0xd2, 0x2a, 0x99, 0x36, 0xeb, 0x67, 0xf0, 0xb7, 0x9f, - 0xcc, 0xdd, 0xea, 0xdb, 0x9e, 0x24, 0x0f, 0xa7, 0x40, 0x51, 0x6f, 0x48, 0xf3, 0xbc, 0x3e, 0x6e, - 0xe6, 0x3e, 0x75, 0xeb, 0x6c, 0x7c, 0xa8, 0xc1, 0x70, 0xe7, 0x45, 0x3e, 0xe4, 0x6d, 0xf9, 0xe1, - 0x97, 0x5c, 0x91, 0xb1, 0xbf, 0x24, 0x49, 0x7d, 0x3f, 0x16, 0x5e, 0xf8, 0x47, 0x34, 0xac, 0x40, - 0x8c, 0xbf, 0x6b, 0x90, 0x13, 0x37, 0xed, 0xe8, 0xd5, 0xc4, 0x7e, 0x56, 0x8b, 0xd2, 0x82, 0xec, - 0x5b, 0xa4, 0x1f, 0x6e, 0xee, 0x95, 0xd4, 0xe6, 0xc6, 0xd9, 0xef, 0x1d, 0x4e, 0x94, 0x7b, 0x8d, - 0xd6, 0x60, 0x9a, 0xec, 0xed, 0x11, 0x9b, 0xc9, 0xd4, 0xf3, 0x6a, 0x54, 0x3f, 0x6d, 0x0a, 0x2a, - 0x4f, 0x10, 0x02, 0x2c, 0x7c, 0xc5, 0x52, 0x98, 0xf7, 0xe5, 0xcc, 0xed, 0x90, 0x75, 0xc7, 0x21, - 0xce, 0x99, 0x5c, 0x47, 0x8a, 0x86, 0x6c, 0x27, 0x52, 0x89, 0x63, 0xed, 0xbc, 0x91, 0xbd, 0xc4, - 0x93, 0x93, 0xb3, 0xe5, 0xdb, 0x56, 0x3b, 0x2c, 0x58, 0x31, 0xd9, 0x23, 0x01, 0xf1, 0x6c, 0x82, - 0xae, 0x42, 0xde, 0xea, 0xba, 0x6f, 0x04, 0x7e, 0x2f, 0xba, 0x40, 0x14, 0xeb, 0xb6, 0x6e, 0xd6, - 0x05, 0x0d, 0x2b, 0x2e, 0x2f, 0x58, 0xf6, 0x5d, 0xcf, 0x91, 0xab, 0xa1, 0x0a, 0x96, 0xb7, 0x5c, - 0xcf, 0xc1, 0x82, 0xa3, 0xca, 0xa5, 0xec, 0xa8, 0x72, 0xc9, 0xb8, 0x03, 0xc5, 0xc4, 0x77, 0x67, - 0x3c, 0x75, 0x77, 0xf8, 0x83, 0x69, 0xb1, 0xd6, 0x60, 0xea, 0xde, 0x8e, 0x18, 0x38, 0x96, 0xa9, - 0x7e, 0xe3, 0xf1, 0xd3, 0xe5, 0x0b, 0x1f, 0x3d, 0x5d, 0xbe, 0xf0, 0xe4, 0xe9, 0xf2, 0x85, 0x0f, - 0x8e, 0x96, 0xb5, 0xc7, 0x47, 0xcb, 0xda, 0x47, 0x47, 0xcb, 0xda, 0x93, 0xa3, 0x65, 0xed, 0xe3, - 0xa3, 0x65, 0xed, 0x67, 0x7f, 0x5d, 0xbe, 0xf0, 0xf5, 0xb5, 0xb1, 0xbe, 0xd4, 0xfc, 0x6f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x3f, 0xf4, 0xc9, 0x6c, 0xe1, 0x29, 0x00, 0x00, + // 2770 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5a, 0xcf, 0x6f, 0x24, 0x47, + 0xf5, 0xdf, 0x9e, 0xf1, 0xd8, 0x33, 0x6f, 0xec, 0x71, 0x5c, 0x1b, 0x47, 0x13, 0x7f, 0x13, 0xdb, + 0xdf, 0x5e, 0x84, 0x36, 0xfc, 0x98, 0xd1, 0x2e, 0x18, 0x36, 0xbb, 0xc9, 0x12, 0xcf, 0xd8, 0xd9, + 0x9d, 0xc4, 0xde, 0x1d, 0x6a, 0xac, 0x08, 0xc1, 0x81, 0xb4, 0xbb, 0xcb, 0xe3, 0x8a, 0x67, 0xba, + 0x9b, 0xae, 0x1a, 0x27, 0x23, 0x10, 0x24, 0x42, 0xa0, 0x00, 0x91, 0xe0, 0x1f, 0xe0, 0xc2, 0x8d, + 0x33, 0xdc, 0x38, 0x70, 0x42, 0x5a, 0x6e, 0xe1, 0x16, 0x21, 0x64, 0x12, 0x23, 0x21, 0x71, 0xe0, + 0x80, 0x38, 0x61, 0x24, 0x84, 0xaa, 0xba, 0xba, 0xba, 0x7b, 0x3c, 0xb3, 0x6b, 0xcf, 0x58, 0xb2, + 0x38, 0xcd, 0xf4, 0x7b, 0xaf, 0xde, 0xa7, 0x5e, 0xd5, 0xfb, 0x55, 0xd5, 0x0d, 0x9b, 0x6d, 0xca, + 0xf7, 0x7b, 0xbb, 0x15, 0xdb, 0xeb, 0x56, 0x6d, 0xda, 0xa1, 0x3d, 0xfd, 0xe3, 0x1f, 0xb4, 0xab, + 0x07, 0xb7, 0x58, 0x95, 0x75, 0x68, 0x57, 0xfe, 0xb1, 0x7c, 0x5a, 0xb5, 0xbd, 0x80, 0x54, 0x0f, + 0x6f, 0x54, 0xdb, 0xc4, 0x25, 0x81, 0xc5, 0x89, 0x53, 0xf1, 0x03, 0x8f, 0x7b, 0x68, 0x2d, 0x56, + 0x53, 0x09, 0xc7, 0x47, 0x3f, 0xfe, 0x41, 0xbb, 0x72, 0x70, 0x8b, 0x55, 0x84, 0x1a, 0xf9, 0xc7, + 0xf2, 0x69, 0x45, 0xa8, 0xa9, 0x1c, 0xde, 0x58, 0x7a, 0xf5, 0x5c, 0xe8, 0xac, 0xda, 0x25, 0xdc, + 0x1a, 0x02, 0xbf, 0xf4, 0xda, 0x39, 0xf5, 0xf4, 0x38, 0xed, 0x54, 0xa9, 0xcb, 0x19, 0x0f, 0x4e, + 0xe9, 0xfa, 0x7c, 0x42, 0x57, 0xdb, 0x6b, 0x7b, 0x55, 0x49, 0xde, 0xed, 0xed, 0xc9, 0x27, 0xf9, + 0x20, 0xff, 0x29, 0x71, 0xa1, 0xb4, 0x42, 0x3d, 0xa1, 0xb7, 0x6b, 0xd9, 0xfb, 0xd4, 0x25, 0x41, + 0x5f, 0xa2, 0x06, 0x3d, 0x97, 0xd3, 0x2e, 0x39, 0xa5, 0xff, 0x4b, 0x4f, 0x1a, 0xc0, 0xec, 0x7d, + 0xd2, 0xb5, 0x06, 0xc7, 0x99, 0x5b, 0x50, 0xaa, 0x77, 0x28, 0x71, 0x79, 0xa3, 0x59, 0xf7, 0xdc, + 0x3d, 0xda, 0x46, 0xb7, 0xa1, 0x24, 0x06, 0x78, 0x3d, 0xde, 0x22, 0xb6, 0xe7, 0x3a, 0xac, 0x6c, + 0xac, 0x1a, 0xd7, 0x73, 0x35, 0x74, 0x7c, 0xb4, 0x52, 0xda, 0x49, 0x71, 0xf0, 0x80, 0xa4, 0xf9, + 0x9b, 0x0c, 0x14, 0xea, 0x9e, 0xcb, 0x2d, 0x81, 0x8f, 0x56, 0x61, 0xca, 0xb5, 0xba, 0x44, 0x8e, + 0x2f, 0xd4, 0x66, 0x1f, 0x1d, 0xad, 0x5c, 0x39, 0x3e, 0x5a, 0x99, 0x7a, 0x60, 0x75, 0x09, 0x96, + 0x1c, 0x74, 0x0d, 0x72, 0xb4, 0x6b, 0xb5, 0x49, 0x39, 0x23, 0x45, 0xe6, 0x94, 0x48, 0xae, 0x21, + 0x88, 0x38, 0xe4, 0x21, 0x0a, 0x39, 0xdf, 0x0b, 0x38, 0x2b, 0x4f, 0xaf, 0x66, 0xaf, 0x17, 0x6f, + 0x6e, 0x54, 0xc6, 0xf2, 0x8a, 0x8a, 0x9e, 0x57, 0xd3, 0x0b, 0x78, 0x0c, 0x25, 0x9e, 0x18, 0x0e, + 0x11, 0xd0, 0x77, 0x60, 0xf6, 0xd0, 0xeb, 0xf4, 0xba, 0x64, 0xdb, 0xeb, 0xb9, 0x9c, 0x95, 0x0b, + 0x12, 0xb1, 0x36, 0x26, 0xe2, 0x1b, 0xb1, 0xaa, 0xda, 0xd3, 0x0a, 0x6f, 0x36, 0x41, 0x64, 0x38, + 0x85, 0x66, 0xfe, 0xc7, 0x80, 0xb9, 0xd4, 0x2c, 0xcf, 0xb0, 0x82, 0x9f, 0x83, 0xfc, 0xbe, 0xc7, + 0xb8, 0x90, 0x96, 0x8b, 0x98, 0xab, 0x3d, 0xa5, 0xa4, 0xf2, 0xf7, 0x15, 0x1d, 0x6b, 0x09, 0x74, + 0x07, 0xe6, 0xec, 0x24, 0x40, 0x39, 0x2b, 0x87, 0x2c, 0xaa, 0x21, 0x69, 0x74, 0x9c, 0x96, 0x45, + 0xb7, 0x20, 0x2f, 0x7d, 0xc6, 0xf6, 0x3a, 0xe5, 0x29, 0x39, 0xa1, 0xe7, 0x22, 0xa8, 0xa6, 0xa2, + 0x9f, 0x24, 0xfe, 0x63, 0x2d, 0x8d, 0x3e, 0x0d, 0xd3, 0x62, 0x0a, 0x8d, 0x66, 0x39, 0x27, 0xc7, + 0x95, 0xd4, 0xb8, 0xe9, 0xfb, 0x92, 0x8a, 0x15, 0xd7, 0xfc, 0x81, 0x01, 0x25, 0x3d, 0x85, 0x16, + 0xb7, 0x38, 0x41, 0x0c, 0x66, 0x82, 0x9e, 0xeb, 0x52, 0xb7, 0x2d, 0xcd, 0x2b, 0xde, 0xdc, 0x9a, + 0x74, 0xfb, 0xa5, 0x5e, 0x1c, 0xea, 0xac, 0x15, 0x8f, 0x8f, 0x56, 0x66, 0xd4, 0x03, 0x8e, 0x90, + 0xcc, 0x1f, 0x1a, 0xb0, 0x38, 0x54, 0x1e, 0x75, 0xa1, 0xc0, 0xb8, 0x15, 0x70, 0xe2, 0xac, 0x73, + 0xb9, 0x2b, 0xc5, 0x9b, 0x2f, 0x9d, 0x6f, 0x42, 0xac, 0x22, 0xd2, 0x8d, 0x98, 0x91, 0x88, 0xa5, + 0xda, 0x82, 0x5a, 0x8a, 0x42, 0x2b, 0x52, 0x8b, 0x63, 0x04, 0xf3, 0xd7, 0x06, 0xcc, 0xa7, 0x26, + 0xd2, 0x63, 0xe8, 0x2d, 0xc8, 0x31, 0x31, 0x25, 0xb5, 0x1e, 0x9b, 0x17, 0xb2, 0x1e, 0x71, 0x3c, + 0x84, 0xe6, 0x86, 0x10, 0x68, 0x0d, 0x8a, 0xda, 0x07, 0x1a, 0x1b, 0xe5, 0xbc, 0xdc, 0xbd, 0xab, + 0x4a, 0xb4, 0x58, 0x8f, 0x59, 0x38, 0x29, 0x27, 0xf6, 0x71, 0x7e, 0xd3, 0x75, 0x7c, 0x8f, 0xba, + 0x7c, 0xdd, 0x71, 0x02, 0xc2, 0x18, 0x5a, 0x82, 0x0c, 0xf5, 0x95, 0x23, 0x83, 0xd2, 0x90, 0x69, + 0x34, 0x71, 0x86, 0xfa, 0x91, 0x13, 0x4b, 0x57, 0xcf, 0x4a, 0x89, 0x94, 0x13, 0x0b, 0x3a, 0xd6, + 0x12, 0xe8, 0x3a, 0xe4, 0x5d, 0xcf, 0x21, 0x22, 0x08, 0x94, 0x1f, 0xce, 0x0a, 0xc9, 0x07, 0x8a, + 0x86, 0x35, 0xd7, 0xfc, 0xad, 0x01, 0xb3, 0xd1, 0x3c, 0xce, 0x18, 0x4f, 0xab, 0x30, 0xe5, 0xc7, + 0xb1, 0xa4, 0x25, 0x64, 0x3c, 0x48, 0x4e, 0x2a, 0x0c, 0xb2, 0xe7, 0x0a, 0x83, 0x1b, 0x50, 0xb4, + 0x7c, 0xbf, 0x99, 0x8e, 0xa1, 0x79, 0xb1, 0x92, 0xeb, 0x31, 0x19, 0x27, 0x65, 0xcc, 0x7f, 0x1a, + 0x50, 0x8a, 0x2c, 0x68, 0xf5, 0x76, 0x19, 0xe1, 0xe8, 0x6d, 0x28, 0x58, 0xe1, 0x9a, 0x12, 0x91, + 0x9a, 0x45, 0x82, 0x7a, 0x75, 0x4c, 0x1f, 0x18, 0xd8, 0xa3, 0xd8, 0x19, 0xd7, 0x23, 0x00, 0x1c, + 0x63, 0xa1, 0xfd, 0x28, 0x0f, 0x67, 0x25, 0x68, 0x7d, 0x42, 0xd0, 0xd1, 0x69, 0xd8, 0xfc, 0x87, + 0x01, 0x85, 0x48, 0x8c, 0xa1, 0x00, 0xf2, 0x22, 0x64, 0x1c, 0x8b, 0x5b, 0x2a, 0xe4, 0x6a, 0xe3, + 0x86, 0xdc, 0xc3, 0xdd, 0xb7, 0x88, 0xcd, 0xb7, 0x09, 0xb7, 0x6a, 0x48, 0x21, 0x43, 0x4c, 0xc3, + 0x1a, 0x07, 0xf9, 0x30, 0xc3, 0xe4, 0x72, 0xb3, 0x72, 0x46, 0x5a, 0xbb, 0x39, 0xa1, 0xb5, 0xe1, + 0xe6, 0xd5, 0xe6, 0x15, 0xea, 0x4c, 0xf8, 0xcc, 0x70, 0x04, 0x63, 0xfe, 0xd5, 0x80, 0x39, 0x6d, + 0xf3, 0x16, 0x65, 0x1c, 0xb9, 0xa7, 0xec, 0x7e, 0x65, 0x5c, 0xbb, 0x85, 0x3e, 0x69, 0xb5, 0x8e, + 0xab, 0x88, 0x92, 0xb0, 0x99, 0x40, 0x8e, 0x72, 0xd2, 0x8d, 0x2c, 0x7e, 0x65, 0x42, 0x8b, 0x59, + 0xa2, 0x9c, 0x0b, 0xb5, 0x38, 0xd4, 0x6e, 0xbe, 0x97, 0x81, 0xab, 0x5b, 0x9e, 0xe5, 0xd4, 0xac, + 0x8e, 0xe5, 0xda, 0x24, 0x68, 0xb8, 0xed, 0x73, 0x25, 0x88, 0xcc, 0x13, 0x13, 0xc4, 0x2d, 0x98, + 0xa6, 0xfe, 0xb6, 0xe7, 0x44, 0xc9, 0x64, 0x55, 0x94, 0x9a, 0x46, 0x53, 0x50, 0x4e, 0x8e, 0x56, + 0x50, 0x0a, 0x5c, 0x52, 0xb1, 0x92, 0x47, 0x7b, 0x91, 0x8b, 0x4f, 0xc9, 0x25, 0x58, 0x1f, 0x73, + 0x09, 0x84, 0x33, 0x87, 0xd9, 0x7a, 0x84, 0x83, 0xff, 0xc4, 0x80, 0xd4, 0x34, 0x54, 0x6a, 0xef, + 0xc1, 0x0c, 0x0d, 0x57, 0x43, 0x05, 0xf6, 0x6b, 0x63, 0x4e, 0x60, 0xc8, 0xfa, 0xc6, 0xae, 0xa7, + 0x08, 0x38, 0xc2, 0x32, 0xbf, 0x07, 0x05, 0x91, 0x01, 0x99, 0x6f, 0xd9, 0xe4, 0x32, 0xa2, 0x4d, + 0xfa, 0xbe, 0x9e, 0xc1, 0xff, 0xb2, 0xef, 0x6b, 0x23, 0x46, 0xf8, 0xfe, 0xa3, 0x0c, 0x4c, 0x89, + 0x3a, 0x75, 0x29, 0x39, 0xcd, 0x82, 0x29, 0xe6, 0x13, 0x5b, 0xf5, 0x0d, 0x5f, 0x19, 0xd7, 0x44, + 0xcf, 0x21, 0x2d, 0x9f, 0xd8, 0x71, 0x6d, 0x14, 0x4f, 0x58, 0xaa, 0x46, 0x14, 0xa6, 0x99, 0x74, + 0x65, 0x19, 0x79, 0xe3, 0x07, 0x90, 0x04, 0x09, 0x03, 0x48, 0xf7, 0x8a, 0xe1, 0x33, 0x56, 0x00, + 0x66, 0x17, 0x8a, 0x42, 0x2a, 0x6a, 0x2f, 0xbe, 0x00, 0x53, 0xbc, 0xef, 0x47, 0x95, 0x7d, 0x25, + 0x9a, 0xdb, 0x4e, 0xdf, 0x17, 0x31, 0x3f, 0x9f, 0x10, 0x15, 0x24, 0x2c, 0x85, 0xd1, 0x0b, 0x30, + 0xa3, 0xca, 0x9b, 0xca, 0x2a, 0x3a, 0x46, 0x94, 0x2c, 0x8e, 0xf8, 0xe6, 0x2f, 0x85, 0x8b, 0x7a, + 0x0e, 0xa9, 0x7b, 0xae, 0x43, 0x39, 0xf5, 0x5c, 0xb4, 0x96, 0x42, 0xfc, 0xff, 0x01, 0xc4, 0x85, + 0x94, 0x70, 0x02, 0xf3, 0x45, 0xbd, 0x44, 0x99, 0xd4, 0x40, 0x65, 0x9f, 0x98, 0xac, 0x1e, 0x96, + 0x36, 0x59, 0xb4, 0xd1, 0x01, 0xb1, 0x98, 0xe7, 0x0e, 0xb6, 0xd1, 0x58, 0x52, 0xb1, 0xe2, 0x9a, + 0x7f, 0x32, 0x40, 0x76, 0x43, 0x97, 0x12, 0x49, 0x6f, 0xa6, 0x23, 0xe9, 0xce, 0x04, 0x1e, 0x30, + 0x22, 0x88, 0xfe, 0xa5, 0xcc, 0x13, 0x7e, 0x27, 0xb6, 0xd0, 0xf7, 0x9c, 0x7a, 0x63, 0x03, 0xab, + 0x8d, 0xd0, 0x5b, 0xd8, 0x0c, 0xc9, 0x38, 0xe2, 0x8b, 0xbe, 0x51, 0xfd, 0x65, 0xe5, 0x99, 0xd5, + 0x6c, 0xd4, 0x37, 0x2a, 0x39, 0x86, 0x35, 0x17, 0xdd, 0x04, 0xf0, 0x03, 0xef, 0x90, 0x3a, 0xb2, + 0xeb, 0x0d, 0x8b, 0x88, 0x8e, 0xad, 0xa6, 0xe6, 0xe0, 0x84, 0x14, 0x72, 0x60, 0x5a, 0xf4, 0xbf, + 0x9c, 0x95, 0x73, 0xd2, 0xf0, 0x97, 0xc6, 0x34, 0x7c, 0x47, 0x28, 0x89, 0xb7, 0x56, 0x3e, 0x32, + 0xac, 0x74, 0x9b, 0xff, 0x36, 0x00, 0xe2, 0xe0, 0x40, 0xef, 0x00, 0xd8, 0x91, 0xb3, 0x44, 0x45, + 0x6b, 0x63, 0x82, 0x15, 0xd7, 0x9e, 0x17, 0x9b, 0xab, 0x49, 0x0c, 0x27, 0xb0, 0x10, 0x4b, 0x76, + 0xa1, 0xb9, 0x89, 0x8e, 0xc9, 0x89, 0xd8, 0x7c, 0x7c, 0x07, 0x6a, 0xfe, 0x2e, 0x03, 0xd9, 0xa6, + 0xe7, 0x5c, 0x4a, 0xf6, 0x7c, 0x33, 0x95, 0x3d, 0xef, 0x8e, 0xdd, 0x19, 0x38, 0x23, 0x93, 0xe7, + 0xfe, 0x40, 0xf2, 0x7c, 0x65, 0x02, 0x8c, 0xc7, 0xe7, 0xce, 0x8f, 0xb2, 0x30, 0x2b, 0xdc, 0x5e, + 0xe7, 0xb2, 0x2f, 0xa6, 0x72, 0xd9, 0xea, 0x40, 0x2e, 0x7b, 0x2a, 0x29, 0x7b, 0x31, 0xa9, 0xac, + 0x0f, 0x73, 0x1d, 0x8b, 0xf1, 0x66, 0xe0, 0xed, 0x12, 0x71, 0x0e, 0x56, 0x26, 0x4f, 0x76, 0x96, + 0xd6, 0xd7, 0x18, 0x5b, 0x49, 0xd5, 0x38, 0x8d, 0x84, 0xde, 0x37, 0x00, 0x09, 0xca, 0x4e, 0x60, + 0xb9, 0x2c, 0x34, 0x89, 0xaa, 0x93, 0xe4, 0xa4, 0x13, 0x58, 0x52, 0x13, 0x40, 0x5b, 0xa7, 0xf4, + 0xe3, 0x21, 0x98, 0x67, 0x4d, 0xe8, 0x22, 0xc9, 0x75, 0x09, 0x63, 0x56, 0x9b, 0x94, 0xa7, 0xd3, + 0x49, 0x6e, 0x3b, 0x24, 0xe3, 0x88, 0x6f, 0x5e, 0x83, 0x5c, 0xd3, 0x73, 0x1a, 0xcd, 0xc7, 0xb5, + 0xd3, 0xe6, 0x1f, 0x0d, 0x10, 0xe9, 0xf1, 0x52, 0xea, 0xc3, 0x37, 0xd3, 0xf5, 0xe1, 0xf6, 0xf8, + 0x4e, 0x3e, 0xa2, 0x3c, 0xd8, 0x20, 0xfc, 0x15, 0x13, 0xcb, 0xa1, 0x2e, 0x61, 0xec, 0x9e, 0xc5, + 0x09, 0x7a, 0x28, 0xef, 0xbd, 0x62, 0x07, 0x56, 0xeb, 0xf2, 0x42, 0xe2, 0xde, 0x2b, 0x66, 0x0e, + 0xf5, 0xf8, 0xf4, 0x78, 0xf3, 0x57, 0x59, 0xb9, 0x82, 0xb2, 0x04, 0xbd, 0x6b, 0x40, 0x89, 0xba, + 0x94, 0xeb, 0xeb, 0x10, 0x56, 0x7e, 0x7a, 0xa2, 0x2e, 0x52, 0x2b, 0xaa, 0x3d, 0xa3, 0x26, 0x58, + 0x6a, 0xa4, 0xf4, 0xe3, 0x01, 0x3c, 0xc4, 0x65, 0x1d, 0x88, 0xd0, 0x33, 0x17, 0x84, 0x9e, 0xac, + 0x01, 0x11, 0x72, 0x02, 0x07, 0xbd, 0x06, 0x88, 0x91, 0xe0, 0x90, 0xda, 0x64, 0xdd, 0xb6, 0xbd, + 0x9e, 0xcb, 0xe5, 0x95, 0x4c, 0x78, 0x49, 0xa4, 0x43, 0xa1, 0x75, 0x4a, 0x02, 0x0f, 0x19, 0x25, + 0x4e, 0x78, 0xfa, 0x52, 0x07, 0xd2, 0x27, 0xbc, 0xd3, 0x17, 0x3b, 0x68, 0x0d, 0x8a, 0xe2, 0xb4, + 0xf7, 0x80, 0xf0, 0xb7, 0xbd, 0xe0, 0xa0, 0x5c, 0x5c, 0x35, 0xae, 0xe7, 0xe3, 0x7b, 0xa9, 0xfb, + 0x31, 0x0b, 0x27, 0xe5, 0xcc, 0x5f, 0xe4, 0xa0, 0xa0, 0xb3, 0x23, 0xaa, 0x42, 0xce, 0xdf, 0xb7, + 0x58, 0xe4, 0x0c, 0xcf, 0xea, 0x93, 0x9a, 0x20, 0x9e, 0x84, 0x9d, 0x81, 0xfc, 0x8f, 0x43, 0x39, + 0xf4, 0x76, 0xaa, 0xda, 0x66, 0x26, 0xba, 0x05, 0x49, 0x3a, 0xd8, 0x13, 0x8b, 0xed, 0x19, 0xef, + 0x4f, 0xd1, 0x35, 0x71, 0x7c, 0x75, 0x1a, 0x4d, 0x95, 0x25, 0x12, 0x67, 0x4f, 0xa7, 0xd1, 0xc4, + 0x21, 0x4f, 0x34, 0x2a, 0xf2, 0x0f, 0x2b, 0xcf, 0x4e, 0xd4, 0xa8, 0x48, 0xa5, 0xf1, 0x54, 0xe4, + 0x23, 0xc3, 0x4a, 0x37, 0xa2, 0xea, 0xa2, 0x54, 0xe6, 0xd6, 0x99, 0x0b, 0xc8, 0xad, 0x73, 0xfa, + 0x92, 0x54, 0xa6, 0xd3, 0x58, 0x3b, 0xfa, 0xa9, 0x01, 0x0b, 0x76, 0xfa, 0x92, 0x94, 0xb0, 0x72, + 0x7e, 0xa2, 0x9b, 0xb1, 0x81, 0x4b, 0x57, 0xed, 0x1c, 0x0b, 0xf5, 0x41, 0x20, 0x7c, 0x1a, 0x1b, + 0xdd, 0x81, 0xfc, 0xb7, 0x3c, 0x56, 0xef, 0x58, 0x8c, 0x95, 0x0b, 0xa9, 0x03, 0x49, 0xfe, 0xab, + 0x0f, 0x5b, 0x92, 0x7e, 0x72, 0xb4, 0x52, 0x6c, 0x7a, 0x4e, 0xf4, 0x88, 0xf5, 0x00, 0xf3, 0x47, + 0x06, 0x40, 0x7c, 0x81, 0xa0, 0x2f, 0x24, 0x8d, 0x33, 0x5d, 0x48, 0x66, 0xce, 0x75, 0x21, 0xb9, + 0x02, 0x39, 0x12, 0x04, 0x5e, 0xa0, 0x5a, 0xdc, 0x82, 0xf0, 0x95, 0x4d, 0x41, 0xc0, 0x21, 0xdd, + 0xfc, 0xfd, 0x14, 0x4c, 0xb7, 0x88, 0x1d, 0x10, 0x7e, 0x29, 0x3d, 0xd7, 0x67, 0xa1, 0x40, 0xbb, + 0xdd, 0x1e, 0xb7, 0x76, 0x3b, 0x44, 0xba, 0x7e, 0x3e, 0x74, 0x83, 0x46, 0x44, 0xc4, 0x31, 0x1f, + 0x05, 0x30, 0x25, 0x27, 0x17, 0xc6, 0xe5, 0xbd, 0x31, 0x37, 0x3e, 0xb4, 0xb6, 0xb2, 0x61, 0x71, + 0x6b, 0xd3, 0xe5, 0x41, 0x5f, 0x37, 0x15, 0x53, 0x82, 0xf4, 0xe3, 0x3f, 0xaf, 0xe4, 0x6a, 0x7d, + 0x4e, 0x18, 0x96, 0x58, 0xe8, 0x3d, 0x03, 0x80, 0xf1, 0x80, 0xba, 0x6d, 0xc1, 0x55, 0x0d, 0xf8, + 0xf6, 0x64, 0xd0, 0x2d, 0xad, 0x2f, 0x9c, 0x80, 0x5e, 0xa2, 0x98, 0x81, 0x13, 0xa0, 0xa8, 0xa2, + 0x7a, 0xb7, 0x6c, 0x2a, 0xef, 0x46, 0xbd, 0x1b, 0x84, 0x5a, 0xe3, 0xae, 0x6d, 0xe9, 0xcb, 0x50, + 0xd0, 0xca, 0xd1, 0x53, 0x90, 0x3d, 0x20, 0xfd, 0x30, 0x03, 0x62, 0xf1, 0x17, 0x3d, 0x0d, 0xb9, + 0x43, 0xab, 0xd3, 0x0b, 0xef, 0xd9, 0x66, 0x71, 0xf8, 0x70, 0x3b, 0x73, 0xcb, 0x58, 0x7a, 0x19, + 0xe6, 0x07, 0xe6, 0xf6, 0xa4, 0xe1, 0x85, 0xc4, 0x70, 0xf3, 0x63, 0x03, 0xd4, 0x64, 0x2e, 0xa5, + 0xef, 0xd8, 0x4d, 0xf7, 0x1d, 0x2f, 0x4f, 0xb4, 0x49, 0x23, 0x5a, 0x8f, 0x3f, 0x64, 0x60, 0x46, + 0xd5, 0xbb, 0x4b, 0x89, 0x17, 0x27, 0x75, 0x46, 0xa9, 0x8d, 0x6d, 0xa2, 0xb4, 0x60, 0xe4, 0x39, + 0xa5, 0x33, 0x70, 0x4e, 0xd9, 0x98, 0x10, 0xe7, 0xf1, 0x67, 0x95, 0x63, 0x03, 0x8a, 0x4a, 0xf2, + 0x52, 0xfc, 0xc6, 0x4e, 0xfb, 0xcd, 0xdd, 0xc9, 0x8c, 0x1d, 0xe1, 0x38, 0x7f, 0xcb, 0x68, 0x23, + 0xcf, 0xf8, 0x9e, 0x6a, 0xfc, 0xa4, 0x3f, 0xf0, 0x16, 0x6a, 0xfa, 0xc9, 0x6f, 0xa1, 0x74, 0x0d, + 0xca, 0x8e, 0xac, 0x41, 0xdf, 0x06, 0xe0, 0x56, 0xd0, 0x26, 0xe1, 0x8b, 0xe8, 0xf0, 0x2c, 0x75, + 0xef, 0xbc, 0xfb, 0xd2, 0xe3, 0xb4, 0x53, 0x09, 0xbf, 0x9f, 0xa8, 0x34, 0x5c, 0xfe, 0x30, 0x08, + 0xd3, 0x4d, 0xec, 0xf4, 0x3b, 0x1a, 0x02, 0x27, 0xe0, 0xa2, 0xde, 0x51, 0x42, 0xe7, 0xd2, 0xef, + 0xc0, 0x1f, 0x28, 0x3a, 0xd6, 0x12, 0xe6, 0x07, 0x45, 0xbd, 0xd6, 0xb2, 0x7d, 0x6f, 0x47, 0x77, + 0xfe, 0xc6, 0x44, 0xb7, 0x18, 0x89, 0xed, 0x1b, 0xf1, 0x71, 0xc1, 0x77, 0x21, 0xcf, 0x48, 0x87, + 0xd8, 0xdc, 0x0b, 0x94, 0x33, 0x35, 0x27, 0x8f, 0xd0, 0x4a, 0x4b, 0xa9, 0x0c, 0x8b, 0x85, 0x36, + 0x3c, 0x22, 0x63, 0x8d, 0x89, 0xaa, 0x50, 0xb0, 0x3b, 0x3d, 0xc6, 0x49, 0xd0, 0x68, 0xaa, 0x6a, + 0xa1, 0xaf, 0x5b, 0xea, 0x11, 0x03, 0xc7, 0x32, 0xa8, 0x02, 0xa0, 0x1f, 0x58, 0x19, 0xc9, 0x2b, + 0xb3, 0x92, 0x6c, 0x53, 0x35, 0x15, 0x27, 0x24, 0x50, 0x55, 0x55, 0xa2, 0xf0, 0xc5, 0xe6, 0xff, + 0x0d, 0x54, 0xa2, 0x68, 0xd1, 0x13, 0x17, 0x08, 0x37, 0xa0, 0x48, 0xde, 0xe1, 0x24, 0x70, 0xad, + 0x8e, 0x40, 0xc8, 0x49, 0x04, 0xe9, 0x8a, 0x9b, 0x31, 0x19, 0x27, 0x65, 0xd0, 0x0e, 0xcc, 0x33, + 0xc2, 0x18, 0xf5, 0xdc, 0xf5, 0xbd, 0x3d, 0x71, 0x0a, 0xea, 0xcb, 0xee, 0xb2, 0x50, 0xfb, 0x8c, + 0x82, 0x9b, 0x6f, 0xa5, 0xd9, 0x27, 0x92, 0x14, 0x9e, 0x37, 0x14, 0x09, 0x0f, 0xaa, 0x40, 0x77, + 0xa1, 0xd4, 0x49, 0xbd, 0x15, 0x52, 0xa7, 0x18, 0x7d, 0xfe, 0x4a, 0xbf, 0x33, 0xc2, 0x03, 0xd2, + 0xe8, 0x6b, 0x50, 0x4e, 0x52, 0x5a, 0x5e, 0x2f, 0xb0, 0x09, 0xb6, 0xdc, 0x36, 0x09, 0xbf, 0x21, + 0x29, 0xd4, 0x9e, 0x3b, 0x3e, 0x5a, 0x29, 0x6f, 0x8d, 0x90, 0xc1, 0x23, 0x47, 0x23, 0x06, 0x8b, + 0x91, 0xf9, 0x3b, 0x81, 0xb5, 0xb7, 0x47, 0xed, 0xa6, 0xd7, 0xa1, 0x76, 0x5f, 0x9e, 0x79, 0x0a, + 0xb5, 0x97, 0xd5, 0x04, 0x17, 0x37, 0x87, 0x09, 0x9d, 0x1c, 0xad, 0x3c, 0xa7, 0x6c, 0x1f, 0xca, + 0xc7, 0xc3, 0x75, 0xa3, 0x6d, 0xb8, 0xba, 0x4f, 0xac, 0x0e, 0xdf, 0xaf, 0xef, 0x13, 0xfb, 0x20, + 0x8a, 0xa1, 0xf2, 0xac, 0x8c, 0xad, 0x68, 0x5f, 0xaf, 0xde, 0x3f, 0x2d, 0x82, 0x87, 0x8d, 0x43, + 0x3f, 0x37, 0x60, 0x71, 0x60, 0xc5, 0xc3, 0x6f, 0x8d, 0xca, 0xa5, 0x89, 0x3e, 0xe9, 0x68, 0x0d, + 0xd3, 0x59, 0x7b, 0x56, 0x2c, 0xc7, 0x50, 0x16, 0x1e, 0x3e, 0x0b, 0x74, 0x1b, 0x80, 0xfa, 0xaf, + 0x5a, 0x5d, 0xda, 0xa1, 0x84, 0x95, 0xaf, 0xca, 0xfd, 0x5a, 0x12, 0x7e, 0xde, 0x68, 0x46, 0x54, + 0x91, 0x4b, 0xd5, 0x53, 0x1f, 0x27, 0xa4, 0xd1, 0x16, 0x94, 0xd4, 0x53, 0x5f, 0x6d, 0xcc, 0x82, + 0xdc, 0x98, 0x4f, 0xc9, 0x53, 0x7b, 0x33, 0xc9, 0x39, 0x39, 0x45, 0xc1, 0x03, 0x63, 0x51, 0x1d, + 0x16, 0x92, 0x9e, 0x10, 0x9e, 0x20, 0x16, 0xa5, 0xc2, 0x45, 0x71, 0xfa, 0xd8, 0x1a, 0x64, 0xe2, + 0xd3, 0xf2, 0xc8, 0x83, 0x45, 0xea, 0x0e, 0x73, 0x99, 0x67, 0xa4, 0xa2, 0x17, 0xc5, 0xfa, 0x34, + 0xdc, 0xc7, 0xbb, 0xcb, 0x50, 0x3e, 0x1e, 0xae, 0x77, 0xe9, 0x0e, 0xcc, 0xa5, 0xb2, 0xd0, 0xb9, + 0xda, 0xc2, 0x0f, 0x32, 0x62, 0x74, 0xa2, 0x13, 0x40, 0xdf, 0x37, 0x60, 0x36, 0x69, 0x95, 0x2a, + 0xf3, 0x8d, 0x0b, 0x78, 0x17, 0xaa, 0x7a, 0x0d, 0xfd, 0x31, 0x56, 0x92, 0x87, 0x53, 0xa0, 0xa8, + 0x37, 0xe4, 0xb0, 0xbf, 0x3e, 0x6e, 0xa7, 0x71, 0xe6, 0xa3, 0xbe, 0xf9, 0xbe, 0x01, 0xc3, 0x9d, + 0x17, 0x79, 0x90, 0xb7, 0xd5, 0x97, 0x7a, 0x6a, 0x45, 0xc6, 0xfe, 0xf4, 0x27, 0xf5, 0xc1, 0x5f, + 0xf8, 0x16, 0x24, 0xa2, 0x61, 0x0d, 0x62, 0xfe, 0xdd, 0x80, 0x9c, 0x7c, 0xfd, 0x80, 0x9e, 0x4f, + 0xec, 0x67, 0xad, 0xa8, 0x2c, 0xc8, 0xbe, 0x4e, 0xfa, 0xe1, 0xe6, 0x5e, 0x4b, 0x6d, 0x6e, 0x5c, + 0xfd, 0xde, 0x10, 0x44, 0xb5, 0xd7, 0x68, 0x0d, 0xa6, 0xc9, 0xde, 0x1e, 0xb1, 0xb9, 0x2a, 0x3d, + 0xcf, 0x47, 0xfd, 0xde, 0xa6, 0xa4, 0x8a, 0x02, 0x21, 0xc1, 0xc2, 0x47, 0xac, 0x84, 0x11, 0x85, + 0x02, 0xa7, 0x5d, 0xb2, 0xee, 0x38, 0xc4, 0xb9, 0x90, 0x3b, 0x5a, 0x79, 0x80, 0xdc, 0x89, 0x54, + 0xe2, 0x58, 0xbb, 0x38, 0x78, 0x3f, 0x2b, 0x8a, 0x93, 0xb3, 0xe5, 0xd9, 0x56, 0x27, 0x6c, 0xb0, + 0x31, 0xd9, 0x23, 0x01, 0x71, 0x6d, 0xf9, 0xd5, 0x91, 0xe5, 0xd3, 0x7b, 0x81, 0xd7, 0x8b, 0x6e, + 0x55, 0xe5, 0xba, 0xad, 0x37, 0x1b, 0x92, 0x86, 0x35, 0x57, 0x74, 0x4b, 0x07, 0xd4, 0x75, 0xd4, + 0x6a, 0xe8, 0x6e, 0xe9, 0x75, 0xea, 0x3a, 0x58, 0x72, 0x74, 0x7b, 0x97, 0x1d, 0xd5, 0xde, 0x99, + 0x77, 0xa1, 0x98, 0xf8, 0x50, 0x50, 0x94, 0xee, 0xae, 0xf8, 0xd3, 0xb4, 0xf8, 0xfe, 0x60, 0xe9, + 0xde, 0x8e, 0x18, 0x38, 0x96, 0xa9, 0x7d, 0xe3, 0xd1, 0x27, 0xcb, 0x57, 0x3e, 0xfc, 0x64, 0xf9, + 0xca, 0x47, 0x9f, 0x2c, 0x5f, 0x79, 0xf7, 0x78, 0xd9, 0x78, 0x74, 0xbc, 0x6c, 0x7c, 0x78, 0xbc, + 0x6c, 0x7c, 0x74, 0xbc, 0x6c, 0x7c, 0x7c, 0xbc, 0x6c, 0xfc, 0xec, 0x2f, 0xcb, 0x57, 0xbe, 0xbe, + 0x36, 0xd6, 0x67, 0xba, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xed, 0x0f, 0xf6, 0x69, 0xde, 0x2b, + 0x00, 0x00, } func (m *ClientIPConfig) Marshal() (dAtA []byte, err error) { @@ -2905,6 +2914,16 @@ func (m *ServicePort) MarshalToSizedBuffer(dAtA []byte) (int, error) { i = encodeVarintGenerated(dAtA, i, uint64(m.NodePort)) i-- dAtA[i] = 0x28 + { + size, err := m.TargetPort.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 i = encodeVarintGenerated(dAtA, i, uint64(m.Port)) i-- dAtA[i] = 0x18 @@ -3884,6 +3903,8 @@ func (m *ServicePort) Size() (n int) { l = len(m.Protocol) n += 1 + l + sovGenerated(uint64(l)) n += 1 + sovGenerated(uint64(m.Port)) + l = m.TargetPort.Size() + n += 1 + l + sovGenerated(uint64(l)) n += 1 + sovGenerated(uint64(m.NodePort)) return n } @@ -4554,6 +4575,7 @@ func (this *ServicePort) String() string { `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `Protocol:` + fmt.Sprintf("%v", this.Protocol) + `,`, `Port:` + fmt.Sprintf("%v", this.Port) + `,`, + `TargetPort:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.TargetPort), "IntOrString", "intstr.IntOrString", 1), `&`, ``, 1) + `,`, `NodePort:` + fmt.Sprintf("%v", this.NodePort) + `,`, `}`, }, "") @@ -9374,6 +9396,39 @@ func (m *ServicePort) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TargetPort", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.TargetPort.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field NodePort", wireType) diff --git a/pkg/k8s/slim/k8s/api/core/v1/generated.proto b/pkg/k8s/slim/k8s/api/core/v1/generated.proto index 719483287ea09..dd2375e54d522 100644 --- a/pkg/k8s/slim/k8s/api/core/v1/generated.proto +++ b/pkg/k8s/slim/k8s/api/core/v1/generated.proto @@ -9,6 +9,7 @@ syntax = "proto2"; package github.com.cilium.cilium.pkg.k8s.slim.k8s.api.core.v1; import "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1/generated.proto"; +import "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/util/intstr/generated.proto"; import "k8s.io/apimachinery/pkg/runtime/generated.proto"; import "k8s.io/apimachinery/pkg/runtime/schema/generated.proto"; @@ -674,6 +675,17 @@ message ServicePort { // The port that will be exposed by this service. optional int32 port = 3; + // Number or name of the port to access on the pods targeted by the service. + // Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + // If this is a string, it will be looked up as a named port in the + // target Pod's container ports. If this is not specified, the value + // of the 'port' field is used (an identity map). + // This field is ignored for services with clusterIP=None, and should be + // omitted or set equal to the 'port' field. + // More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + // +optional + optional github.com.cilium.cilium.pkg.k8s.slim.k8s.apis.util.intstr.IntOrString targetPort = 4; + // The port on each node on which this service is exposed when type is // NodePort or LoadBalancer. Usually assigned by the system. If a value is // specified, in-range, and not in use it will be used, otherwise the diff --git a/pkg/k8s/slim/k8s/api/core/v1/types.go b/pkg/k8s/slim/k8s/api/core/v1/types.go index a4e6d69182ace..0071e3d9b20c1 100644 --- a/pkg/k8s/slim/k8s/api/core/v1/types.go +++ b/pkg/k8s/slim/k8s/api/core/v1/types.go @@ -7,6 +7,7 @@ package v1 import ( slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1" + "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/util/intstr" ) const ( @@ -832,6 +833,17 @@ type ServicePort struct { // The port that will be exposed by this service. Port int32 `json:"port" protobuf:"varint,3,opt,name=port"` + // Number or name of the port to access on the pods targeted by the service. + // Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + // If this is a string, it will be looked up as a named port in the + // target Pod's container ports. If this is not specified, the value + // of the 'port' field is used (an identity map). + // This field is ignored for services with clusterIP=None, and should be + // omitted or set equal to the 'port' field. + // More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + // +optional + TargetPort intstr.IntOrString `json:"targetPort,omitempty" protobuf:"bytes,4,opt,name=targetPort"` + // The port on each node on which this service is exposed when type is // NodePort or LoadBalancer. Usually assigned by the system. If a value is // specified, in-range, and not in use it will be used, otherwise the diff --git a/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepcopy.go b/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepcopy.go index 62e4787571811..5a24427aeaaac 100644 --- a/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepcopy.go +++ b/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepcopy.go @@ -869,6 +869,7 @@ func (in *ServiceList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServicePort) DeepCopyInto(out *ServicePort) { *out = *in + out.TargetPort = in.TargetPort return } diff --git a/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepequal.go b/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepequal.go index ce6e352f0ee71..0f7e8ad98a7a4 100644 --- a/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepequal.go +++ b/pkg/k8s/slim/k8s/api/core/v1/zz_generated.deepequal.go @@ -1082,6 +1082,10 @@ func (in *ServicePort) DeepEqual(other *ServicePort) bool { if in.Port != other.Port { return false } + if in.TargetPort != other.TargetPort { + return false + } + if in.NodePort != other.NodePort { return false } diff --git a/pkg/k8s/slim/k8s/apis/util/intstr/intstr.go b/pkg/k8s/slim/k8s/apis/util/intstr/intstr.go index 529d5a903f4c5..b3dbc8480c2e2 100644 --- a/pkg/k8s/slim/k8s/apis/util/intstr/intstr.go +++ b/pkg/k8s/slim/k8s/apis/util/intstr/intstr.go @@ -8,7 +8,11 @@ package intstr import ( "encoding/json" "fmt" + math "math" + "runtime/debug" "strconv" + + "k8s.io/klog" ) // IntOrString is a type that can hold an int32 or a string. When used in @@ -34,6 +38,37 @@ const ( String // The IntOrString holds a string. ) +// FromInt creates an IntOrString object with an int32 value. It is +// your responsibility not to call this method with a value greater +// than int32. +// Deprecated: use FromInt32 instead. +func FromInt(val int) IntOrString { + if val > math.MaxInt32 || val < math.MinInt32 { + klog.Errorf("value: %d overflows int32\n%s\n", val, debug.Stack()) + } + return IntOrString{Type: Int, IntVal: int32(val)} +} + +// FromInt32 creates an IntOrString object with an int32 value. +func FromInt32(val int32) IntOrString { + return IntOrString{Type: Int, IntVal: val} +} + +// FromString creates an IntOrString object with a string value. +func FromString(val string) IntOrString { + return IntOrString{Type: String, StrVal: val} +} + +// Parse the given string and try to convert it to an int32 integer before +// setting it as a string value. +func Parse(val string) IntOrString { + i, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return FromString(val) + } + return FromInt32(int32(i)) +} + // UnmarshalJSON implements the json.Unmarshaller interface. func (intstr *IntOrString) UnmarshalJSON(value []byte) error { if value[0] == '"' { diff --git a/pkg/k8s/watchers/watcher.go b/pkg/k8s/watchers/watcher.go index 0a0b3935e843c..ad54f984fb476 100644 --- a/pkg/k8s/watchers/watcher.go +++ b/pkg/k8s/watchers/watcher.go @@ -723,53 +723,60 @@ func (k *K8sWatcher) delK8sSVCs(svc k8s.ServiceID, svcInfo *k8s.Service, se *k8s logfields.K8sNamespace: svc.Namespace, }) - repPorts := svcInfo.UniquePorts() + for _, checkProtocol := range []bool{true, false} { + if !checkProtocol { + svcInfo = stripServiceProtocol(svcInfo) + } - frontends := []*loadbalancer.L3n4Addr{} + repPorts := svcInfo.UniquePorts() - for portName, svcPort := range svcInfo.Ports { - if !repPorts[svcPort.Port] { - continue - } - repPorts[svcPort.Port] = false + frontends := []*loadbalancer.L3n4Addr{} - for _, feIP := range svcInfo.FrontendIPs { - fe := loadbalancer.NewL3n4Addr(svcPort.Protocol, cmtypes.MustAddrClusterFromIP(feIP), svcPort.Port, loadbalancer.ScopeExternal) - frontends = append(frontends, fe) - } + for portName, svcPort := range svcInfo.Ports { + if !repPorts[svcPort.String()] { + continue + } + repPorts[svcPort.String()] = false - for _, nodePortFE := range svcInfo.NodePorts[portName] { - frontends = append(frontends, &nodePortFE.L3n4Addr) - if svcInfo.ExtTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal || svcInfo.IntTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal { - cpFE := nodePortFE.L3n4Addr.DeepCopy() - cpFE.Scope = loadbalancer.ScopeInternal - frontends = append(frontends, cpFE) + for _, feIP := range svcInfo.FrontendIPs { + fe := loadbalancer.NewL3n4Addr(svcPort.Protocol, cmtypes.MustAddrClusterFromIP(feIP), svcPort.Port, loadbalancer.ScopeExternal) + frontends = append(frontends, fe) } - } - for _, k8sExternalIP := range svcInfo.K8sExternalIPs { - frontends = append(frontends, loadbalancer.NewL3n4Addr(svcPort.Protocol, cmtypes.MustAddrClusterFromIP(k8sExternalIP), svcPort.Port, loadbalancer.ScopeExternal)) - } + for _, nodePortFE := range svcInfo.NodePorts[portName] { + frontends = append(frontends, &nodePortFE.L3n4Addr) + if svcInfo.ExtTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal || svcInfo.IntTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal { + cpFE := nodePortFE.L3n4Addr.DeepCopy() + cpFE.Scope = loadbalancer.ScopeInternal + frontends = append(frontends, cpFE) + } + } - for _, ip := range svcInfo.LoadBalancerIPs { - addrCluster := cmtypes.MustAddrClusterFromIP(ip) - frontends = append(frontends, loadbalancer.NewL3n4Addr(svcPort.Protocol, addrCluster, svcPort.Port, loadbalancer.ScopeExternal)) - if svcInfo.ExtTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal || svcInfo.IntTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal { - frontends = append(frontends, loadbalancer.NewL3n4Addr(svcPort.Protocol, addrCluster, svcPort.Port, loadbalancer.ScopeInternal)) + for _, k8sExternalIP := range svcInfo.K8sExternalIPs { + frontends = append(frontends, loadbalancer.NewL3n4Addr(svcPort.Protocol, cmtypes.MustAddrClusterFromIP(k8sExternalIP), svcPort.Port, loadbalancer.ScopeExternal)) + } + + for _, ip := range svcInfo.LoadBalancerIPs { + addrCluster := cmtypes.MustAddrClusterFromIP(ip) + frontends = append(frontends, loadbalancer.NewL3n4Addr(svcPort.Protocol, addrCluster, svcPort.Port, loadbalancer.ScopeExternal)) + if svcInfo.ExtTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal || svcInfo.IntTrafficPolicy == loadbalancer.SVCTrafficPolicyLocal { + frontends = append(frontends, loadbalancer.NewL3n4Addr(svcPort.Protocol, addrCluster, svcPort.Port, loadbalancer.ScopeInternal)) + } } } - } - for _, fe := range frontends { - if found, err := k.svcManager.DeleteService(*fe); err != nil { - scopedLog.WithError(err).WithField(logfields.Object, logfields.Repr(fe)). - Warn("Error deleting service by frontend") - } else if !found { - scopedLog.WithField(logfields.Object, logfields.Repr(fe)).Warn("service not found") - } else { - scopedLog.Debugf("# cilium lb delete-service %s %d 0", fe.AddrCluster.String(), fe.Port) + for _, fe := range frontends { + if found, err := k.svcManager.DeleteService(*fe); err != nil { + scopedLog.WithError(err).WithField(logfields.Object, logfields.Repr(fe)). + Warn("Error deleting service by frontend") + } else if !found { + scopedLog.WithField(logfields.Object, logfields.Repr(fe)).Warn("service not found") + } else { + scopedLog.Debugf("# cilium lb delete-service %s %d 0", fe.AddrCluster.String(), fe.Port) + } } } + return nil } @@ -866,10 +873,10 @@ func datapathSVCs(svc *k8s.Service, endpoints *k8s.Endpoints) (svcs []loadbalanc clusterIPPorts := map[loadbalancer.FEPortName]*loadbalancer.L4Addr{} for fePortName, fePort := range svc.Ports { - if !uniqPorts[fePort.Port] { + if !uniqPorts[fePort.String()] { continue } - uniqPorts[fePort.Port] = false + uniqPorts[fePort.String()] = false clusterIPPorts[fePortName] = fePort } @@ -930,6 +937,38 @@ func hashSVCMap(svcs []loadbalancer.SVC) map[string]loadbalancer.L3n4Addr { return m } +func stripServiceProtocol(svc *k8s.Service) *k8s.Service { + if svc == nil { + return nil + } + + svc = svc.DeepCopy() + + for _, port := range svc.Ports { + port.Protocol = "ANY" + } + + for _, nodePort := range svc.NodePorts { + for _, port := range nodePort { + port.Protocol = "ANY" + } + } + + return svc +} + +func stripEndpointsProtocol(endpoints *k8s.Endpoints) *k8s.Endpoints { + endpoints = endpoints.DeepCopy() + + for _, backend := range endpoints.Backends { + for _, port := range backend.Ports { + port.Protocol = "ANY" + } + } + + return endpoints +} + func (k *K8sWatcher) addK8sSVCs(svcID k8s.ServiceID, oldSvc, svc *k8s.Service, endpoints *k8s.Endpoints) error { // Headless services do not need any datapath implementation if svc.IsHeadless { @@ -941,6 +980,12 @@ func (k *K8sWatcher) addK8sSVCs(svcID k8s.ServiceID, oldSvc, svc *k8s.Service, e logfields.K8sNamespace: svcID.Namespace, }) + if !option.Config.LoadBalancerProtocolDifferentiation { + oldSvc = stripServiceProtocol(oldSvc) + svc = stripServiceProtocol(svc) + endpoints = stripEndpointsProtocol(endpoints) + } + svcs := datapathSVCs(svc, endpoints) svcMap := hashSVCMap(svcs) diff --git a/pkg/k8s/watchers/watcher_test.go b/pkg/k8s/watchers/watcher_test.go index 4ba92578bc065..6038039cb4448 100644 --- a/pkg/k8s/watchers/watcher_test.go +++ b/pkg/k8s/watchers/watcher_test.go @@ -16,6 +16,7 @@ import ( "github.com/cilium/cilium/pkg/k8s" slim_corev1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1" slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1" + "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/util/intstr" "github.com/cilium/cilium/pkg/labels" "github.com/cilium/cilium/pkg/loadbalancer" "github.com/cilium/cilium/pkg/lock" @@ -241,6 +242,8 @@ func (s *K8sWatcherSuite) TestUpdateToServiceEndpointsGH9525(c *C) { } func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { + option.Config.LoadBalancerProtocolDifferentiation = true + k8sSvc := &slim_corev1.Service{ ObjectMeta: slim_metav1.ObjectMeta{ Name: "foo", @@ -256,15 +259,12 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { Protocol: slim_corev1.ProtocolUDP, Port: 80, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // { - // Name: "port-tcp-80", - // Protocol: slim_corev1.ProtocolTCP, - // Port: 80, - // TargetPort: intstr.FromString("port-80-t"), - // }, + { + Name: "port-tcp-80", + Protocol: slim_corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromString("port-80-t"), + }, { Name: "port-tcp-81", Protocol: slim_corev1.ProtocolTCP, @@ -296,14 +296,11 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { Port: 8080, Protocol: slim_corev1.ProtocolUDP, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // { - // Name: "port-tcp-80", - // Protocol:slim_corev1.ProtocolTCP, - // Port: 8081, - // }, + { + Name: "port-tcp-80", + Protocol: slim_corev1.ProtocolTCP, + Port: 8081, + }, { Name: "port-tcp-81", Protocol: slim_corev1.ProtocolTCP, @@ -315,7 +312,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { } lb1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) - // lb2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) + lb2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) lb3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0) upsert1stWanted := map[string]loadbalancer.SVC{ lb1.Hash(): { @@ -335,24 +332,23 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // lb2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *lb2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + lb2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *lb2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, lb3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *lb3, @@ -407,33 +403,34 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // lb2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *lb2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + lb2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *lb2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, lb3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *lb3, @@ -466,7 +463,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { del1stWanted := map[string]struct{}{ lb1.Hash(): {}, - // lb2.Hash(): {}, + lb2.Hash(): {}, lb3.Hash(): {}, } @@ -511,6 +508,10 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ClusterIP(c *C) { return false, 0, nil }, OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) { + if fe.Protocol == loadbalancer.ANY { + return true, nil + } + del1st[fe.Hash()] = struct{}{} svcDeleteManagerCalls++ return true, nil @@ -706,6 +707,7 @@ func (s *K8sWatcherSuite) TestChangeSVCPort(c *C) { func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { enableNodePortBak := option.Config.EnableNodePort option.Config.EnableNodePort = true + option.Config.LoadBalancerProtocolDifferentiation = true defer func() { option.Config.EnableNodePort = enableNodePortBak }() @@ -726,16 +728,13 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { Port: 80, NodePort: 18080, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // { - // Name: "port-tcp-80", - // Protocol: slim_corev1.ProtocolTCP, - // Port: 80, - // TargetPort: intstr.FromString("port-80-t"), - // NodePort: 18080, - // }, + { + Name: "port-tcp-80", + Protocol: slim_corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromString("port-80-t"), + NodePort: 18080, + }, { Name: "port-tcp-81", Protocol: slim_corev1.ProtocolTCP, @@ -768,14 +767,11 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { Port: 8080, Protocol: slim_corev1.ProtocolUDP, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // { - // Name: "port-tcp-80", - // Protocol:slim_corev1.ProtocolTCP, - // Port: 8081, - // }, + { + Name: "port-tcp-80", + Protocol: slim_corev1.ProtocolTCP, + Port: 8081, + }, { Name: "port-tcp-81", Protocol: slim_corev1.ProtocolTCP, @@ -787,7 +783,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { } clusterIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) - // clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) + clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) clusterIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0) upsert1stWanted := map[string]loadbalancer.SVC{ @@ -808,24 +804,23 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // clusterIP2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *clusterIP2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + clusterIP2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *clusterIP2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, clusterIP3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *clusterIP3, @@ -872,28 +867,30 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { }, } } - // nodePortIPs2 := []*loadbalancer.L3n4AddrID{ - // loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0), - // loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.Pv4NodePortAddress, 18080, loadbalancer.ScopeExternal, 0), - // loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.IPv4InternalAddress, 18080, loadbalancer.ScopeExternal, 0), - // } - // for _, nodePort := range nodePortIPs2 { - // upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeNodePort, - // Frontend: *nodePort, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + nodePortIPs2 := []*loadbalancer.L3n4AddrID{ + loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0), + loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18080, loadbalancer.ScopeExternal, 0), + loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4InternalAddrCluster, 18080, loadbalancer.ScopeExternal, 0), + } + for _, nodePort := range nodePortIPs2 { + upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeNodePort, + Frontend: *nodePort, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } nodePortIPs3 := []*loadbalancer.L3n4AddrID{ loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18081, loadbalancer.ScopeExternal, 0), loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18081, loadbalancer.ScopeExternal, 0), @@ -954,33 +951,34 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // clusterIP2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *clusterIP2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + clusterIP2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *clusterIP2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, clusterIP3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *clusterIP3, @@ -1041,32 +1039,36 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { }, } } - // for _, nodePort := range nodePortIPs2 { - // upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeNodePort, - // Frontend: *nodePort, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + for _, nodePort := range nodePortIPs2 { + upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeNodePort, + Frontend: *nodePort, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } for _, nodePort := range nodePortIPs3 { upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{ Type: loadbalancer.SVCTypeNodePort, @@ -1100,10 +1102,10 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { del1stWanted := map[string]struct{}{ clusterIP1.Hash(): {}, - // clusterIP2.Hash(): {}, + clusterIP2.Hash(): {}, clusterIP3.Hash(): {}, } - for _, nodePort := range append(nodePortIPs1, nodePortIPs3...) { + for _, nodePort := range append(nodePortIPs1, append(nodePortIPs2, nodePortIPs3...)...) { del1stWanted[nodePort.Hash()] = struct{}{} } @@ -1148,6 +1150,10 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_NodePort(c *C) { return false, 0, nil }, OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) { + if fe.Protocol == loadbalancer.ANY { + return true, nil + } + del1st[fe.Hash()] = struct{}{} svcDeleteManagerCalls++ return true, nil @@ -1203,6 +1209,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_GH9576_1(c *C) { enableNodePortBak := option.Config.EnableNodePort option.Config.EnableNodePort = true + option.Config.LoadBalancerProtocolDifferentiation = true defer func() { option.Config.EnableNodePort = enableNodePortBak }() @@ -1516,6 +1523,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_GH9576_2(c *C) { enableNodePortBak := option.Config.EnableNodePort option.Config.EnableNodePort = true + option.Config.LoadBalancerProtocolDifferentiation = true defer func() { option.Config.EnableNodePort = enableNodePortBak }() @@ -1821,6 +1829,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_GH9576_2(c *C) { func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { enableNodePortBak := option.Config.EnableNodePort option.Config.EnableNodePort = true + option.Config.LoadBalancerProtocolDifferentiation = true defer func() { option.Config.EnableNodePort = enableNodePortBak }() @@ -1841,16 +1850,13 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { Port: 80, NodePort: 18080, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // { - // Name: "port-tcp-80", - // Protocol: slim_corev1.ProtocolTCP, - // Port: 80, - // TargetPort: intstr.FromString("port-80-t"), - // NodePort: 18080, - // }, + { + Name: "port-tcp-80", + Protocol: slim_corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromString("port-80-t"), + NodePort: 18080, + }, { Name: "port-tcp-81", Protocol: slim_corev1.ProtocolTCP, @@ -1886,14 +1892,11 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { Port: 8080, Protocol: slim_corev1.ProtocolUDP, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // { - // Name: "port-tcp-80", - // Protocol:slim_corev1.ProtocolTCP, - // Port: 8081, - // }, + { + Name: "port-tcp-80", + Protocol: slim_corev1.ProtocolTCP, + Port: 8081, + }, { Name: "port-tcp-81", Protocol: slim_corev1.ProtocolTCP, @@ -1911,7 +1914,7 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { ) clusterIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) - // clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) + clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0) clusterIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0) upsert1stWanted := map[string]loadbalancer.SVC{ @@ -1932,24 +1935,23 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // clusterIP2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *clusterIP2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + clusterIP2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *clusterIP2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, clusterIP3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *clusterIP3, @@ -1970,10 +1972,10 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { } externalIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("127.8.8.8"), 80, loadbalancer.ScopeExternal, 0) - // externalIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("127.8.8.8"), 80, loadbalancer.ScopeExternal, 0) + externalIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("127.8.8.8"), 80, loadbalancer.ScopeExternal, 0) externalIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("127.8.8.8"), 81, loadbalancer.ScopeExternal, 0) externalIP4 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("127.9.9.9"), 80, loadbalancer.ScopeExternal, 0) - // externalIP5 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("127.9.9.9"), 80, loadbalancer.ScopeExternal, 0) + externalIP5 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("127.9.9.9"), 80, loadbalancer.ScopeExternal, 0) externalIP6 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("127.9.9.9"), 81, loadbalancer.ScopeExternal, 0) for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP1, externalIP4} { upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{ @@ -1994,23 +1996,25 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, } } - // for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2, externalIP5} { - // upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeExternalIPs, - // Frontend: *externalIP, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.UDP, - // Port: 8080, - // }, - // }, - // }, - // }, - // } - // } + for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2, externalIP5} { + upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeExternalIPs, + Frontend: *externalIP, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP3, externalIP6} { upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{ Type: loadbalancer.SVCTypeExternalIPs, @@ -2058,28 +2062,30 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, } } - // nodePortIPs2 := []*loadbalancer.L3n4AddrID{ - // loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0), - // loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.IPv4NodePortAddress, 18080, loadbalancer.ScopeExternal, 0), - // loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.IPv4InternalAddress, 18080, loadbalancer.ScopeExternal, 0), - // } - // for _, nodePort := range nodePortIPs2 { - // upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeNodePort, - // Frontend: *nodePort, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + nodePortIPs2 := []*loadbalancer.L3n4AddrID{ + loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0), + loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18080, loadbalancer.ScopeExternal, 0), + loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4InternalAddrCluster, 18080, loadbalancer.ScopeExternal, 0), + } + for _, nodePort := range nodePortIPs2 { + upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeNodePort, + Frontend: *nodePort, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } nodePortIPs3 := []*loadbalancer.L3n4AddrID{ loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18081, loadbalancer.ScopeExternal, 0), loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18081, loadbalancer.ScopeExternal, 0), @@ -2134,33 +2140,34 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // clusterIP2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *clusterIP2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + clusterIP2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *clusterIP2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, clusterIP3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *clusterIP3, @@ -2221,32 +2228,36 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, } } - // for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2, externalIP5} { - // upsert2ndWanted[externalIP.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeExternalIPs, - // Frontend: *externalIP, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.UDP, - // Port: 8080, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2, externalIP5} { + upsert2ndWanted[externalIP.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeExternalIPs, + Frontend: *externalIP, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP3, externalIP6} { upsert2ndWanted[externalIP.Hash()] = loadbalancer.SVC{ Type: loadbalancer.SVCTypeExternalIPs, @@ -2308,32 +2319,36 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, } } - // for _, nodePort := range nodePortIPs2 { - // upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeNodePort, - // Frontend: *nodePort, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + for _, nodePort := range nodePortIPs2 { + upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeNodePort, + Frontend: *nodePort, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } for _, nodePort := range nodePortIPs3 { upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{ Type: loadbalancer.SVCTypeNodePort, @@ -2394,33 +2409,34 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, }, }, - // FIXME: We don't distinguish about the protocol being used - // so we can't tell if a UDP/80 maps to port 8080/udp - // or if TCP/80 maps to port 8081/TCP - // clusterIP2.Hash(): { - // Type: loadbalancer.SVCTypeClusterIP, - // Frontend: *clusterIP2, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // }, + clusterIP2.Hash(): { + Type: loadbalancer.SVCTypeClusterIP, + Frontend: *clusterIP2, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + }, clusterIP3.Hash(): { Type: loadbalancer.SVCTypeClusterIP, Frontend: *clusterIP3, @@ -2481,32 +2497,36 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, } } - // for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2} { - // upsert3rdWanted[externalIP.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeExternalIPs, - // Frontend: *externalIP, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2} { + upsert3rdWanted[externalIP.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeExternalIPs, + Frontend: *externalIP, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP3} { upsert3rdWanted[externalIP.Hash()] = loadbalancer.SVC{ Type: loadbalancer.SVCTypeExternalIPs, @@ -2568,32 +2588,36 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { }, } } - // for _, nodePort := range nodePortIPs2 { - // upsert3rdWanted[nodePort.Hash()] = loadbalancer.SVC{ - // Type: loadbalancer.SVCTypeNodePort, - // Frontend: *nodePort, - // Backends: []*loadbalancer.Backend{ - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.2"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // { - // L3n4Addr: loadbalancer.L3n4Addr{ - // IP: net.ParseIP("10.0.0.3"), - // L4Addr: loadbalancer.L4Addr{ - // Protocol: loadbalancer.TCP, - // Port: 8081, - // }, - // }, - // }, - // }, - // } - // } + for _, nodePort := range nodePortIPs2 { + upsert3rdWanted[nodePort.Hash()] = loadbalancer.SVC{ + Type: loadbalancer.SVCTypeNodePort, + Frontend: *nodePort, + Backends: []*loadbalancer.Backend{ + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + { + FEPortName: "port-tcp-80", + L3n4Addr: loadbalancer.L3n4Addr{ + AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"), + L4Addr: loadbalancer.L4Addr{ + Protocol: loadbalancer.TCP, + Port: 8081, + }, + }, + Weight: loadbalancer.DefaultBackendWeight, + }, + }, + } + } for _, nodePort := range nodePortIPs3 { upsert3rdWanted[nodePort.Hash()] = loadbalancer.SVC{ Type: loadbalancer.SVCTypeNodePort, @@ -2627,18 +2651,18 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { del1stWanted := map[string]struct{}{ externalIP4.Hash(): {}, - // externalIP5.Hash():{}, + externalIP5.Hash(): {}, externalIP6.Hash(): {}, } del2ndWanted := map[string]struct{}{ - clusterIP1.Hash(): {}, - // clusterIP2.Hash(): {}, + clusterIP1.Hash(): {}, + clusterIP2.Hash(): {}, clusterIP3.Hash(): {}, externalIP1.Hash(): {}, - // externalIP2.Hash():{}, + externalIP2.Hash(): {}, externalIP3.Hash(): {}, } - for _, nodePort := range append(nodePortIPs1, nodePortIPs3...) { + for _, nodePort := range append(nodePortIPs1, append(nodePortIPs2, nodePortIPs3...)...) { del2ndWanted[nodePort.Hash()] = struct{}{} } @@ -2692,6 +2716,10 @@ func (s *K8sWatcherSuite) Test_addK8sSVCs_ExternalIPs(c *C) { return false, 0, nil }, OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) { + if fe.Protocol == loadbalancer.ANY { + return true, nil + } + switch { // 1st update endpoints case svcDeleteManagerCalls < len(del1stWanted): diff --git a/pkg/loadbalancer/experimental/testdata/clusterip/expected.tables b/pkg/loadbalancer/experimental/testdata/clusterip/expected.tables new file mode 100644 index 0000000000000..e3c269a790e5e --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/clusterip/expected.tables @@ -0,0 +1,11 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +test/echo k8s Cluster Cluster 0 0 false + +--- Frontends --- +Address Type ServiceName PortName Backends Status +10.96.50.104:80/TCP ClusterIP test/echo http 10.244.1.1:80/TCP (active) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.1.1:80/TCP active test/echo (http) nodeport-worker 0 diff --git a/pkg/loadbalancer/experimental/testdata/dualstack/expected.tables b/pkg/loadbalancer/experimental/testdata/dualstack/expected.tables new file mode 100644 index 0000000000000..c10679355c989 --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/dualstack/expected.tables @@ -0,0 +1,25 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +default/echo-dualstack k8s Cluster Cluster 0 0 false + +--- Frontends --- +Address Type ServiceName PortName Backends Status +[::]:30181/TCP NodePort default/echo-dualstack http [fd00:10:244:1::247e]:80/TCP (active), [fd00:10:244:2::a314]:80/TCP (active) Done (??? ago) +[::]:32757/UDP NodePort default/echo-dualstack tftp [fd00:10:244:1::247e]:69/UDP (active), [fd00:10:244:2::a314]:69/UDP (active) Done (??? ago) +0.0.0.0:30181/TCP NodePort default/echo-dualstack http 10.244.1.54:80/TCP (active), 10.244.2.9:80/TCP (active) Done (??? ago) +0.0.0.0:32757/UDP NodePort default/echo-dualstack tftp 10.244.1.54:69/UDP (active), 10.244.2.9:69/UDP (active) Done (??? ago) +10.96.207.99:69/UDP ClusterIP default/echo-dualstack tftp 10.244.1.54:69/UDP (active), 10.244.2.9:69/UDP (active) Done (??? ago) +10.96.207.99:80/TCP ClusterIP default/echo-dualstack http 10.244.1.54:80/TCP (active), 10.244.2.9:80/TCP (active) Done (??? ago) +[fd00:10:96::b050]:69/UDP ClusterIP default/echo-dualstack tftp [fd00:10:244:1::247e]:69/UDP (active), [fd00:10:244:2::a314]:69/UDP (active) Done (??? ago) +[fd00:10:96::b050]:80/TCP ClusterIP default/echo-dualstack http [fd00:10:244:1::247e]:80/TCP (active), [fd00:10:244:2::a314]:80/TCP (active) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.1.54:69/UDP active default/echo-dualstack (tftp) dual-stack-worker 0 +10.244.1.54:80/TCP active default/echo-dualstack (http) dual-stack-worker 0 +10.244.2.9:69/UDP active default/echo-dualstack (tftp) dual-stack-worker2 0 +10.244.2.9:80/TCP active default/echo-dualstack (http) dual-stack-worker2 0 +[fd00:10:244:1::247e]:69/UDP active default/echo-dualstack (tftp) dual-stack-worker 0 +[fd00:10:244:1::247e]:80/TCP active default/echo-dualstack (http) dual-stack-worker 0 +[fd00:10:244:2::a314]:69/UDP active default/echo-dualstack (tftp) dual-stack-worker2 0 +[fd00:10:244:2::a314]:80/TCP active default/echo-dualstack (http) dual-stack-worker2 0 diff --git a/pkg/loadbalancer/experimental/testdata/graceful-termination/expected.tables b/pkg/loadbalancer/experimental/testdata/graceful-termination/expected.tables new file mode 100644 index 0000000000000..f890a32a8b91d --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/graceful-termination/expected.tables @@ -0,0 +1,11 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +test/graceful-term-svc k8s Cluster Cluster 0 0 false + +--- Frontends --- +Address Type ServiceName PortName Backends Status +10.96.116.33:8081/TCP ClusterIP test/graceful-term-svc 10.244.0.112:8081/TCP (terminating) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.0.112:8081/TCP terminating test/graceful-term-svc graceful-term-control-plane 0 diff --git a/pkg/loadbalancer/experimental/testdata/hostport-simple/expected.tables b/pkg/loadbalancer/experimental/testdata/hostport-simple/expected.tables new file mode 100644 index 0000000000000..9b391bf91b81f --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/hostport-simple/expected.tables @@ -0,0 +1,11 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +default/my-app-85f46c4bd9-nnk25/host-port/4444 k8s Cluster Cluster 0 0 false + +--- Frontends --- +Address Type ServiceName PortName Backends Status +0.0.0.0:4444/TCP HostPort default/my-app-85f46c4bd9-nnk25/host-port/4444 10.244.1.113:80/TCP (active) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.1.113:80/TCP active default/my-app-85f46c4bd9-nnk25/host-port/4444 0 diff --git a/pkg/loadbalancer/experimental/testdata/multiport/expected.tables b/pkg/loadbalancer/experimental/testdata/multiport/expected.tables new file mode 100644 index 0000000000000..6f51cf8699111 --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/multiport/expected.tables @@ -0,0 +1,13 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +test/echo k8s Cluster Cluster 0 0 false + +--- Frontends --- +Address Type ServiceName PortName Backends Status +10.96.50.104:80/TCP ClusterIP test/echo http 10.244.1.1:80/TCP (active) Done (??? ago) +10.96.50.104:443/TCP ClusterIP test/echo https 10.244.1.1:443/TCP (active) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.1.1:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.1:443/TCP active test/echo (https) nodeport-worker 0 diff --git a/pkg/loadbalancer/experimental/testdata/nodeport-simple/expected.tables b/pkg/loadbalancer/experimental/testdata/nodeport-simple/expected.tables new file mode 100644 index 0000000000000..4461b56e13506 --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/nodeport-simple/expected.tables @@ -0,0 +1,32 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +test/echo k8s Cluster Cluster 0 0 false +test/echo2 k8s Cluster Cluster 0 0 false + +--- Frontends --- +Address Type ServiceName PortName Backends Status +0.0.0.0:30781/TCP NodePort test/echo http 10.244.1.1:80/TCP (active), 10.244.1.2:80/TCP (active), 10.244.1.3:80/TCP (active), 10.244.1.4:80/TCP (active), 10.244.1.5:80/TCP (active), 10.244.1.6:80/TCP (active), 10.244.1.7:80/TCP (active), 10.244.1.8:80/TCP (active), 10.244.1.9:80/TCP (active) Done (??? ago) +0.0.0.0:30782/TCP NodePort test/echo2 http2 10.244.2.1:80/TCP (active), 10.244.2.2:80/TCP (active), 10.244.2.3:80/TCP (active), 10.244.2.4:80/TCP (active), 10.244.2.5:80/TCP (active), 10.244.2.6:80/TCP (active), 10.244.2.7:80/TCP (active), 10.244.2.8:80/TCP (active), 10.244.2.9:80/TCP (active) Done (??? ago) +10.96.50.104:80/TCP ClusterIP test/echo http 10.244.1.1:80/TCP (active), 10.244.1.2:80/TCP (active), 10.244.1.3:80/TCP (active), 10.244.1.4:80/TCP (active), 10.244.1.5:80/TCP (active), 10.244.1.6:80/TCP (active), 10.244.1.7:80/TCP (active), 10.244.1.8:80/TCP (active), 10.244.1.9:80/TCP (active) Done (??? ago) +10.96.50.105:80/TCP ClusterIP test/echo2 http2 10.244.2.1:80/TCP (active), 10.244.2.2:80/TCP (active), 10.244.2.3:80/TCP (active), 10.244.2.4:80/TCP (active), 10.244.2.5:80/TCP (active), 10.244.2.6:80/TCP (active), 10.244.2.7:80/TCP (active), 10.244.2.8:80/TCP (active), 10.244.2.9:80/TCP (active) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.1.1:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.2:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.3:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.4:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.5:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.6:80/TCP active test/echo (http) nodeport-worker 0 +10.244.1.7:80/TCP active test/echo (http) nodeport-worker2 0 +10.244.1.8:80/TCP active test/echo (http) nodeport-worker2 0 +10.244.1.9:80/TCP active test/echo (http) nodeport-worker2 0 +10.244.2.1:80/TCP active test/echo2 (http2) nodeport-worker 0 +10.244.2.2:80/TCP active test/echo2 (http2) nodeport-worker 0 +10.244.2.3:80/TCP active test/echo2 (http2) nodeport-worker 0 +10.244.2.4:80/TCP active test/echo2 (http2) nodeport-worker 0 +10.244.2.5:80/TCP active test/echo2 (http2) nodeport-worker 0 +10.244.2.6:80/TCP active test/echo2 (http2) nodeport-worker 0 +10.244.2.7:80/TCP active test/echo2 (http2) nodeport-worker2 0 +10.244.2.8:80/TCP active test/echo2 (http2) nodeport-worker2 0 +10.244.2.9:80/TCP active test/echo2 (http2) nodeport-worker2 0 diff --git a/pkg/loadbalancer/experimental/testdata/source-ranges/expected.tables b/pkg/loadbalancer/experimental/testdata/source-ranges/expected.tables new file mode 100644 index 0000000000000..6b7d11d7f875a --- /dev/null +++ b/pkg/loadbalancer/experimental/testdata/source-ranges/expected.tables @@ -0,0 +1,12 @@ +--- Services --- +Name Source NatPolicy ExtTrafficPolicy IntTrafficPolicy SessionAffinity L7ProxyPort HealthCheckNodePort LoopbackHostPort SourceRanges +test/echo k8s Cluster Cluster 0 0 false 10.0.0.0/8 + +--- Frontends --- +Address Type ServiceName PortName Backends Status +10.0.0.1:80/TCP ClusterIP test/echo http 10.244.1.1:80/TCP (active) Done (??? ago) +10.0.0.2:80/TCP LoadBalancer test/echo http 10.244.1.1:80/TCP (active) Done (??? ago) + +--- Backends --- +Address State Instances NodeName ZoneID +10.244.1.1:80/TCP active test/echo (http) nodeport-worker 0 diff --git a/pkg/loadbalancer/loadbalancer.go b/pkg/loadbalancer/loadbalancer.go index 92c4bd1260e3c..ead2ef528c1a0 100644 --- a/pkg/loadbalancer/loadbalancer.go +++ b/pkg/loadbalancer/loadbalancer.go @@ -13,6 +13,7 @@ import ( "github.com/cilium/cilium/pkg/cidr" cmtypes "github.com/cilium/cilium/pkg/clustermesh/types" "github.com/cilium/cilium/pkg/option" + "github.com/cilium/cilium/pkg/u8proto" ) // SVCType is a type of a service. @@ -229,7 +230,10 @@ func (s ServiceFlags) UInt16() uint16 { } const ( + // NONE type. NONE = L4Type("NONE") + // ANY type. + ANY = L4Type("ANY") // TCP type. TCP = L4Type("TCP") // UDP type. @@ -524,6 +528,10 @@ func IsValidBackendState(state string) bool { func NewL4Type(name string) (L4Type, error) { switch strings.ToLower(name) { + case "none": + return NONE, nil + case "any": + return ANY, nil case "tcp": return TCP, nil case "udp": @@ -535,6 +543,19 @@ func NewL4Type(name string) (L4Type, error) { } } +func NewL4TypeFromNumber(proto uint8) L4Type { + switch proto { + case 6: + return TCP + case 17: + return UDP + case 132: + return SCTP + default: + return ANY + } +} + // L4Addr is an abstraction for the backend port with a L4Type, usually tcp or udp, and // the Port number. // @@ -558,6 +579,22 @@ func NewL4Addr(protocol L4Type, number uint16) *L4Addr { return &L4Addr{Protocol: protocol, Port: number} } +// Equals returns true if both L4Addr are considered equal. +func (l *L4Addr) Equals(o *L4Addr) bool { + switch { + case (l == nil) != (o == nil): + return false + case (l == nil) && (o == nil): + return true + } + return l.Port == o.Port && l.Protocol == o.Protocol +} + +// String returns a string representation of an L4Addr +func (l *L4Addr) String() string { + return fmt.Sprintf("%d/%s", l.Port, l.Protocol) +} + // L3n4Addr is used to store, as an unique L3+L4 address in the KVStore. It also // includes the lookup scope for frontend addresses which is used in service // handling for externalTrafficPolicy=Local and internalTrafficPolicy=Local, @@ -659,8 +696,7 @@ func NewBackendFromBackendModel(base *models.BackendAddress) (*Backend, error) { return nil, fmt.Errorf("missing IP address") } - // FIXME: Should this be NONE ? - l4addr := NewL4Addr(NONE, base.Port) + l4addr := NewL4Addr(base.Protocol, base.Port) addrCluster, err := cmtypes.ParseAddrCluster(*base.IP) if err != nil { return nil, err @@ -693,8 +729,7 @@ func NewL3n4AddrFromBackendModel(base *models.BackendAddress) (*L3n4Addr, error) return nil, fmt.Errorf("missing IP address") } - // FIXME: Should this be NONE ? - l4addr := NewL4Addr(NONE, base.Port) + l4addr := NewL4Addr(base.Protocol, base.Port) addrCluster, err := cmtypes.ParseAddrCluster(*base.IP) if err != nil { return nil, err @@ -712,9 +747,10 @@ func (a *L3n4Addr) GetModel() *models.FrontendAddress { scope = models.FrontendAddressScopeInternal } return &models.FrontendAddress{ - IP: a.AddrCluster.String(), - Port: a.Port, - Scope: scope, + IP: a.AddrCluster.String(), + Protocol: a.Protocol, + Port: a.Port, + Scope: scope, } } @@ -727,6 +763,7 @@ func (b *Backend) GetBackendModel() *models.BackendAddress { stateStr, _ := b.State.String() return &models.BackendAddress{ IP: &addrClusterStr, + Protocol: b.Protocol, Port: b.Port, NodeName: b.NodeName, State: stateStr, @@ -735,17 +772,10 @@ func (b *Backend) GetBackendModel() *models.BackendAddress { } } -// String returns the L3n4Addr in the "IPv4:Port[/Scope]" format for IPv4 and -// "[IPv6]:Port[/Scope]" format for IPv6. +// String returns the L3n4Addr in the "IPv4:Port/Protocol[/Scope]" format for IPv4 and +// "[IPv6]:Port/Protocol[/Scope]" format for IPv6. func (a *L3n4Addr) String() string { - var scope string - if a.Scope == ScopeInternal { - scope = "/i" - } - if a.IsIPv6() { - return fmt.Sprintf("[%s]:%d%s", a.AddrCluster.String(), a.Port, scope) - } - return fmt.Sprintf("%s:%d%s", a.AddrCluster.String(), a.Port, scope) + return a.StringWithProtocol() } // StringWithProtocol returns the L3n4Addr in the "IPv4:Port/Protocol[/Scope]" @@ -763,8 +793,6 @@ func (a *L3n4Addr) StringWithProtocol() string { // StringID returns the L3n4Addr as string to be used for unique identification func (a *L3n4Addr) StringID() string { - // This does not include the protocol right now as the datapath does - // not include the protocol in the lookup of the service IP. return a.String() } @@ -772,14 +800,15 @@ func (a *L3n4Addr) StringID() string { // Note: the resulting string is meant to be used as a key for maps and is not // readable by a human eye when printed out. func (a L3n4Addr) Hash() string { - const lenProto = 0 // proto is omitted for now + const lenProto = 1 // proto is uint8 const lenScope = 1 // scope is uint8 which is an alias for byte const lenPort = 2 // port is uint16 which is 2 bytes b := make([]byte, cmtypes.AddrClusterLen+lenProto+lenScope+lenPort) ac20 := a.AddrCluster.As20() copy(b, ac20[:]) - // FIXME: add Protocol once we care about protocols + u8p, _ := u8proto.ParseProtocol(a.Protocol) + b[net.IPv6len] = byte(u8p) // scope is a uint8 which is an alias for byte so a cast is safe b[net.IPv6len+lenProto] = byte(a.Scope) // port is a uint16, so 2 bytes diff --git a/pkg/maps/lbmap/ipv4.go b/pkg/maps/lbmap/ipv4.go index 1821c2d0e5816..750a5b4c64231 100644 --- a/pkg/maps/lbmap/ipv4.go +++ b/pkg/maps/lbmap/ipv4.go @@ -250,6 +250,7 @@ func NewService4Key(ip net.IP, port uint16, proto u8proto.U8proto, scope uint8, func (k *Service4Key) String() string { kHost := k.ToHost().(*Service4Key) addr := net.JoinHostPort(kHost.Address.String(), fmt.Sprintf("%d", kHost.Port)) + addr += fmt.Sprintf("/%s", u8proto.U8proto(kHost.Proto).String()) if kHost.Scope == loadbalancer.ScopeInternal { addr += "/i" } @@ -268,6 +269,7 @@ func (k *Service4Key) SetScope(scope uint8) { k.Scope = scope } func (k *Service4Key) GetScope() uint8 { return k.Scope } func (k *Service4Key) GetAddress() net.IP { return k.Address.IP() } func (k *Service4Key) GetPort() uint16 { return k.Port } +func (k *Service4Key) GetProtocol() uint8 { return k.Proto } func (k *Service4Key) MapDelete() error { return k.Map().Delete(k.ToNetwork()) } func (k *Service4Key) RevNatValue() RevNatValue { @@ -413,8 +415,9 @@ func (b *Backend4Value) GetAddress() net.IP { return b.Address.IP() } func (b *Backend4Value) GetIPCluster() cmtypes.AddrCluster { return cmtypes.AddrClusterFrom(b.Address.Addr(), 0) } -func (b *Backend4Value) GetPort() uint16 { return b.Port } -func (b *Backend4Value) GetFlags() uint8 { return b.Flags } +func (b *Backend4Value) GetPort() uint16 { return b.Port } +func (b *Backend4Value) GetProtocol() uint8 { return uint8(b.Proto) } +func (b *Backend4Value) GetFlags() uint8 { return b.Flags } func (v *Backend4Value) ToNetwork() BackendValue { n := *v @@ -475,8 +478,9 @@ func (b *Backend4ValueV3) GetAddress() net.IP { return b.Address.IP() } func (b *Backend4ValueV3) GetIPCluster() cmtypes.AddrCluster { return cmtypes.AddrClusterFrom(b.Address.Addr(), uint32(b.ClusterID)) } -func (b *Backend4ValueV3) GetPort() uint16 { return b.Port } -func (b *Backend4ValueV3) GetFlags() uint8 { return b.Flags } +func (b *Backend4ValueV3) GetPort() uint16 { return b.Port } +func (b *Backend4ValueV3) GetProtocol() uint8 { return uint8(b.Proto) } +func (b *Backend4ValueV3) GetFlags() uint8 { return b.Flags } func (v *Backend4ValueV3) ToNetwork() BackendValue { n := *v diff --git a/pkg/maps/lbmap/ipv6.go b/pkg/maps/lbmap/ipv6.go index 340cf9acc36d0..55beb19a78344 100644 --- a/pkg/maps/lbmap/ipv6.go +++ b/pkg/maps/lbmap/ipv6.go @@ -147,9 +147,9 @@ func NewService6Key(ip net.IP, port uint16, proto u8proto.U8proto, scope uint8, func (k *Service6Key) String() string { kHost := k.ToHost().(*Service6Key) if kHost.Scope == loadbalancer.ScopeInternal { - return fmt.Sprintf("[%s]:%d/i (%d)", kHost.Address, kHost.Port, kHost.BackendSlot) + return fmt.Sprintf("[%s]:%d/%s/i (%d)", kHost.Address, kHost.Port, u8proto.U8proto(kHost.Proto).String(), kHost.BackendSlot) } else { - return fmt.Sprintf("[%s]:%d (%d)", kHost.Address, kHost.Port, kHost.BackendSlot) + return fmt.Sprintf("[%s]:%d/%s (%d)", kHost.Address, kHost.Port, u8proto.U8proto(kHost.Proto).String(), kHost.BackendSlot) } } @@ -164,6 +164,7 @@ func (k *Service6Key) SetScope(scope uint8) { k.Scope = scope } func (k *Service6Key) GetScope() uint8 { return k.Scope } func (k *Service6Key) GetAddress() net.IP { return k.Address.IP() } func (k *Service6Key) GetPort() uint16 { return k.Port } +func (k *Service6Key) GetProtocol() uint8 { return k.Proto } func (k *Service6Key) MapDelete() error { return k.Map().Delete(k.ToNetwork()) } func (k *Service6Key) RevNatValue() RevNatValue { @@ -307,8 +308,9 @@ func (b *Backend6Value) GetAddress() net.IP { return b.Address.IP() } func (b *Backend6Value) GetIPCluster() cmtypes.AddrCluster { return cmtypes.AddrClusterFrom(b.Address.Addr(), 0) } -func (b *Backend6Value) GetPort() uint16 { return b.Port } -func (b *Backend6Value) GetFlags() uint8 { return b.Flags } +func (b *Backend6Value) GetPort() uint16 { return b.Port } +func (b *Backend6Value) GetProtocol() uint8 { return uint8(b.Proto) } +func (b *Backend6Value) GetFlags() uint8 { return b.Flags } func (v *Backend6Value) ToNetwork() BackendValue { n := *v @@ -369,8 +371,9 @@ func (b *Backend6ValueV3) GetAddress() net.IP { return b.Address.IP() } func (b *Backend6ValueV3) GetIPCluster() cmtypes.AddrCluster { return cmtypes.AddrClusterFrom(b.Address.Addr(), uint32(b.ClusterID)) } -func (b *Backend6ValueV3) GetPort() uint16 { return b.Port } -func (b *Backend6ValueV3) GetFlags() uint8 { return b.Flags } +func (b *Backend6ValueV3) GetPort() uint16 { return b.Port } +func (b *Backend6ValueV3) GetProtocol() uint8 { return uint8(b.Proto) } +func (b *Backend6ValueV3) GetFlags() uint8 { return b.Flags } func (v *Backend6ValueV3) ToNetwork() BackendValue { n := *v diff --git a/pkg/maps/lbmap/lbmap.go b/pkg/maps/lbmap/lbmap.go index 08a76ba57cc18..5971747541de9 100644 --- a/pkg/maps/lbmap/lbmap.go +++ b/pkg/maps/lbmap/lbmap.go @@ -6,8 +6,6 @@ package lbmap import ( "errors" "fmt" - "net" - "strconv" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -76,10 +74,10 @@ func (lbmap *LBBPFMap) upsertServiceProto(p *datapathTypes.UpsertServiceParams, backendsOk := ipv6 || !ipv6 && p.NatPolicy != loadbalancer.SVCNatPolicyNat46 if ipv6 { - svcKey = NewService6Key(p.IP, p.Port, u8proto.ANY, p.Scope, 0) + svcKey = NewService6Key(p.IP, p.Port, u8proto.U8proto(p.Protocol), p.Scope, 0) svcVal = &Service6Value{} } else { - svcKey = NewService4Key(p.IP, p.Port, u8proto.ANY, p.Scope, 0) + svcKey = NewService4Key(p.IP, p.Port, u8proto.U8proto(p.Protocol), p.Scope, 0) svcVal = &Service4Value{} } @@ -200,11 +198,16 @@ func deleteServiceProto(svc loadbalancer.L3n4AddrID, backendCount int, useMaglev revNATKey RevNatKey ) + u8p, err := u8proto.ParseProtocol(svc.Protocol) + if err != nil { + return err + } + if ipv6 { - svcKey = NewService6Key(svc.AddrCluster.AsNetIP(), svc.Port, u8proto.ANY, svc.Scope, 0) + svcKey = NewService6Key(svc.AddrCluster.AsNetIP(), svc.Port, u8p, svc.Scope, 0) revNATKey = NewRevNat6Key(uint16(svc.ID)) } else { - svcKey = NewService4Key(svc.AddrCluster.AsNetIP(), svc.Port, u8proto.ANY, svc.Scope, 0) + svcKey = NewService4Key(svc.AddrCluster.AsNetIP(), svc.Port, u8p, svc.Scope, 0) revNATKey = NewRevNat4Key(uint16(svc.ID)) } @@ -454,11 +457,8 @@ func (*LBBPFMap) DumpServiceMaps() ([]*loadbalancer.SVC, []error) { if svcKey.GetBackendSlot() == 0 { // Build a cache of flags stored in the value of the master key to // map it later. - // FIXME proto is being ignored everywhere in the datapath. - addrStr := svcKey.GetAddress().String() - portStr := strconv.Itoa(int(svcKey.GetPort())) - flagsCache[net.JoinHostPort(addrStr, portStr)] = loadbalancer.ServiceFlags(svcValue.GetFlags()) + flagsCache[fe.String()] = loadbalancer.ServiceFlags(svcValue.GetFlags()) newSVCMap.addFE(fe) return } @@ -502,13 +502,11 @@ func (*LBBPFMap) DumpServiceMaps() ([]*loadbalancer.SVC, []error) { newSVCList := make([]*loadbalancer.SVC, 0, len(newSVCMap)) for hash := range newSVCMap { svc := newSVCMap[hash] - addrStr := svc.Frontend.AddrCluster.String() - portStr := strconv.Itoa(int(svc.Frontend.Port)) - host := net.JoinHostPort(addrStr, portStr) - svc.Type = flagsCache[host].SVCType() - svc.ExtTrafficPolicy = flagsCache[host].SVCExtTrafficPolicy() - svc.IntTrafficPolicy = flagsCache[host].SVCIntTrafficPolicy() - svc.NatPolicy = flagsCache[host].SVCNatPolicy(svc.Frontend.L3n4Addr) + key := svc.Frontend.String() + svc.Type = flagsCache[key].SVCType() + svc.ExtTrafficPolicy = flagsCache[key].SVCExtTrafficPolicy() + svc.IntTrafficPolicy = flagsCache[key].SVCIntTrafficPolicy() + svc.NatPolicy = flagsCache[key].SVCNatPolicy(svc.Frontend.L3n4Addr) newSVCList = append(newSVCList, &svc) } @@ -546,7 +544,7 @@ func (*LBBPFMap) DumpBackendMaps() ([]*loadbalancer.Backend, error) { ip := backendVal.GetAddress() addrCluster := cmtypes.MustAddrClusterFromIP(ip) port := backendVal.GetPort() - proto := loadbalancer.NONE + proto := loadbalancer.NewL4TypeFromNumber(backendVal.GetProtocol()) state := loadbalancer.GetBackendStateFromFlags(backendVal.GetFlags()) lbBackend := loadbalancer.NewBackendWithState(backendID, proto, addrCluster, port, state) lbBackends = append(lbBackends, lbBackend) @@ -611,11 +609,17 @@ func getBackend(backend *loadbalancer.Backend, ipv6 bool) (Backend, error) { return lbBackend, fmt.Errorf("invalid backend ID 0") } + u8p, err := u8proto.ParseProtocol(backend.Protocol) + if err != nil { + return nil, fmt.Errorf("unable to parse protocol lbBackend (%d, %s, %d, %s, %t): %w", + backend.ID, backend.AddrCluster.String(), backend.Port, backend.Protocol, ipv6, err) + } + if ipv6 { - lbBackend, err = NewBackend6V3(backend.ID, backend.AddrCluster, backend.Port, u8proto.ANY, + lbBackend, err = NewBackend6V3(backend.ID, backend.AddrCluster, backend.Port, u8p, backend.State) } else { - lbBackend, err = NewBackend4V3(backend.ID, backend.AddrCluster, backend.Port, u8proto.ANY, + lbBackend, err = NewBackend4V3(backend.ID, backend.AddrCluster, backend.Port, u8p, backend.State) } if err != nil { diff --git a/pkg/maps/lbmap/types.go b/pkg/maps/lbmap/types.go index 5adfe088b2e5b..3b8adab2a95ad 100644 --- a/pkg/maps/lbmap/types.go +++ b/pkg/maps/lbmap/types.go @@ -11,7 +11,7 @@ import ( "github.com/cilium/cilium/pkg/loadbalancer" ) -// ServiceKey is the interface describing protocol independent key for services map v2. +// ServiceKey is the interface describing key for services map v2. type ServiceKey interface { bpf.MapKey @@ -42,6 +42,9 @@ type ServiceKey interface { // Get frontend port GetPort() uint16 + // Get protocol + GetProtocol() uint8 + // Returns a RevNatValue matching a ServiceKey RevNatValue() RevNatValue @@ -113,7 +116,7 @@ type BackendKey interface { GetID() loadbalancer.BackendID } -// BackendValue is the interface describing protocol independent backend value. +// BackendValue is the interface describing backend value. type BackendValue interface { bpf.MapValue @@ -126,6 +129,9 @@ type BackendValue interface { // Get backend port GetPort() uint16 + // Get backend protocol + GetProtocol() uint8 + // Get backend flags GetFlags() uint8 @@ -177,7 +183,8 @@ type RevNatValue interface { func svcFrontend(svcKey ServiceKey, svcValue ServiceValue) *loadbalancer.L3n4AddrID { feIP := svcKey.GetAddress() feAddrCluster := cmtypes.MustAddrClusterFromIP(feIP) - feL3n4Addr := loadbalancer.NewL3n4Addr(loadbalancer.NONE, feAddrCluster, svcKey.GetPort(), svcKey.GetScope()) + p := loadbalancer.NewL4TypeFromNumber(svcKey.GetProtocol()) + feL3n4Addr := loadbalancer.NewL3n4Addr(p, feAddrCluster, svcKey.GetPort(), svcKey.GetScope()) feL3n4AddrID := &loadbalancer.L3n4AddrID{ L3n4Addr: *feL3n4Addr, ID: loadbalancer.ID(svcValue.GetRevNat()), @@ -189,7 +196,7 @@ func svcBackend(backendID loadbalancer.BackendID, backend BackendValue) *loadbal beIP := backend.GetAddress() beAddrCluster := cmtypes.MustAddrClusterFromIP(beIP) bePort := backend.GetPort() - beProto := loadbalancer.NONE + beProto := loadbalancer.NewL4TypeFromNumber(backend.GetProtocol()) beState := loadbalancer.GetBackendStateFromFlags(backend.GetFlags()) beBackend := loadbalancer.NewBackendWithState(backendID, beProto, beAddrCluster, bePort, beState) return beBackend diff --git a/pkg/option/config.go b/pkg/option/config.go index 2cf10d13e6ae8..384cdb862aa9e 100644 --- a/pkg/option/config.go +++ b/pkg/option/config.go @@ -284,6 +284,13 @@ const ( // Alias to NodePortAcceleration LoadBalancerAcceleration = "bpf-lb-acceleration" + // LoadBalancerExternalControlPlane switch skips connectivity to kube-apiserver + // which is relevant in lb-only mode + LoadBalancerExternalControlPlane = "bpf-lb-external-control-plane" + + // LoadBalancerProtocolDifferentiation enables support for service protocol differentiation (TCP, UDP, SCTP) + LoadBalancerProtocolDifferentiation = "bpf-lb-proto-diff" + // MaglevTableSize determines the size of the backend table per service MaglevTableSize = "bpf-lb-maglev-table-size" @@ -2018,6 +2025,13 @@ type DaemonConfig struct { LoadBalancerRSSv6CIDR string LoadBalancerRSSv6 net.IPNet + // LoadBalancerExternalControlPlane tells whether to not use kube-apiserver as + // its control plane in lb-only mode. + LoadBalancerExternalControlPlane bool + + // LoadBalancerProtocolDifferentiation enables support for service protocol differentiation (TCP, UDP, SCTP) + LoadBalancerProtocolDifferentiation bool + // EnablePMTUDiscovery indicates whether to send ICMP fragmentation-needed // replies to the client (when needed). EnablePMTUDiscovery bool @@ -3523,6 +3537,8 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) { // To support K8s NetworkPolicy c.EnableK8sNetworkPolicy = vp.GetBool(EnableK8sNetworkPolicy) + + c.LoadBalancerProtocolDifferentiation = vp.GetBool(LoadBalancerProtocolDifferentiation) } func (c *DaemonConfig) populateDevices(vp *viper.Viper) { diff --git a/pkg/service/id_local.go b/pkg/service/id_local.go index 02a5c10b8ef11..6e9c519659b6b 100644 --- a/pkg/service/id_local.go +++ b/pkg/service/id_local.go @@ -51,6 +51,7 @@ func NewIDAllocator(nextID uint32, maxID uint32) *IDAllocator { } } +// addID assumes the lock is held func (alloc *IDAllocator) addID(svc loadbalancer.L3n4Addr, id uint32) *loadbalancer.L3n4AddrID { svcID := newID(svc, id) alloc.entitiesID[id] = svcID diff --git a/pkg/service/id_test.go b/pkg/service/id_test.go index 78a2fc855ac92..fe27725f9d155 100644 --- a/pkg/service/id_test.go +++ b/pkg/service/id_test.go @@ -84,8 +84,6 @@ func (s *IDAllocTestSuite) TestServices(c *C) { c.Assert(err, Equals, nil) c.Assert(l3n4AddrID.ID, Equals, loadbalancer.ID(ffsIDu16+1)) - // l3n4Addr3 should have the same ID as l3n4Addr2 since we are omitting the - // protocol type. l3n4AddrID, err = AcquireID(l3n4Addr3, 0) c.Assert(err, Equals, nil) c.Assert(l3n4AddrID.ID, Equals, loadbalancer.ID(ffsIDu16+1)) diff --git a/pkg/service/service.go b/pkg/service/service.go index eaae4626eba91..46dd112c8df5c 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -8,7 +8,6 @@ import ( "net" "strconv" "sync/atomic" - "time" "github.com/sirupsen/logrus" "go.uber.org/multierr" @@ -32,6 +31,8 @@ import ( nodeTypes "github.com/cilium/cilium/pkg/node/types" "github.com/cilium/cilium/pkg/option" "github.com/cilium/cilium/pkg/service/healthserver" + "github.com/cilium/cilium/pkg/time" + "github.com/cilium/cilium/pkg/u8proto" ) const anyPort = "*" @@ -822,6 +823,7 @@ func (s *Service) UpdateBackendsState(backends []*lb.Backend) error { be.State = updatedB.State be.Preferred = updatedB.Preferred + nextService: for id, info := range s.svcByID { var p *datapathTypes.UpsertServiceParams for i, b := range info.backends { @@ -836,10 +838,18 @@ func (s *Service) UpdateBackendsState(backends []*lb.Backend) error { found := false if p, found = updateSvcs[id]; !found { + proto, err := u8proto.ParseProtocol(info.frontend.L4Addr.Protocol) + if err != nil { + e := fmt.Errorf("failed to parse service protocol for frontend %+v: %w", info.frontend, err) + errs = multierr.Append(errs, e) + continue nextService + } + p = &datapathTypes.UpsertServiceParams{ ID: uint16(id), IP: info.frontend.L3n4Addr.AddrCluster.AsNetIP(), Port: info.frontend.L3n4Addr.L4Addr.Port, + Protocol: byte(proto), PrevBackendsCount: len(info.backends), IPv6: info.frontend.IsIPv6(), Type: info.svcType, @@ -1145,8 +1155,36 @@ func (s *Service) createSVCInfoIfNotExist(p *lb.SVC) (*svcInfo, bool, bool, prevSessionAffinity := false prevLoadBalancerSourceRanges := []*cidr.CIDR{} + // when Cilium is upgraded to a version that supports service protocol differentiation, and such feature is + // enabled, we may end up in a situation where some existing services do not have the protocol set. + // + // As in such cases we want to preserve the existing services (in order to not break existing connections to + // those services), when trying to create a new one check first if an "old" service without the protocol + // already exists, by overwriting its protocol to NONE. + // If it doesn't then do a second lookup in the svcByHash map with the protocol set. + // + // Note that this logic can be removed once we stop supporting services without protocol. + proto := p.Frontend.L3n4Addr.L4Addr.Protocol + p.Frontend.L3n4Addr.L4Addr.Protocol = "NONE" + + backendProtos := []lb.L4Type{} + for _, backend := range p.Backends { + backendProtos = append(backendProtos, backend.L3n4Addr.L4Addr.Protocol) + backend.L3n4Addr.L4Addr.Protocol = "NONE" + } + hash := p.Frontend.Hash() svc, found := s.svcByHash[hash] + if !found { + p.Frontend.L3n4Addr.L4Addr.Protocol = proto + for i, backend := range p.Backends { + backend.L3n4Addr.L4Addr.Protocol = backendProtos[i] + } + + hash = p.Frontend.Hash() + svc, found = s.svcByHash[hash] + } + if !found { // Allocate service ID for the new service addrID, err := AcquireID(p.Frontend.L3n4Addr, uint32(p.Frontend.ID)) @@ -1360,11 +1398,16 @@ func (s *Service) upsertServiceIntoLBMaps(svc *svcInfo, isExtLocal, isIntLocal b } } svc.svcNatPolicy = natPolicy + protocol, err := u8proto.ParseProtocol(svc.frontend.L3n4Addr.L4Addr.Protocol) + if err != nil { + return err + } p := &datapathTypes.UpsertServiceParams{ ID: uint16(svc.frontend.ID), IP: svc.frontend.L3n4Addr.AddrCluster.AsNetIP(), Port: svc.frontend.L3n4Addr.L4Addr.Port, + Protocol: uint8(protocol), PreferredBackends: preferredBackends, ActiveBackends: activeBackends, NonActiveBackends: nonActiveBackends, diff --git a/pkg/testutils/mockmaps/lbmap.go b/pkg/testutils/mockmaps/lbmap.go index e2e8feeb1b06e..077a4d98e30c5 100644 --- a/pkg/testutils/mockmaps/lbmap.go +++ b/pkg/testutils/mockmaps/lbmap.go @@ -12,6 +12,7 @@ import ( "github.com/cilium/cilium/pkg/ip" lb "github.com/cilium/cilium/pkg/loadbalancer" "github.com/cilium/cilium/pkg/lock" + "github.com/cilium/cilium/pkg/u8proto" ) type LBMockMap struct { @@ -59,7 +60,11 @@ func (m *LBMockMap) UpsertService(p *datapathTypes.UpsertServiceParams) error { } svc, found := m.ServiceByID[p.ID] if !found { - frontend := lb.NewL3n4AddrID(lb.NONE, cmtypes.MustAddrClusterFromIP(p.IP), p.Port, p.Scope, lb.ID(p.ID)) + u8p, err := u8proto.FromNumber(p.Protocol) + if err != nil { + return err + } + frontend := lb.NewL3n4AddrID(u8p.String(), cmtypes.MustAddrClusterFromIP(p.IP), p.Port, p.Scope, lb.ID(p.ID)) svc = &lb.SVC{Frontend: *frontend} } else { if p.PrevBackendsCount != len(svc.Backends) { @@ -137,7 +142,7 @@ func (m *LBMockMap) UpdateBackendWithState(b *lb.Backend) error { if !found { return fmt.Errorf("update failed : backend %d doesn't exist", id) } - if b.ID != be.ID || b.Port != be.Port || !b.AddrCluster.Equal(be.AddrCluster) { + if b.ID != be.ID || b.Port != be.Port || b.Protocol != be.Protocol || !b.AddrCluster.Equal(be.AddrCluster) { return fmt.Errorf("backend in the map %+v doesn't match %+v: only backend"+ "state can be updated", be.String(), b.String()) } diff --git a/pkg/u8proto/u8proto.go b/pkg/u8proto/u8proto.go index 2df035a044173..988d11e077266 100644 --- a/pkg/u8proto/u8proto.go +++ b/pkg/u8proto/u8proto.go @@ -34,6 +34,7 @@ var protoNames = map[U8proto]string{ var ProtoIDs = map[string]U8proto{ "all": 0, "any": 0, + "none": 0, "icmp": 1, "tcp": 6, "udp": 17, @@ -56,3 +57,11 @@ func ParseProtocol(proto string) (U8proto, error) { } return 0, fmt.Errorf("unknown protocol '%s'", proto) } + +func FromNumber(proto uint8) (U8proto, error) { + _, ok := protoNames[U8proto(proto)] + if !ok { + return 0, fmt.Errorf("unknown protocol %d", proto) + } + return U8proto(proto), nil +} diff --git a/test/controlplane/pod/hostport/lbmap1.golden b/test/controlplane/pod/hostport/lbmap1.golden index cb99cb9f1e810..f7c8ea691d79d 100644 --- a/test/controlplane/pod/hostport/lbmap1.golden +++ b/test/controlplane/pod/hostport/lbmap1.golden @@ -1,10 +1,10 @@ - - Services ------------------------------------------------------------------------ -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------------------+----------+--------------------+------------- -| 0 | test/hostport-1/host-port/8081 | HostPort | 0.0.0.0:8081/NONE | 0 | -| 1 | test/hostport-1/host-port/8081 | HostPort | 10.0.0.2:8081/NONE | 0 | -| 2 | test/hostport-1/host-port/8081 | HostPort | 10.0.0.3:8081/NONE | 0 | - ----------------------------------------------------------------------------------- + - Services ----------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------------------+----------+-------------------+------------- +| 0 | test/hostport-1/host-port/8081 | HostPort | 0.0.0.0:8081/TCP | 0 | +| 1 | test/hostport-1/host-port/8081 | HostPort | 10.0.0.2:8081/TCP | 0 | +| 2 | test/hostport-1/host-port/8081 | HostPort | 10.0.0.3:8081/TCP | 0 | + ---------------------------------------------------------------------------------- - Backends ------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/pod/hostport/lbmap2.golden b/test/controlplane/pod/hostport/lbmap2.golden index 0f13da08dbdf9..c8d20fd66fe42 100644 --- a/test/controlplane/pod/hostport/lbmap2.golden +++ b/test/controlplane/pod/hostport/lbmap2.golden @@ -1,10 +1,10 @@ - - Services ------------------------------------------------------------------------ -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------------------+----------+--------------------+------------- -| 0 | test/hostport-2/host-port/8081 | HostPort | 0.0.0.0:8081/NONE | 0 | -| 1 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.2:8081/NONE | 0 | -| 2 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.3:8081/NONE | 0 | - ----------------------------------------------------------------------------------- + - Services ----------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------------------+----------+-------------------+------------- +| 0 | test/hostport-2/host-port/8081 | HostPort | 0.0.0.0:8081/TCP | 0 | +| 1 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.2:8081/TCP | 0 | +| 2 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.3:8081/TCP | 0 | + ---------------------------------------------------------------------------------- - Backends ------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/pod/hostport/lbmap3.golden b/test/controlplane/pod/hostport/lbmap3.golden index 0f13da08dbdf9..c8d20fd66fe42 100644 --- a/test/controlplane/pod/hostport/lbmap3.golden +++ b/test/controlplane/pod/hostport/lbmap3.golden @@ -1,10 +1,10 @@ - - Services ------------------------------------------------------------------------ -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------------------+----------+--------------------+------------- -| 0 | test/hostport-2/host-port/8081 | HostPort | 0.0.0.0:8081/NONE | 0 | -| 1 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.2:8081/NONE | 0 | -| 2 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.3:8081/NONE | 0 | - ----------------------------------------------------------------------------------- + - Services ----------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------------------+----------+-------------------+------------- +| 0 | test/hostport-2/host-port/8081 | HostPort | 0.0.0.0:8081/TCP | 0 | +| 1 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.2:8081/TCP | 0 | +| 2 | test/hostport-2/host-port/8081 | HostPort | 10.0.0.3:8081/TCP | 0 | + ---------------------------------------------------------------------------------- - Backends ------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/dualstack/v1.24/lbmap1.golden b/test/controlplane/services/dualstack/v1.24/lbmap1.golden index f2dd5625d4b2d..6921b61af9d45 100644 --- a/test/controlplane/services/dualstack/v1.24/lbmap1.golden +++ b/test/controlplane/services/dualstack/v1.24/lbmap1.golden @@ -1,24 +1,24 @@ - - Services ------------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+------------------------+-----------+----------------------------+------------- -| 0 | default/echo-dualstack | ClusterIP | 10.96.207.99:69/NONE | 0, 2 | -| 1 | default/echo-dualstack | ClusterIP | 10.96.207.99:80/NONE | 1, 3 | -| 2 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:69/NONE | 5, 7 | -| 3 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:80/NONE | 6, 8 | -| 4 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 4 | -| 5 | default/echo-dualstack | NodePort | 0.0.0.0:30181/NONE | 1, 3 | -| 6 | default/echo-dualstack | NodePort | 0.0.0.0:32757/NONE | 0, 2 | -| 7 | default/echo-dualstack | NodePort | 10.0.0.2:30181/NONE | 1, 3 | -| 8 | default/echo-dualstack | NodePort | 10.0.0.2:32757/NONE | 0, 2 | -| 9 | default/echo-dualstack | NodePort | 10.0.0.3:30181/NONE | 1, 3 | -| 10 | default/echo-dualstack | NodePort | 10.0.0.3:32757/NONE | 0, 2 | -| 11 | default/echo-dualstack | NodePort | [::]:30181/NONE | 6, 8 | -| 12 | default/echo-dualstack | NodePort | [::]:32757/NONE | 5, 7 | -| 13 | default/echo-dualstack | NodePort | [f00d::1]:30181/NONE | 6, 8 | -| 14 | default/echo-dualstack | NodePort | [f00d::1]:32757/NONE | 5, 7 | -| 15 | default/echo-dualstack | NodePort | [f00d::2]:30181/NONE | 6, 8 | -| 16 | default/echo-dualstack | NodePort | [f00d::2]:32757/NONE | 5, 7 | - ------------------------------------------------------------------------------------ + - Services ------------------------------------------------------------------------ +| ID | Name | Type | Frontend | Backend IDs | +|----+------------------------+-----------+---------------------------+------------- +| 0 | default/echo-dualstack | ClusterIP | 10.96.207.99:69/UDP | 0, 2 | +| 1 | default/echo-dualstack | ClusterIP | 10.96.207.99:80/TCP | 1, 3 | +| 2 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:69/UDP | 5, 7 | +| 3 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:80/TCP | 6, 8 | +| 4 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 4 | +| 5 | default/echo-dualstack | NodePort | 0.0.0.0:30181/TCP | 1, 3 | +| 6 | default/echo-dualstack | NodePort | 0.0.0.0:32757/UDP | 0, 2 | +| 7 | default/echo-dualstack | NodePort | 10.0.0.2:30181/TCP | 1, 3 | +| 8 | default/echo-dualstack | NodePort | 10.0.0.2:32757/UDP | 0, 2 | +| 9 | default/echo-dualstack | NodePort | 10.0.0.3:30181/TCP | 1, 3 | +| 10 | default/echo-dualstack | NodePort | 10.0.0.3:32757/UDP | 0, 2 | +| 11 | default/echo-dualstack | NodePort | [::]:30181/TCP | 6, 8 | +| 12 | default/echo-dualstack | NodePort | [::]:32757/UDP | 5, 7 | +| 13 | default/echo-dualstack | NodePort | [f00d::1]:30181/TCP | 6, 8 | +| 14 | default/echo-dualstack | NodePort | [f00d::1]:32757/UDP | 5, 7 | +| 15 | default/echo-dualstack | NodePort | [f00d::2]:30181/TCP | 6, 8 | +| 16 | default/echo-dualstack | NodePort | [f00d::2]:32757/UDP | 5, 7 | + ----------------------------------------------------------------------------------- - Backends --------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/dualstack/v1.25/lbmap1.golden b/test/controlplane/services/dualstack/v1.25/lbmap1.golden index 2a77e14ff176c..76426aa5823cf 100644 --- a/test/controlplane/services/dualstack/v1.25/lbmap1.golden +++ b/test/controlplane/services/dualstack/v1.25/lbmap1.golden @@ -1,24 +1,24 @@ - - Services ------------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+------------------------+-----------+----------------------------+------------- -| 0 | default/echo-dualstack | ClusterIP | 10.96.255.196:69/NONE | 0, 2 | -| 1 | default/echo-dualstack | ClusterIP | 10.96.255.196:80/NONE | 1, 3 | -| 2 | default/echo-dualstack | ClusterIP | [fd00:10:96::31c2]:69/NONE | 5, 7 | -| 3 | default/echo-dualstack | ClusterIP | [fd00:10:96::31c2]:80/NONE | 6, 8 | -| 4 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 4 | -| 5 | default/echo-dualstack | NodePort | 0.0.0.0:31659/NONE | 0, 2 | -| 6 | default/echo-dualstack | NodePort | 0.0.0.0:32066/NONE | 1, 3 | -| 7 | default/echo-dualstack | NodePort | 10.0.0.2:31659/NONE | 0, 2 | -| 8 | default/echo-dualstack | NodePort | 10.0.0.2:32066/NONE | 1, 3 | -| 9 | default/echo-dualstack | NodePort | 10.0.0.3:31659/NONE | 0, 2 | -| 10 | default/echo-dualstack | NodePort | 10.0.0.3:32066/NONE | 1, 3 | -| 11 | default/echo-dualstack | NodePort | [::]:31659/NONE | 5, 7 | -| 12 | default/echo-dualstack | NodePort | [::]:32066/NONE | 6, 8 | -| 13 | default/echo-dualstack | NodePort | [f00d::1]:31659/NONE | 5, 7 | -| 14 | default/echo-dualstack | NodePort | [f00d::1]:32066/NONE | 6, 8 | -| 15 | default/echo-dualstack | NodePort | [f00d::2]:31659/NONE | 5, 7 | -| 16 | default/echo-dualstack | NodePort | [f00d::2]:32066/NONE | 6, 8 | - ------------------------------------------------------------------------------------ + - Services ------------------------------------------------------------------------ +| ID | Name | Type | Frontend | Backend IDs | +|----+------------------------+-----------+---------------------------+------------- +| 0 | default/echo-dualstack | ClusterIP | 10.96.255.196:69/UDP | 0, 2 | +| 1 | default/echo-dualstack | ClusterIP | 10.96.255.196:80/TCP | 1, 3 | +| 2 | default/echo-dualstack | ClusterIP | [fd00:10:96::31c2]:69/UDP | 5, 7 | +| 3 | default/echo-dualstack | ClusterIP | [fd00:10:96::31c2]:80/TCP | 6, 8 | +| 4 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 4 | +| 5 | default/echo-dualstack | NodePort | 0.0.0.0:31659/UDP | 0, 2 | +| 6 | default/echo-dualstack | NodePort | 0.0.0.0:32066/TCP | 1, 3 | +| 7 | default/echo-dualstack | NodePort | 10.0.0.2:31659/UDP | 0, 2 | +| 8 | default/echo-dualstack | NodePort | 10.0.0.2:32066/TCP | 1, 3 | +| 9 | default/echo-dualstack | NodePort | 10.0.0.3:31659/UDP | 0, 2 | +| 10 | default/echo-dualstack | NodePort | 10.0.0.3:32066/TCP | 1, 3 | +| 11 | default/echo-dualstack | NodePort | [::]:31659/UDP | 5, 7 | +| 12 | default/echo-dualstack | NodePort | [::]:32066/TCP | 6, 8 | +| 13 | default/echo-dualstack | NodePort | [f00d::1]:31659/UDP | 5, 7 | +| 14 | default/echo-dualstack | NodePort | [f00d::1]:32066/TCP | 6, 8 | +| 15 | default/echo-dualstack | NodePort | [f00d::2]:31659/UDP | 5, 7 | +| 16 | default/echo-dualstack | NodePort | [f00d::2]:32066/TCP | 6, 8 | + ----------------------------------------------------------------------------------- - Backends --------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/dualstack/v1.26/lbmap1.golden b/test/controlplane/services/dualstack/v1.26/lbmap1.golden index f2dd5625d4b2d..6921b61af9d45 100644 --- a/test/controlplane/services/dualstack/v1.26/lbmap1.golden +++ b/test/controlplane/services/dualstack/v1.26/lbmap1.golden @@ -1,24 +1,24 @@ - - Services ------------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+------------------------+-----------+----------------------------+------------- -| 0 | default/echo-dualstack | ClusterIP | 10.96.207.99:69/NONE | 0, 2 | -| 1 | default/echo-dualstack | ClusterIP | 10.96.207.99:80/NONE | 1, 3 | -| 2 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:69/NONE | 5, 7 | -| 3 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:80/NONE | 6, 8 | -| 4 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 4 | -| 5 | default/echo-dualstack | NodePort | 0.0.0.0:30181/NONE | 1, 3 | -| 6 | default/echo-dualstack | NodePort | 0.0.0.0:32757/NONE | 0, 2 | -| 7 | default/echo-dualstack | NodePort | 10.0.0.2:30181/NONE | 1, 3 | -| 8 | default/echo-dualstack | NodePort | 10.0.0.2:32757/NONE | 0, 2 | -| 9 | default/echo-dualstack | NodePort | 10.0.0.3:30181/NONE | 1, 3 | -| 10 | default/echo-dualstack | NodePort | 10.0.0.3:32757/NONE | 0, 2 | -| 11 | default/echo-dualstack | NodePort | [::]:30181/NONE | 6, 8 | -| 12 | default/echo-dualstack | NodePort | [::]:32757/NONE | 5, 7 | -| 13 | default/echo-dualstack | NodePort | [f00d::1]:30181/NONE | 6, 8 | -| 14 | default/echo-dualstack | NodePort | [f00d::1]:32757/NONE | 5, 7 | -| 15 | default/echo-dualstack | NodePort | [f00d::2]:30181/NONE | 6, 8 | -| 16 | default/echo-dualstack | NodePort | [f00d::2]:32757/NONE | 5, 7 | - ------------------------------------------------------------------------------------ + - Services ------------------------------------------------------------------------ +| ID | Name | Type | Frontend | Backend IDs | +|----+------------------------+-----------+---------------------------+------------- +| 0 | default/echo-dualstack | ClusterIP | 10.96.207.99:69/UDP | 0, 2 | +| 1 | default/echo-dualstack | ClusterIP | 10.96.207.99:80/TCP | 1, 3 | +| 2 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:69/UDP | 5, 7 | +| 3 | default/echo-dualstack | ClusterIP | [fd00:10:96::b050]:80/TCP | 6, 8 | +| 4 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 4 | +| 5 | default/echo-dualstack | NodePort | 0.0.0.0:30181/TCP | 1, 3 | +| 6 | default/echo-dualstack | NodePort | 0.0.0.0:32757/UDP | 0, 2 | +| 7 | default/echo-dualstack | NodePort | 10.0.0.2:30181/TCP | 1, 3 | +| 8 | default/echo-dualstack | NodePort | 10.0.0.2:32757/UDP | 0, 2 | +| 9 | default/echo-dualstack | NodePort | 10.0.0.3:30181/TCP | 1, 3 | +| 10 | default/echo-dualstack | NodePort | 10.0.0.3:32757/UDP | 0, 2 | +| 11 | default/echo-dualstack | NodePort | [::]:30181/TCP | 6, 8 | +| 12 | default/echo-dualstack | NodePort | [::]:32757/UDP | 5, 7 | +| 13 | default/echo-dualstack | NodePort | [f00d::1]:30181/TCP | 6, 8 | +| 14 | default/echo-dualstack | NodePort | [f00d::1]:32757/UDP | 5, 7 | +| 15 | default/echo-dualstack | NodePort | [f00d::2]:30181/TCP | 6, 8 | +| 16 | default/echo-dualstack | NodePort | [f00d::2]:32757/UDP | 5, 7 | + ----------------------------------------------------------------------------------- - Backends --------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/graceful-termination/lbmap1.golden b/test/controlplane/services/graceful-termination/lbmap1.golden index 681f0a59ce583..43afce5932c02 100644 --- a/test/controlplane/services/graceful-termination/lbmap1.golden +++ b/test/controlplane/services/graceful-termination/lbmap1.golden @@ -1,9 +1,9 @@ - - Services --------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+------------------------+-----------+------------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/graceful-term-svc | ClusterIP | 10.96.116.33:8081/NONE | 0 | - -------------------------------------------------------------------------------- + - Services -------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+------------------------+-----------+-----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/graceful-term-svc | ClusterIP | 10.96.116.33:8081/TCP | 0 | + ------------------------------------------------------------------------------- - Backends -------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/graceful-termination/lbmap2.golden b/test/controlplane/services/graceful-termination/lbmap2.golden index c4b19a32e3737..e0f49a910f404 100644 --- a/test/controlplane/services/graceful-termination/lbmap2.golden +++ b/test/controlplane/services/graceful-termination/lbmap2.golden @@ -1,9 +1,9 @@ - - Services --------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+------------------------+-----------+------------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/graceful-term-svc | ClusterIP | 10.96.116.33:8081/NONE | 0 | - -------------------------------------------------------------------------------- + - Services -------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+------------------------+-----------+-----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/graceful-term-svc | ClusterIP | 10.96.116.33:8081/TCP | 0 | + ------------------------------------------------------------------------------- - Backends ------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/graceful-termination/lbmap3.golden b/test/controlplane/services/graceful-termination/lbmap3.golden index d4dfb402d35a2..6cb38154a7f66 100644 --- a/test/controlplane/services/graceful-termination/lbmap3.golden +++ b/test/controlplane/services/graceful-termination/lbmap3.golden @@ -1,9 +1,9 @@ - - Services --------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+------------------------+-----------+------------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 0 | -| 1 | test/graceful-term-svc | ClusterIP | 10.96.116.33:8081/NONE | | - -------------------------------------------------------------------------------- + - Services -------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+------------------------+-----------+-----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 0 | +| 1 | test/graceful-term-svc | ClusterIP | 10.96.116.33:8081/TCP | | + ------------------------------------------------------------------------------- - Backends ------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-control-plane.golden b/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-control-plane.golden index 1d9fbe634eedd..1a7fa46ef1164 100644 --- a/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-control-plane.golden +++ b/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-control-plane.golden @@ -1,19 +1,19 @@ - - Services ---------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.223.82:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.247.58:80/NONE | 0 | -| 3 | test/echo | NodePort | 0.0.0.0:31010/NONE | 0 | -| 4 | test/echo | NodePort | 10.0.0.2:31010/NONE | 0 | -| 5 | test/echo | NodePort | 10.0.0.3:31010/NONE | 0 | -| 6 | test/echo-local | NodePort | 0.0.0.0:32556/NONE | | -| 7 | test/echo-local | NodePort | 0.0.0.0:32556/NONE/i | 0 | -| 8 | test/echo-local | NodePort | 10.0.0.2:32556/NONE | | -| 9 | test/echo-local | NodePort | 10.0.0.2:32556/NONE/i | 0 | -| 10 | test/echo-local | NodePort | 10.0.0.3:32556/NONE | | -| 11 | test/echo-local | NodePort | 10.0.0.3:32556/NONE/i | 0 | - --------------------------------------------------------------------------- + - Services --------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.223.82:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.247.58:80/TCP | 0 | +| 3 | test/echo | NodePort | 0.0.0.0:31010/TCP | 0 | +| 4 | test/echo | NodePort | 10.0.0.2:31010/TCP | 0 | +| 5 | test/echo | NodePort | 10.0.0.3:31010/TCP | 0 | +| 6 | test/echo-local | NodePort | 0.0.0.0:32556/TCP | | +| 7 | test/echo-local | NodePort | 0.0.0.0:32556/TCP/i | 0 | +| 8 | test/echo-local | NodePort | 10.0.0.2:32556/TCP | | +| 9 | test/echo-local | NodePort | 10.0.0.2:32556/TCP/i | 0 | +| 10 | test/echo-local | NodePort | 10.0.0.3:32556/TCP | | +| 11 | test/echo-local | NodePort | 10.0.0.3:32556/TCP/i | 0 | + -------------------------------------------------------------------------- - Backends -------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker.golden b/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker.golden index 4e00a29de7fcd..fc8810e2cd370 100644 --- a/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker.golden +++ b/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker.golden @@ -1,22 +1,22 @@ - - Services --------------------------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+-------------------------------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.223.82:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.247.58:80/NONE | 0 | -| 3 | test/echo-78d445655d-cfrdf/host-port/8080 | HostPort | 0.0.0.0:8080/NONE | 0 | -| 4 | test/echo-78d445655d-cfrdf/host-port/8080 | HostPort | 10.0.0.2:8080/NONE | 0 | -| 5 | test/echo-78d445655d-cfrdf/host-port/8080 | HostPort | 10.0.0.3:8080/NONE | 0 | -| 6 | test/echo | NodePort | 0.0.0.0:31010/NONE | 0 | -| 7 | test/echo | NodePort | 10.0.0.2:31010/NONE | 0 | -| 8 | test/echo | NodePort | 10.0.0.3:31010/NONE | 0 | -| 9 | test/echo-local | NodePort | 0.0.0.0:32556/NONE | 0 | -| 10 | test/echo-local | NodePort | 0.0.0.0:32556/NONE/i | 0 | -| 11 | test/echo-local | NodePort | 10.0.0.2:32556/NONE | 0 | -| 12 | test/echo-local | NodePort | 10.0.0.2:32556/NONE/i | 0 | -| 13 | test/echo-local | NodePort | 10.0.0.3:32556/NONE | 0 | -| 14 | test/echo-local | NodePort | 10.0.0.3:32556/NONE/i | 0 | - -------------------------------------------------------------------------------------------------- + - Services -------------------------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+-------------------------------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.223.82:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.247.58:80/TCP | 0 | +| 3 | test/echo-78d445655d-cfrdf/host-port/8080 | HostPort | 0.0.0.0:8080/TCP | 0 | +| 4 | test/echo-78d445655d-cfrdf/host-port/8080 | HostPort | 10.0.0.2:8080/TCP | 0 | +| 5 | test/echo-78d445655d-cfrdf/host-port/8080 | HostPort | 10.0.0.3:8080/TCP | 0 | +| 6 | test/echo | NodePort | 0.0.0.0:31010/TCP | 0 | +| 7 | test/echo | NodePort | 10.0.0.2:31010/TCP | 0 | +| 8 | test/echo | NodePort | 10.0.0.3:31010/TCP | 0 | +| 9 | test/echo-local | NodePort | 0.0.0.0:32556/TCP | 0 | +| 10 | test/echo-local | NodePort | 0.0.0.0:32556/TCP/i | 0 | +| 11 | test/echo-local | NodePort | 10.0.0.2:32556/TCP | 0 | +| 12 | test/echo-local | NodePort | 10.0.0.2:32556/TCP/i | 0 | +| 13 | test/echo-local | NodePort | 10.0.0.3:32556/TCP | 0 | +| 14 | test/echo-local | NodePort | 10.0.0.3:32556/TCP/i | 0 | + ------------------------------------------------------------------------------------------------- - Backends ------------------------------------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker2.golden b/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker2.golden index 1d9fbe634eedd..1a7fa46ef1164 100644 --- a/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker2.golden +++ b/test/controlplane/services/nodeport/v1.24/lbmap1_nodeport-worker2.golden @@ -1,19 +1,19 @@ - - Services ---------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.223.82:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.247.58:80/NONE | 0 | -| 3 | test/echo | NodePort | 0.0.0.0:31010/NONE | 0 | -| 4 | test/echo | NodePort | 10.0.0.2:31010/NONE | 0 | -| 5 | test/echo | NodePort | 10.0.0.3:31010/NONE | 0 | -| 6 | test/echo-local | NodePort | 0.0.0.0:32556/NONE | | -| 7 | test/echo-local | NodePort | 0.0.0.0:32556/NONE/i | 0 | -| 8 | test/echo-local | NodePort | 10.0.0.2:32556/NONE | | -| 9 | test/echo-local | NodePort | 10.0.0.2:32556/NONE/i | 0 | -| 10 | test/echo-local | NodePort | 10.0.0.3:32556/NONE | | -| 11 | test/echo-local | NodePort | 10.0.0.3:32556/NONE/i | 0 | - --------------------------------------------------------------------------- + - Services --------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.223.82:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.247.58:80/TCP | 0 | +| 3 | test/echo | NodePort | 0.0.0.0:31010/TCP | 0 | +| 4 | test/echo | NodePort | 10.0.0.2:31010/TCP | 0 | +| 5 | test/echo | NodePort | 10.0.0.3:31010/TCP | 0 | +| 6 | test/echo-local | NodePort | 0.0.0.0:32556/TCP | | +| 7 | test/echo-local | NodePort | 0.0.0.0:32556/TCP/i | 0 | +| 8 | test/echo-local | NodePort | 10.0.0.2:32556/TCP | | +| 9 | test/echo-local | NodePort | 10.0.0.2:32556/TCP/i | 0 | +| 10 | test/echo-local | NodePort | 10.0.0.3:32556/TCP | | +| 11 | test/echo-local | NodePort | 10.0.0.3:32556/TCP/i | 0 | + -------------------------------------------------------------------------- - Backends -------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-control-plane.golden b/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-control-plane.golden index b30e9e1016b8c..216a9e71e6419 100644 --- a/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-control-plane.golden +++ b/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-control-plane.golden @@ -1,19 +1,19 @@ - - Services ---------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.50.104:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/NONE | 0 | -| 3 | test/echo | NodePort | 0.0.0.0:30781/NONE | 0 | -| 4 | test/echo | NodePort | 10.0.0.2:30781/NONE | 0 | -| 5 | test/echo | NodePort | 10.0.0.3:30781/NONE | 0 | -| 6 | test/echo-local | NodePort | 0.0.0.0:30867/NONE | | -| 7 | test/echo-local | NodePort | 0.0.0.0:30867/NONE/i | 0 | -| 8 | test/echo-local | NodePort | 10.0.0.2:30867/NONE | | -| 9 | test/echo-local | NodePort | 10.0.0.2:30867/NONE/i | 0 | -| 10 | test/echo-local | NodePort | 10.0.0.3:30867/NONE | | -| 11 | test/echo-local | NodePort | 10.0.0.3:30867/NONE/i | 0 | - --------------------------------------------------------------------------- + - Services --------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.50.104:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/TCP | 0 | +| 3 | test/echo | NodePort | 0.0.0.0:30781/TCP | 0 | +| 4 | test/echo | NodePort | 10.0.0.2:30781/TCP | 0 | +| 5 | test/echo | NodePort | 10.0.0.3:30781/TCP | 0 | +| 6 | test/echo-local | NodePort | 0.0.0.0:30867/TCP | | +| 7 | test/echo-local | NodePort | 0.0.0.0:30867/TCP/i | 0 | +| 8 | test/echo-local | NodePort | 10.0.0.2:30867/TCP | | +| 9 | test/echo-local | NodePort | 10.0.0.2:30867/TCP/i | 0 | +| 10 | test/echo-local | NodePort | 10.0.0.3:30867/TCP | | +| 11 | test/echo-local | NodePort | 10.0.0.3:30867/TCP/i | 0 | + -------------------------------------------------------------------------- - Backends -------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker.golden b/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker.golden index 6698f993b4e05..7d02748280af3 100644 --- a/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker.golden +++ b/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker.golden @@ -1,22 +1,22 @@ - - Services --------------------------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+-------------------------------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.50.104:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/NONE | 0 | -| 3 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 0.0.0.0:8080/NONE | 0 | -| 4 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.2:8080/NONE | 0 | -| 5 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.3:8080/NONE | 0 | -| 6 | test/echo | NodePort | 0.0.0.0:30781/NONE | 0 | -| 7 | test/echo | NodePort | 10.0.0.2:30781/NONE | 0 | -| 8 | test/echo | NodePort | 10.0.0.3:30781/NONE | 0 | -| 9 | test/echo-local | NodePort | 0.0.0.0:30867/NONE | 0 | -| 10 | test/echo-local | NodePort | 0.0.0.0:30867/NONE/i | 0 | -| 11 | test/echo-local | NodePort | 10.0.0.2:30867/NONE | 0 | -| 12 | test/echo-local | NodePort | 10.0.0.2:30867/NONE/i | 0 | -| 13 | test/echo-local | NodePort | 10.0.0.3:30867/NONE | 0 | -| 14 | test/echo-local | NodePort | 10.0.0.3:30867/NONE/i | 0 | - -------------------------------------------------------------------------------------------------- + - Services -------------------------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+-------------------------------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.50.104:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/TCP | 0 | +| 3 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 0.0.0.0:8080/TCP | 0 | +| 4 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.2:8080/TCP | 0 | +| 5 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.3:8080/TCP | 0 | +| 6 | test/echo | NodePort | 0.0.0.0:30781/TCP | 0 | +| 7 | test/echo | NodePort | 10.0.0.2:30781/TCP | 0 | +| 8 | test/echo | NodePort | 10.0.0.3:30781/TCP | 0 | +| 9 | test/echo-local | NodePort | 0.0.0.0:30867/TCP | 0 | +| 10 | test/echo-local | NodePort | 0.0.0.0:30867/TCP/i | 0 | +| 11 | test/echo-local | NodePort | 10.0.0.2:30867/TCP | 0 | +| 12 | test/echo-local | NodePort | 10.0.0.2:30867/TCP/i | 0 | +| 13 | test/echo-local | NodePort | 10.0.0.3:30867/TCP | 0 | +| 14 | test/echo-local | NodePort | 10.0.0.3:30867/TCP/i | 0 | + ------------------------------------------------------------------------------------------------- - Backends ------------------------------------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker2.golden b/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker2.golden index b30e9e1016b8c..216a9e71e6419 100644 --- a/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker2.golden +++ b/test/controlplane/services/nodeport/v1.25/lbmap1_nodeport-worker2.golden @@ -1,19 +1,19 @@ - - Services ---------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.50.104:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/NONE | 0 | -| 3 | test/echo | NodePort | 0.0.0.0:30781/NONE | 0 | -| 4 | test/echo | NodePort | 10.0.0.2:30781/NONE | 0 | -| 5 | test/echo | NodePort | 10.0.0.3:30781/NONE | 0 | -| 6 | test/echo-local | NodePort | 0.0.0.0:30867/NONE | | -| 7 | test/echo-local | NodePort | 0.0.0.0:30867/NONE/i | 0 | -| 8 | test/echo-local | NodePort | 10.0.0.2:30867/NONE | | -| 9 | test/echo-local | NodePort | 10.0.0.2:30867/NONE/i | 0 | -| 10 | test/echo-local | NodePort | 10.0.0.3:30867/NONE | | -| 11 | test/echo-local | NodePort | 10.0.0.3:30867/NONE/i | 0 | - --------------------------------------------------------------------------- + - Services --------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.50.104:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/TCP | 0 | +| 3 | test/echo | NodePort | 0.0.0.0:30781/TCP | 0 | +| 4 | test/echo | NodePort | 10.0.0.2:30781/TCP | 0 | +| 5 | test/echo | NodePort | 10.0.0.3:30781/TCP | 0 | +| 6 | test/echo-local | NodePort | 0.0.0.0:30867/TCP | | +| 7 | test/echo-local | NodePort | 0.0.0.0:30867/TCP/i | 0 | +| 8 | test/echo-local | NodePort | 10.0.0.2:30867/TCP | | +| 9 | test/echo-local | NodePort | 10.0.0.2:30867/TCP/i | 0 | +| 10 | test/echo-local | NodePort | 10.0.0.3:30867/TCP | | +| 11 | test/echo-local | NodePort | 10.0.0.3:30867/TCP/i | 0 | + -------------------------------------------------------------------------- - Backends -------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-control-plane.golden b/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-control-plane.golden index b30e9e1016b8c..216a9e71e6419 100644 --- a/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-control-plane.golden +++ b/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-control-plane.golden @@ -1,19 +1,19 @@ - - Services ---------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.50.104:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/NONE | 0 | -| 3 | test/echo | NodePort | 0.0.0.0:30781/NONE | 0 | -| 4 | test/echo | NodePort | 10.0.0.2:30781/NONE | 0 | -| 5 | test/echo | NodePort | 10.0.0.3:30781/NONE | 0 | -| 6 | test/echo-local | NodePort | 0.0.0.0:30867/NONE | | -| 7 | test/echo-local | NodePort | 0.0.0.0:30867/NONE/i | 0 | -| 8 | test/echo-local | NodePort | 10.0.0.2:30867/NONE | | -| 9 | test/echo-local | NodePort | 10.0.0.2:30867/NONE/i | 0 | -| 10 | test/echo-local | NodePort | 10.0.0.3:30867/NONE | | -| 11 | test/echo-local | NodePort | 10.0.0.3:30867/NONE/i | 0 | - --------------------------------------------------------------------------- + - Services --------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.50.104:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/TCP | 0 | +| 3 | test/echo | NodePort | 0.0.0.0:30781/TCP | 0 | +| 4 | test/echo | NodePort | 10.0.0.2:30781/TCP | 0 | +| 5 | test/echo | NodePort | 10.0.0.3:30781/TCP | 0 | +| 6 | test/echo-local | NodePort | 0.0.0.0:30867/TCP | | +| 7 | test/echo-local | NodePort | 0.0.0.0:30867/TCP/i | 0 | +| 8 | test/echo-local | NodePort | 10.0.0.2:30867/TCP | | +| 9 | test/echo-local | NodePort | 10.0.0.2:30867/TCP/i | 0 | +| 10 | test/echo-local | NodePort | 10.0.0.3:30867/TCP | | +| 11 | test/echo-local | NodePort | 10.0.0.3:30867/TCP/i | 0 | + -------------------------------------------------------------------------- - Backends -------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker.golden b/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker.golden index 6698f993b4e05..7d02748280af3 100644 --- a/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker.golden +++ b/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker.golden @@ -1,22 +1,22 @@ - - Services --------------------------------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+-------------------------------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.50.104:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/NONE | 0 | -| 3 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 0.0.0.0:8080/NONE | 0 | -| 4 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.2:8080/NONE | 0 | -| 5 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.3:8080/NONE | 0 | -| 6 | test/echo | NodePort | 0.0.0.0:30781/NONE | 0 | -| 7 | test/echo | NodePort | 10.0.0.2:30781/NONE | 0 | -| 8 | test/echo | NodePort | 10.0.0.3:30781/NONE | 0 | -| 9 | test/echo-local | NodePort | 0.0.0.0:30867/NONE | 0 | -| 10 | test/echo-local | NodePort | 0.0.0.0:30867/NONE/i | 0 | -| 11 | test/echo-local | NodePort | 10.0.0.2:30867/NONE | 0 | -| 12 | test/echo-local | NodePort | 10.0.0.2:30867/NONE/i | 0 | -| 13 | test/echo-local | NodePort | 10.0.0.3:30867/NONE | 0 | -| 14 | test/echo-local | NodePort | 10.0.0.3:30867/NONE/i | 0 | - -------------------------------------------------------------------------------------------------- + - Services -------------------------------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+-------------------------------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.50.104:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/TCP | 0 | +| 3 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 0.0.0.0:8080/TCP | 0 | +| 4 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.2:8080/TCP | 0 | +| 5 | test/echo-757d4cb97f-9gmf7/host-port/8080 | HostPort | 10.0.0.3:8080/TCP | 0 | +| 6 | test/echo | NodePort | 0.0.0.0:30781/TCP | 0 | +| 7 | test/echo | NodePort | 10.0.0.2:30781/TCP | 0 | +| 8 | test/echo | NodePort | 10.0.0.3:30781/TCP | 0 | +| 9 | test/echo-local | NodePort | 0.0.0.0:30867/TCP | 0 | +| 10 | test/echo-local | NodePort | 0.0.0.0:30867/TCP/i | 0 | +| 11 | test/echo-local | NodePort | 10.0.0.2:30867/TCP | 0 | +| 12 | test/echo-local | NodePort | 10.0.0.2:30867/TCP/i | 0 | +| 13 | test/echo-local | NodePort | 10.0.0.3:30867/TCP | 0 | +| 14 | test/echo-local | NodePort | 10.0.0.3:30867/TCP/i | 0 | + ------------------------------------------------------------------------------------------------- - Backends ------------------------------------------------------------------------ | ID | L3n4Addr | State | Linked Services | diff --git a/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker2.golden b/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker2.golden index b30e9e1016b8c..216a9e71e6419 100644 --- a/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker2.golden +++ b/test/controlplane/services/nodeport/v1.26/lbmap1_nodeport-worker2.golden @@ -1,19 +1,19 @@ - - Services ---------------------------------------------------------------- -| ID | Name | Type | Frontend | Backend IDs | -|----+--------------------+-----------+-----------------------+------------- -| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/NONE | 1 | -| 1 | test/echo | ClusterIP | 10.96.50.104:80/NONE | 0 | -| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/NONE | 0 | -| 3 | test/echo | NodePort | 0.0.0.0:30781/NONE | 0 | -| 4 | test/echo | NodePort | 10.0.0.2:30781/NONE | 0 | -| 5 | test/echo | NodePort | 10.0.0.3:30781/NONE | 0 | -| 6 | test/echo-local | NodePort | 0.0.0.0:30867/NONE | | -| 7 | test/echo-local | NodePort | 0.0.0.0:30867/NONE/i | 0 | -| 8 | test/echo-local | NodePort | 10.0.0.2:30867/NONE | | -| 9 | test/echo-local | NodePort | 10.0.0.2:30867/NONE/i | 0 | -| 10 | test/echo-local | NodePort | 10.0.0.3:30867/NONE | | -| 11 | test/echo-local | NodePort | 10.0.0.3:30867/NONE/i | 0 | - --------------------------------------------------------------------------- + - Services --------------------------------------------------------------- +| ID | Name | Type | Frontend | Backend IDs | +|----+--------------------+-----------+----------------------+------------- +| 0 | default/kubernetes | ClusterIP | 10.96.0.1:443/TCP | 1 | +| 1 | test/echo | ClusterIP | 10.96.50.104:80/TCP | 0 | +| 2 | test/echo-local | ClusterIP | 10.96.155.18:80/TCP | 0 | +| 3 | test/echo | NodePort | 0.0.0.0:30781/TCP | 0 | +| 4 | test/echo | NodePort | 10.0.0.2:30781/TCP | 0 | +| 5 | test/echo | NodePort | 10.0.0.3:30781/TCP | 0 | +| 6 | test/echo-local | NodePort | 0.0.0.0:30867/TCP | | +| 7 | test/echo-local | NodePort | 0.0.0.0:30867/TCP/i | 0 | +| 8 | test/echo-local | NodePort | 10.0.0.2:30867/TCP | | +| 9 | test/echo-local | NodePort | 10.0.0.2:30867/TCP/i | 0 | +| 10 | test/echo-local | NodePort | 10.0.0.3:30867/TCP | | +| 11 | test/echo-local | NodePort | 10.0.0.3:30867/TCP/i | 0 | + -------------------------------------------------------------------------- - Backends -------------------------------------------------- | ID | L3n4Addr | State | Linked Services | diff --git a/test/helpers/kubectl.go b/test/helpers/kubectl.go index 972adda1ef951..1f859ac01773d 100644 --- a/test/helpers/kubectl.go +++ b/test/helpers/kubectl.go @@ -1894,7 +1894,7 @@ func (kub *Kubectl) validateServicePlumbingInCiliumPod(fullName, ciliumPod strin for _, port := range serviceObj.Spec.Ports { var foundPort *v1.ServicePort for _, realizedService := range realizedServices { - if port.Port == int32(realizedService.FrontendAddress.Port) { + if compareServicePortToFrontEnd(&port, realizedService.FrontendAddress) { foundPort = &port break } @@ -1904,7 +1904,8 @@ func (kub *Kubectl) validateServicePlumbingInCiliumPod(fullName, ciliumPod strin port.Port, fullName, serviceObj.Spec.ClusterIP, ciliumPod) } - if _, ok := lbMap[net.JoinHostPort(serviceObj.Spec.ClusterIP, fmt.Sprintf("%d", port.Port))]; !ok { + lKey := serviceAddressKey(serviceObj.Spec.ClusterIP, fmt.Sprintf("%d", port.Port), string(port.Protocol), "") + if _, ok := lbMap[lKey]; !ok { return fmt.Errorf("port %d of service %s (%s) not found in cilium bpf lb list of pod %s", port.Port, fullName, serviceObj.Spec.ClusterIP, ciliumPod) } @@ -1916,10 +1917,11 @@ func (kub *Kubectl) validateServicePlumbingInCiliumPod(fullName, ciliumPod strin foundBackend, foundBackendLB := false, false for _, realizedService := range realizedServices { frontEnd := realizedService.FrontendAddress - lb := lbMap[net.JoinHostPort(frontEnd.IP, fmt.Sprintf("%d", frontEnd.Port))] - + lbKey := serviceAddressKey(frontEnd.IP, fmt.Sprintf("%d", frontEnd.Port), string(frontEnd.Protocol), "") + lb := lbMap[lbKey] for _, backAddr := range realizedService.BackendAddresses { - if addr.IP == *backAddr.IP && uint16(port.Port) == backAddr.Port { + if addr.IP == *backAddr.IP && uint16(port.Port) == backAddr.Port && + compareProto(string(port.Protocol), backAddr.Protocol) { foundBackend = true for _, backend := range lb { if strings.Contains(backend, net.JoinHostPort(*backAddr.IP, fmt.Sprintf("%d", port.Port))) { @@ -4153,7 +4155,7 @@ CILIUM_SERVICES: for _, cSvc := range ciliumSvcs { if cSvc.Status.Realized.FrontendAddress.IP == k8sService.Spec.ClusterIP { for _, port := range k8sService.Spec.Ports { - if int32(cSvc.Status.Realized.FrontendAddress.Port) == port.Port { + if compareServicePortToFrontEnd(&port, cSvc.Status.Realized.FrontendAddress) { ciliumService = &cSvc break CILIUM_SERVICES } @@ -4174,7 +4176,11 @@ CILIUM_SERVICES: } // CiliumServiceAdd adds the given service on a 'pod' running Cilium -func (kub *Kubectl) CiliumServiceAdd(pod string, id int64, frontend string, backends []string, svcType, trafficPolicy string) error { +func (kub *Kubectl) CiliumServiceAdd(pod string, id int64, protocol string, frontend string, backends []string, svcType, trafficPolicy string) error { + protocol = strings.ToLower(protocol) + if protocol == "" || protocol == "any" || protocol == "none" { + protocol = "tcp" + } var opts []string switch strings.ToLower(svcType) { case "nodeport": @@ -4201,8 +4207,8 @@ func (kub *Kubectl) CiliumServiceAdd(pod string, id int64, frontend string, back backendsStr := strings.Join(backends, ",") ctx, cancel := context.WithTimeout(context.Background(), ShortCommandTimeout) defer cancel() - return kub.CiliumExecContext(ctx, pod, fmt.Sprintf("cilium service update --id %d --frontend %q --backends %q %s", - id, frontend, backendsStr, optsStr)).GetErr("cilium service update") + return kub.CiliumExecContext(ctx, pod, fmt.Sprintf("cilium service update --id %d --protocol %q --frontend %q --backends %q %s", + id, protocol, frontend, backendsStr, optsStr)).GetErr("cilium service update") } // CiliumServiceDel deletes the service with 'id' on a 'pod' running Cilium @@ -4484,7 +4490,7 @@ func validateCiliumSvc(cSvc models.Service, k8sSvcs []v1.Service, k8sEps []v1.En var k8sServicePort *v1.ServicePort for _, k8sPort := range k8sService.Spec.Ports { - if k8sPort.Port == int32(cSvc.Status.Realized.FrontendAddress.Port) { + if compareServicePortToFrontEnd(&k8sPort, cSvc.Status.Realized.FrontendAddress) { k8sServicePort = &k8sPort k8sServicesFound[serviceKey(*k8sService)] = true break @@ -4513,14 +4519,16 @@ func validateCiliumSvc(cSvc models.Service, k8sSvcs []v1.Service, k8sEps []v1.En } func validateCiliumSvcLB(cSvc models.Service, lbMap map[string][]string) error { - scope := "" + var scope string if cSvc.Status.Realized.FrontendAddress.Scope == models.FrontendAddressScopeInternal { scope = "/i" } - - frontendAddress := net.JoinHostPort( + frontendAddress := serviceAddressKey( cSvc.Status.Realized.FrontendAddress.IP, - strconv.Itoa(int(cSvc.Status.Realized.FrontendAddress.Port))) + scope + strconv.Itoa(int(cSvc.Status.Realized.FrontendAddress.Port)), + cSvc.Status.Realized.FrontendAddress.Protocol, + scope, + ) bpfBackends, ok := lbMap[frontendAddress] if !ok { return fmt.Errorf("%s bpf lb map entry not found", frontendAddress) @@ -4557,7 +4565,9 @@ func getK8sEndpointAddresses(ep v1.Endpoints) []*models.BackendAddress { } func addrsEqual(addr1, addr2 *models.BackendAddress) bool { - return *addr1.IP == *addr2.IP && addr1.Port == addr2.Port + return *addr1.IP == *addr2.IP && + addr1.Port == addr2.Port && + compareProto(addr1.Protocol, addr2.Protocol) } // GenerateNamespaceForTest generates a namespace based off of the current test @@ -4871,3 +4881,32 @@ func (kub *Kubectl) DelVXLAN(nodeName string, vxlanId int) *CmdRes { cmd := fmt.Sprintf("ip link del dev vxlan%d", vxlanId) return kub.ExecInHostNetNS(context.TODO(), nodeName, cmd) } + +func compareProto(proto1, proto2 string) bool { + proto1 = strings.ToLower(proto1) + if proto1 == "" || proto1 == "none" || proto1 == "any" { + return true + } + proto2 = strings.ToLower(proto2) + if proto2 == "" || proto2 == "none" || proto2 == "any" { + return true + } + return proto1 == proto2 +} + +func compareServicePortToFrontEnd(sP *v1.ServicePort, fA *models.FrontendAddress) bool { + return sP.Port == int32(fA.Port) && compareProto(string(sP.Protocol), fA.Protocol) +} + +func serviceAddressKey(ip, port, proto, scope string) string { + newOutputStyle := HasNewServiceOutput(GetRunningCiliumVersion()) + k := net.JoinHostPort(ip, port) + if newOutputStyle { + p := strings.ToLower(proto) + if p == "" || p == "none" || p == "any" { + proto = "ANY" + } + return fmt.Sprintf("%s/%s%s", k, proto, scope) + } + return fmt.Sprintf("%s%s", k, scope) +} diff --git a/test/helpers/utils.go b/test/helpers/utils.go index 223bc294ea311..0c9d3ea036a73 100644 --- a/test/helpers/utils.go +++ b/test/helpers/utils.go @@ -28,6 +28,7 @@ import ( // ensure that our random numbers are seeded differently on each run var randGen = rand.NewSafeRand(time.Now().UnixNano()) +var runningCiliumVersion string // IsRunningOnJenkins detects if the currently running Ginkgo application is // most likely running in a Jenkins environment. Returns true if certain @@ -47,6 +48,35 @@ func IsRunningOnJenkins() bool { return result } +// SetCiliumVersion sets the currently running cilium version +// and returns a function that unsets it. +func SetRunningCiliumVersion(v string) func() { + runningCiliumVersion = v + return func() { + runningCiliumVersion = "" + } +} + +// GetRunningCiliumVersion gets the currently running cilium version. +func GetRunningCiliumVersion() string { + return runningCiliumVersion +} + +// HasNewServiceOutput checks to see if the current running cilium +// version uses the old style service output (e.g. "0.0.0.0:53") vs +// the new style (e.g. "0.0.0.0:53/TCP"). +func HasNewServiceOutput(ver string) bool { + cst, err := versioncheck.Version(ver) + // If the version is not parseable it is probably + // someone's custom build or not set. + // Either way, it is probably using the new output + // format. + if err != nil { + return true + } + return versioncheck.MustCompile(">=1.17.0")(cst) +} + // Sleep sleeps for the specified duration in seconds func Sleep(delay time.Duration) { time.Sleep(delay * time.Second) @@ -205,6 +235,7 @@ func GetAppPods(apps []string, namespace string, kubectl *Kubectl, appFmt string res, err := kubectl.GetPodNames(namespace, fmt.Sprintf("%s=%s", appFmt, v)) Expect(err).Should(BeNil()) Expect(res).Should(Not(BeNil())) + Expect(len(res)).To(BeNumerically(">", 0)) appPods[v] = res[0] log.Infof("GetAppPods: pod=%q assigned to %q", res[0], v) } diff --git a/test/k8s/service_helpers.go b/test/k8s/service_helpers.go index ff8d557a48554..67ff0eb5d11e7 100644 --- a/test/k8s/service_helpers.go +++ b/test/k8s/service_helpers.go @@ -43,7 +43,7 @@ func ciliumAddService(kubectl *helpers.Kubectl, id int64, frontend string, backe ciliumPods, err := kubectl.GetCiliumPods() ExpectWithOffset(1, err).To(BeNil(), "Cannot get cilium pods") for _, pod := range ciliumPods { - err := kubectl.CiliumServiceAdd(pod, id, frontend, backends, svcType, trafficPolicy) + err := kubectl.CiliumServiceAdd(pod, id, "", frontend, backends, svcType, trafficPolicy) ExpectWithOffset(1, err).To(BeNil(), "Failed to add cilium service") } } diff --git a/test/k8s/updates.go b/test/k8s/updates.go index aa3c0bec4e1bd..31f0739de4b25 100644 --- a/test/k8s/updates.go +++ b/test/k8s/updates.go @@ -271,6 +271,8 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers opts) }, time.Second*30, time.Second*1).Should(helpers.CMDSuccess(), fmt.Sprintf("Cilium %q was not able to be deployed", oldHelmChartVersion)) + helpers.SetRunningCiliumVersion(oldHelmChartVersion) + // Cilium is only ready if kvstore is ready, the kvstore is ready if // kube-dns is running. ExpectCiliumReady(kubectl) @@ -457,6 +459,8 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers opts) }, time.Second*30, time.Second*1).Should(helpers.CMDSuccess(), fmt.Sprintf("Cilium %q was not able to be deployed", newHelmChartVersion)) + helpers.SetRunningCiliumVersion(newHelmChartVersion) + By("Validating pods have the right image version upgraded") err = helpers.WithTimeout( waitForUpdateImage(newImageVersion), @@ -499,6 +503,8 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers cmd = kubectl.ExecMiddle("helm rollback cilium 1 --namespace=" + helpers.CiliumNamespace) ExpectWithOffset(1, cmd).To(helpers.CMDSuccess(), "Cilium %q was not able to be deployed", oldHelmChartVersion) + helpers.SetRunningCiliumVersion(oldHelmChartVersion) + err = helpers.WithTimeout( waitForUpdateImage(oldImageVersion), "Cilium Pods are not updating correctly",