diff --git a/api/pkg/apis/projectcalico/v3/felixconfig.go b/api/pkg/apis/projectcalico/v3/felixconfig.go index a44240514e2..bb62c894357 100644 --- a/api/pkg/apis/projectcalico/v3/felixconfig.go +++ b/api/pkg/apis/projectcalico/v3/felixconfig.go @@ -778,6 +778,11 @@ type FelixConfigurationSpec struct { //+kubebuilder:validation:Enum=Enabled;Disabled;L2Only BPFRedirectToPeer string `json:"bpfRedirectToPeer,omitempty"` + // BPFProfiling controls profiling of BPF programs. At the monent, it can be + // Disabled or Enabled. [Default: Disabled] + //+kubebuilder:validation:Enum=Enabled;Disabled + BPFProfiling string `json:"bpfProfiling,omitempty"` + // RouteSource configures where Felix gets its routing information. // - WorkloadIPs: use workload endpoints to construct routes. // - CalicoIPAM: the default - use IPAM data to construct routes. diff --git a/api/pkg/openapi/generated.openapi.go b/api/pkg/openapi/generated.openapi.go index 98994d1eec3..d54f12fddfe 100644 --- a/api/pkg/openapi/generated.openapi.go +++ b/api/pkg/openapi/generated.openapi.go @@ -3213,6 +3213,13 @@ func schema_pkg_apis_projectcalico_v3_FelixConfigurationSpec(ref common.Referenc Format: "", }, }, + "bpfProfiling": { + SchemaProps: spec.SchemaProps{ + Description: "BPFProfiling controls profiling of BPF programs. At the monent, it can be Disabled or Enabled. [Default: Disabled]", + Type: []string{"string"}, + Format: "", + }, + }, "routeSource": { SchemaProps: spec.SchemaProps{ Description: "RouteSource configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes.", diff --git a/felix/Makefile b/felix/Makefile index a5460d76ab4..7a8784c1ada 100644 --- a/felix/Makefile +++ b/felix/Makefile @@ -278,7 +278,7 @@ $(FELIX_CONTAINER_CREATED): register \ docker-image/felix.cfg \ docker-image/Dockerfile \ $(shell test "$(FELIX_IMAGE_ID)" || echo force-rebuild) - $(DOCKER_BUILD) -t $(FELIX_IMAGE_WITH_TAG) -f ./docker-image/Dockerfile docker-image + $(DOCKER_BUILD) --network=host -t $(FELIX_IMAGE_WITH_TAG) -f ./docker-image/Dockerfile docker-image $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest touch $(FELIX_CONTAINER_CREATED) diff --git a/felix/bpf-gpl/bpf.h b/felix/bpf-gpl/bpf.h index 4f49918bfe1..6b747f92ac6 100644 --- a/felix/bpf-gpl/bpf.h +++ b/felix/bpf-gpl/bpf.h @@ -309,6 +309,7 @@ extern const volatile struct cali_tc_preamble_globals __globals; #define HOST_TUNNEL_IP CALI_CONFIGURABLE_IP(host_tunnel_ip) #define WG_PORT CALI_CONFIGURABLE(wg_port) #define NATIN_IFACE CALI_CONFIGURABLE(natin_idx) +#define PROFILING CALI_CONFIGURABLE(profiling) #ifdef UNITTEST #define CALI_PATCH_DEFINE(name, pattern) \ diff --git a/felix/bpf-gpl/fib.h b/felix/bpf-gpl/fib.h index 544f8bd72bf..9c54dd6e680 100644 --- a/felix/bpf-gpl/fib.h +++ b/felix/bpf-gpl/fib.h @@ -8,6 +8,7 @@ #include "types.h" #include "skb.h" #include "ifstate.h" +#include "profiling.h" #if CALI_FIB_ENABLED #define fwd_fib(fwd) ((fwd)->fib) @@ -382,22 +383,32 @@ static CALI_BPF_INLINE int forward_or_drop(struct cali_tc_ctx *ctx) skb_set_mark(ctx->skb, ctx->fwd.mark); /* make sure that each pkt has SEEN mark */ } - if (CALI_LOG_LEVEL >= CALI_LOG_LEVEL_INFO) { - __u64 prog_end_time = bpf_ktime_get_ns(); - CALI_INFO("Final result=ALLOW (%d). Program execution time: %lluns", - reason, prog_end_time-state->prog_start_time); - } - - return rc; + goto allow; deny: - if (CALI_LOG_LEVEL >= CALI_LOG_LEVEL_INFO) { + rc = TC_ACT_SHOT; + +allow: + if (CALI_LOG_LEVEL_INFO >= CALI_LOG_LEVEL_INFO || PROFILING) { __u64 prog_end_time = bpf_ktime_get_ns(); - CALI_INFO("Final result=DENY (%x). Program execution time: %lluns", - reason, prog_end_time-state->prog_start_time); + + if (PROFILING) { + prof_record_sample(ctx->skb->ifindex, + (CALI_F_FROM_HEP || CALI_F_TO_WEP ? 0 : 2) + + (ct_result_rc(ctx->state->ct_result.rc) == CALI_CT_NEW ? 0 : 1), + state->prog_start_time, prog_end_time); + } + + if (rc == TC_ACT_SHOT) { + CALI_INFO("Final result=DENY (%d). Program execution time: %lluns", + reason, prog_end_time-state->prog_start_time); + } else { + CALI_INFO("Final result=ALLOW (%d). Program execution time: %lluns", + reason, prog_end_time-state->prog_start_time); + } } - return TC_ACT_SHOT; + return rc; } #endif /* __CALI_FIB_H__ */ diff --git a/felix/bpf-gpl/globals.h b/felix/bpf-gpl/globals.h index cd852933799..27c0dbb10b1 100644 --- a/felix/bpf-gpl/globals.h +++ b/felix/bpf-gpl/globals.h @@ -19,7 +19,7 @@ struct name { \ ip_t host_tunnel_ip; \ __be32 flags; \ __be16 wg_port; \ - __be16 __pad; \ + __be16 profiling; \ __u32 natin_idx; \ __u32 natout_idx; \ __u8 iface_name[16]; \ diff --git a/felix/bpf-gpl/profiling.h b/felix/bpf-gpl/profiling.h new file mode 100644 index 00000000000..670fc739b1a --- /dev/null +++ b/felix/bpf-gpl/profiling.h @@ -0,0 +1,47 @@ +// Project Calico BPF dataplane programs. +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +#ifndef __CALI_BPF_PROFILING_H__ +#define __CALI_BPF_PROFILING_H__ + +struct prof_key { + __u32 ifindex; + __u32 kind; +}; + +struct prof_val { + __u64 time; + __u64 samples; +}; + +CALI_MAP(cali_profiling, 2, + BPF_MAP_TYPE_PERCPU_HASH, + struct prof_key, struct prof_val, + 20000, 0) + +static CALI_BPF_INLINE void prof_record_sample(__u32 ifindex, __u32 kind, __u64 start, __u64 end) +{ + struct prof_key key = { + .ifindex = ifindex, + .kind = kind, + }; + + __u64 diff = end - start; + + struct prof_val *val = cali_profiling_lookup_elem(&key); + + if (val) { + val->time += diff; + val->samples++; + } else { + struct prof_val val = { + .time = diff, + .samples = 1, + }; + + cali_profiling_update_elem(&key, &val, 0); + } +} + +#endif /* __CALI_BPF_PROFILING_H__ */ diff --git a/felix/bpf-gpl/tc.c b/felix/bpf-gpl/tc.c index 734577ace03..f2784e18129 100644 --- a/felix/bpf-gpl/tc.c +++ b/felix/bpf-gpl/tc.c @@ -152,7 +152,7 @@ int calico_tc_main(struct __sk_buff *skb) counter_inc(ctx, COUNTER_TOTAL_PACKETS); - if (CALI_LOG_LEVEL >= CALI_LOG_LEVEL_INFO) { + if (CALI_LOG_LEVEL >= CALI_LOG_LEVEL_INFO || PROFILING) { ctx->state->prog_start_time = bpf_ktime_get_ns(); } diff --git a/felix/bpf/attach.go b/felix/bpf/attach.go index 41d2767c92d..ff76cd4598f 100644 --- a/felix/bpf/attach.go +++ b/felix/bpf/attach.go @@ -54,6 +54,7 @@ type AttachPoint struct { PolicyIdxV6 int Iface string LogLevel string + Profiling string } func (ap *AttachPoint) LogVal() string { diff --git a/felix/bpf/bpfmap/bpf_maps.go b/felix/bpf/bpfmap/bpf_maps.go index 618290e8514..451a454d39d 100644 --- a/felix/bpf/bpfmap/bpf_maps.go +++ b/felix/bpf/bpfmap/bpf_maps.go @@ -30,6 +30,7 @@ import ( "github.com/projectcalico/calico/felix/bpf/jump" "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/nat" + "github.com/projectcalico/calico/felix/bpf/profiling" "github.com/projectcalico/calico/felix/bpf/routes" "github.com/projectcalico/calico/felix/bpf/state" ) @@ -56,6 +57,7 @@ type CommonMaps struct { JumpMap maps.MapWithDeleteIfExists XDPProgramsMap maps.Map XDPJumpMap maps.MapWithDeleteIfExists + ProfilingMap maps.Map } type Maps struct { @@ -87,6 +89,7 @@ func getCommonMaps() *CommonMaps { JumpMap: jump.Map().(maps.MapWithDeleteIfExists), XDPProgramsMap: hook.NewXDPProgramsMap(), XDPJumpMap: jump.XDPMap().(maps.MapWithDeleteIfExists), + ProfilingMap: profiling.Map(), } } @@ -168,6 +171,7 @@ func (c *CommonMaps) slice() []maps.Map { c.JumpMap, c.XDPProgramsMap, c.XDPJumpMap, + c.ProfilingMap, } } diff --git a/felix/bpf/libbpf/libbpf.go b/felix/bpf/libbpf/libbpf.go index 58bf48c430a..e8fa0999f0d 100644 --- a/felix/bpf/libbpf/libbpf.go +++ b/felix/bpf/libbpf/libbpf.go @@ -447,6 +447,7 @@ func TcSetGlobals( C.uint(globalData.Flags), C.ushort(globalData.WgPort), C.ushort(globalData.Wg6Port), + C.ushort(globalData.Profiling), C.uint(globalData.NatIn), C.uint(globalData.NatOut), C.uint(globalData.LogFilterJmp), diff --git a/felix/bpf/libbpf/libbpf_api.h b/felix/bpf/libbpf/libbpf_api.h index cb02acfb97f..c7947ff6f4c 100644 --- a/felix/bpf/libbpf/libbpf_api.h +++ b/felix/bpf/libbpf/libbpf_api.h @@ -153,6 +153,7 @@ void bpf_tc_set_globals(struct bpf_map *map, uint flags, ushort wg_port, ushort wg6_port, + ushort profiling, uint natin, uint natout, uint log_filter_jmp, @@ -167,6 +168,7 @@ void bpf_tc_set_globals(struct bpf_map *map, .psnat_len = psnat_len, .flags = flags, .wg_port = wg_port, + .profiling = profiling, .natin_idx = natin, .natout_idx = natout, .log_filter_jmp = log_filter_jmp, @@ -334,7 +336,7 @@ void bpf_xdp_set_globals(struct bpf_map *map, char *iface_name, uint *jumps, uin strncpy(data.v4.iface_name, iface_name, sizeof(data.v4.iface_name)); data.v4.iface_name[sizeof(data.v4.iface_name)-1] = '\0'; data.v6 = data.v4; - + int i; for (i = 0; i < sizeof(data.v4.jumps)/sizeof(__u32); i++) { diff --git a/felix/bpf/libbpf/libbpf_common.go b/felix/bpf/libbpf/libbpf_common.go index 6e6fe0fc953..1a4d9753aa5 100644 --- a/felix/bpf/libbpf/libbpf_common.go +++ b/felix/bpf/libbpf/libbpf_common.go @@ -27,6 +27,7 @@ type TcGlobalData struct { Flags uint32 WgPort uint16 Wg6Port uint16 + Profiling uint16 NatIn uint32 NatOut uint32 LogFilterJmp uint32 diff --git a/felix/bpf/profiling/map.go b/felix/bpf/profiling/map.go new file mode 100644 index 00000000000..5168e55abc2 --- /dev/null +++ b/felix/bpf/profiling/map.go @@ -0,0 +1,63 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profiling + +import ( + "encoding/binary" + + "github.com/projectcalico/calico/felix/bpf/maps" +) + +const ( + KeySize = 8 // 2 x uint32 + ValueSize = 16 // 2 x uint64 +) + +var MapParameters = maps.MapParameters{ + Type: "percpu_hash", + KeySize: KeySize, + ValueSize: ValueSize, + MaxEntries: 20000, + Name: "cali_profiling", + Version: 2, +} + +func Map() maps.Map { + return maps.NewPinnedMap(MapParameters) +} + +type Key struct { + Ifindex int + Kind int +} + +func KeyFromBytes(b []byte) Key { + return Key{ + Ifindex: int(binary.LittleEndian.Uint32(b[0:4])), + Kind: int(binary.LittleEndian.Uint32(b[4:8])), + } +} + +type Value struct { + Time int + Samples int +} + +func ValueFromBytes(b []byte) Value { + return Value{ + Time: int(binary.LittleEndian.Uint64(b[0:8])), + Samples: int(binary.LittleEndian.Uint64(b[8:16])), + } +} diff --git a/felix/bpf/tc/attach.go b/felix/bpf/tc/attach.go index 6448851fbdc..a7a213d1a6c 100644 --- a/felix/bpf/tc/attach.go +++ b/felix/bpf/tc/attach.go @@ -420,6 +420,10 @@ func (ap *AttachPoint) ConfigureProgram(m *libbpf.Map) error { LogFilterJmp: uint32(ap.LogFilterIdx), } + if ap.Profiling == "Enabled" { + globalData.Profiling = 1 + } + copy(globalData.HostIPv4[0:4], ap.HostIPv4.To4()) copy(globalData.HostIPv6[:], ap.HostIPv6.To16()) diff --git a/felix/bpf/ut/bpf_prog_test.go b/felix/bpf/ut/bpf_prog_test.go index 775dded7a16..87ab78cef9e 100644 --- a/felix/bpf/ut/bpf_prog_test.go +++ b/felix/bpf/ut/bpf_prog_test.go @@ -51,6 +51,7 @@ import ( "github.com/projectcalico/calico/felix/bpf/maps" "github.com/projectcalico/calico/felix/bpf/nat" "github.com/projectcalico/calico/felix/bpf/polprog" + "github.com/projectcalico/calico/felix/bpf/profiling" "github.com/projectcalico/calico/felix/bpf/routes" "github.com/projectcalico/calico/felix/bpf/state" "github.com/projectcalico/calico/felix/bpf/tc" @@ -576,6 +577,7 @@ var ( natMap, natBEMap, ctMap, rtMap, ipsMap, testStateMap, affinityMap, arpMap, fsafeMap maps.Map natMapV6, natBEMapV6, ctMapV6, rtMapV6, ipsMapV6, affinityMapV6, arpMapV6, fsafeMapV6 maps.Map stateMap, countersMap, ifstateMap, progMap, progMapXDP, policyJumpMap, policyJumpMapXDP maps.Map + profilingMap maps.Map allMaps []maps.Map ) @@ -603,10 +605,11 @@ func initMapsOnce() { ifstateMap = ifstate.Map() policyJumpMap = jump.Map() policyJumpMapXDP = jump.XDPMap() + profilingMap = profiling.Map() allMaps = []maps.Map{natMap, natBEMap, natMapV6, natBEMapV6, ctMap, ctMapV6, rtMap, rtMapV6, ipsMap, ipsMapV6, stateMap, testStateMap, affinityMap, affinityMapV6, arpMap, arpMapV6, fsafeMap, fsafeMapV6, - countersMap, ifstateMap, + countersMap, ifstateMap, profilingMap, policyJumpMap, policyJumpMapXDP} for _, m := range allMaps { err := m.EnsureExists() diff --git a/felix/cmd/calico-bpf/commands/counters.go b/felix/cmd/calico-bpf/commands/counters.go index bc31ffc8e5e..cf289eb9a2e 100644 --- a/felix/cmd/calico-bpf/commands/counters.go +++ b/felix/cmd/calico-bpf/commands/counters.go @@ -17,7 +17,6 @@ package commands import ( "fmt" "net" - "os" "github.com/olekukonko/tablewriter" log "github.com/sirupsen/logrus" @@ -56,14 +55,14 @@ var countersDumpCmd = &cobra.Command{ defer m.Close() if iface == "" { - doForAllInterfaces("dump", dumpInterface) + doForAllInterfaces(cmd, "dump", dumpInterface) } else { i, err := net.InterfaceByName(iface) if err != nil { log.WithError(err).Errorf("No such interface: %s", iface) return } - if err := dumpInterface(m, i); err != nil { + if err := dumpInterface(cmd, m, i); err != nil { log.WithError(err).Error("Failed to dump counter map.") } } @@ -83,14 +82,14 @@ var countersFlushCmd = &cobra.Command{ defer m.Close() if iface == "" { - doForAllInterfaces("flush", flushInterface) + doForAllInterfaces(cmd, "flush", flushInterface) } else { i, err := net.InterfaceByName(iface) if err != nil { log.WithError(err).Errorf("No such interface: %s", iface) return } - if err := flushInterface(m, i); err != nil { + if err := flushInterface(cmd, m, i); err != nil { log.WithError(err).Error("Failed to flush counter map.") } } @@ -105,7 +104,7 @@ func parseFlags(cmd *cobra.Command) string { return iface } -func doForAllInterfaces(action string, fn func(maps.Map, *net.Interface) error) { +func doForAllInterfaces(cmd *cobra.Command, action string, fn func(*cobra.Command, maps.Map, *net.Interface) error) { interfaces, err := net.Interfaces() if err != nil { log.WithError(err).Error("failed to get list of interfaces.") @@ -120,7 +119,7 @@ func doForAllInterfaces(action string, fn func(maps.Map, *net.Interface) error) defer m.Close() for _, i := range interfaces { - err = fn(m, &i) + err = fn(cmd, m, &i) if err != nil { log.WithError(err).Errorf("Failed to %s interface %s", action, i.Name) continue @@ -128,7 +127,7 @@ func doForAllInterfaces(action string, fn func(maps.Map, *net.Interface) error) } } -func dumpInterface(m maps.Map, iface *net.Interface) error { +func dumpInterface(cmd *cobra.Command, m maps.Map, iface *net.Interface) error { values := make([][]uint64, len(hook.All)) for _, hook := range hook.All { val, err := counters.Read(m, iface.Index, hook) @@ -141,7 +140,7 @@ func dumpInterface(m maps.Map, iface *net.Interface) error { values[hook] = val } - table := tablewriter.NewWriter(os.Stdout) + table := tablewriter.NewWriter(cmd.OutOrStdout()) table.SetCaption(true, fmt.Sprintf("dumped %s counters.", iface.Name)) table.SetHeader([]string{"CATEGORY", "TYPE", "INGRESS", "EGRESS", "XDP"}) @@ -165,7 +164,7 @@ func dumpInterface(m maps.Map, iface *net.Interface) error { return nil } -func flushInterface(m maps.Map, iface *net.Interface) error { +func flushInterface(cmd *cobra.Command, m maps.Map, iface *net.Interface) error { for _, hook := range hook.All { err := counters.Flush(m, iface.Index, hook) if err != nil { diff --git a/felix/cmd/calico-bpf/commands/profiling.go b/felix/cmd/calico-bpf/commands/profiling.go new file mode 100644 index 00000000000..26b03d4ee8f --- /dev/null +++ b/felix/cmd/calico-bpf/commands/profiling.go @@ -0,0 +1,127 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "fmt" + "net" + + "github.com/olekukonko/tablewriter" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/projectcalico/calico/felix/bpf/maps" + "github.com/projectcalico/calico/felix/bpf/profiling" +) + +// profilingCmd represents the profiling command +var profilingCmd = &cobra.Command{ + Use: "profiling", + Short: "Show and reset profiling data", +} + +func init() { + profilingCmd.AddCommand(profilingE2ECmd) + rootCmd.AddCommand(profilingCmd) +} + +var profilingE2ECmd = &cobra.Command{ + Use: "e2e", + Short: "Shows average e2e latency for every interface since last query", + Run: e2eLatency, +} + +func e2eLatency(cmd *cobra.Command, args []string) { + m := profiling.Map() + if err := m.Open(); err != nil { + log.WithError(err).Error("Failed to open profiling map.") + return + } + defer m.Close() + + data := make(map[profiling.Key]profiling.Value) + + err := m.Iter(func(k, v []byte) maps.IteratorAction { + key := profiling.KeyFromBytes(k) + + switch key.Kind { + case 0, 1, 2, 3: + // nothing + default: + return maps.IterNone + } + + var val profiling.Value + + for i := 0; i < maps.NumPossibleCPUs(); i++ { + x := profiling.ValueFromBytes(v[i*profiling.ValueSize:]) + val.Time += x.Time + val.Samples += x.Samples + } + + data[key] = val + + return maps.IterDelete + }) + + if err != nil { + log.WithError(err).Error("Failed to read profiling map") + } + + ifaces, err := net.Interfaces() + if err != nil { + log.WithError(err).Error("Failed to list network interfaces") + } + + table := tablewriter.NewWriter(cmd.OutOrStdout()) + table.SetHeader([]string{"IFACE", "INGRESS new", "#", "INGRESS est", "#", "EGRESS new", "#", "EGRESS ets", "#"}) + + var rows [][]string + + for _, i := range ifaces { + k := profiling.Key{ + Ifindex: i.Index, + } + + r := []string{i.Name} + hit := false + + for kind := 0; kind < 4; kind++ { + k.Kind = kind + v, ok := data[k] + if !ok { + r = append(r, "---", "---") + continue + } + + hit = true + + if v.Samples != 0 { + r = append(r, fmt.Sprintf("%.3f ns", float64(v.Time)/float64(v.Samples)), fmt.Sprintf("%d", v.Samples)) + } else { + r = append(r, "---", "0") + } + } + + if hit { + rows = append(rows, r) + } + } + + table.AppendBulk(rows) + table.SetAutoMergeCells(true) + table.SetAutoMergeCellsByColumnIndex([]int{0}) + table.Render() +} diff --git a/felix/config/config_params.go b/felix/config/config_params.go index 5fd538eec9d..e8a5b2714e7 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022 Tigera, Inc. All rights reserved. +// Copyright (c) 2020-2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -209,6 +209,7 @@ type Config struct { BPFDisableGROForIfaces *regexp.Regexp `config:"regexp;"` BPFExcludeCIDRsFromNAT []string `config:"cidr-list;;"` BPFRedirectToPeer string `config:"oneof(Disabled,Enabled,L2Only);L2Only;non-zero"` + BPFProfiling string `config:"oneof(Disabled,Enabled);Disabled;non-zero"` // DebugBPFCgroupV2 controls the cgroup v2 path that we apply the connect-time load balancer to. Most distros // are configured for cgroup v1, which prevents all but the root cgroup v2 from working so this is only useful diff --git a/felix/dataplane/driver.go b/felix/dataplane/driver.go index 03258ba1859..c68c311dd5a 100644 --- a/felix/dataplane/driver.go +++ b/felix/dataplane/driver.go @@ -384,6 +384,7 @@ func StartDataplaneDriver( MTUIfacePattern: configParams.MTUIfacePattern, BPFExcludeCIDRsFromNAT: configParams.BPFExcludeCIDRsFromNAT, BPFRedirectToPeer: configParams.BPFRedirectToPeer, + BPFProfiling: configParams.BPFProfiling, ServiceLoopPrevention: configParams.ServiceLoopPrevention, KubeClientSet: k8sClientSet, diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index e2bbfd512e3..7b4d00dc1af 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -367,6 +367,7 @@ type bpfEndpointManager struct { services map[serviceKey][]ip.CIDR dirtyServices set.Set[serviceKey] natExcludedCIDRs *ip.CIDRTrie + profiling string // Maps for policy rule counters polNameToMatchIDs map[string]set.Set[polprog.RuleMatchID] @@ -526,6 +527,7 @@ func newBPFEndpointManager( healthAggregator: healthAggregator, features: dataplanefeatures, + profiling: config.BPFProfiling, } specialInterfaces := []string{"egress.calico"} @@ -2910,6 +2912,7 @@ func (m *bpfEndpointManager) calculateTCAttachPoint(ifaceName string) *tc.Attach ap.PSNATStart = m.psnatPorts.MinPort ap.PSNATEnd = m.psnatPorts.MaxPort ap.TunnelMTU = uint16(m.vxlanMTU) + ap.Profiling = m.profiling switch m.rpfEnforceOption { case "Strict": diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go index 9ccafd9fa7d..be605ea1a2d 100644 --- a/felix/dataplane/linux/int_dataplane.go +++ b/felix/dataplane/linux/int_dataplane.go @@ -229,6 +229,7 @@ type Config struct { BPFDisableGROForIfaces *regexp.Regexp BPFExcludeCIDRsFromNAT []string BPFRedirectToPeer string + BPFProfiling string KubeProxyMinSyncPeriod time.Duration SidecarAccelerationEnabled bool ServiceLoopPrevention string diff --git a/felix/docs/config-params.json b/felix/docs/config-params.json index 58bc28c2999..a8562b0b545 100644 --- a/felix/docs/config-params.json +++ b/felix/docs/config-params.json @@ -3443,6 +3443,35 @@ "UserEditable": true, "GoType": "*bool" }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFProfiling", + "NameEnvVar": "FELIX_BPFProfiling", + "NameYAML": "bpfProfiling", + "NameGoAPI": "BPFProfiling", + "StringSchema": "One of: `Disabled`, `Enabled` (case insensitive)", + "StringSchemaHTML": "One of: Disabled, Enabled (case insensitive)", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "`Disabled`", + "`Enabled`" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled.", + "YAMLDefault": "Disabled", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls profiling of BPF programs. At the monent, it can be Disabled or Enabled.", + "DescriptionHTML": "

Controls profiling of BPF programs. At the monent, it can be Disabled or Enabled.

", + "UserEditable": true, + "GoType": "string" + }, { "Group": "Dataplane: eBPF", "GroupWithSortPrefix": "22 Dataplane: eBPF", diff --git a/felix/docs/config-params.md b/felix/docs/config-params.md index 9d879875425..fac6af83852 100644 --- a/felix/docs/config-params.md +++ b/felix/docs/config-params.md @@ -1723,6 +1723,20 @@ When true, Felix records detailed information about the BPF policy programs, whi | `FelixConfiguration` schema | Boolean. | | Default value (YAML) | `true` | +### `BPFProfiling` (config file) / `bpfProfiling` (YAML) + +Controls profiling of BPF programs. At the monent, it can be Disabled or Enabled. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_BPFProfiling` | +| Encoding (env var/config file) | One of: Disabled, Enabled (case insensitive) | +| Default value (above encoding) | `Disabled` | +| `FelixConfiguration` field | `bpfProfiling` (YAML) `BPFProfiling` (Go API) | +| `FelixConfiguration` schema | One of: Disabled, Enabled. | +| Default value (YAML) | `Disabled` | +| Notes | Required. | + ### `BPFRedirectToPeer` (config file) / `bpfRedirectToPeer` (YAML) Controls which whether it is allowed to forward straight to the peer side of the workload devices. It is allowed for any host L2 devices by default (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses it on ingress. Value of Enabled also allows redirection from L3 host devices like IPIP tunnel or Wireguard directly to the peer side of the workload's device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. diff --git a/felix/fv/bpf_test.go b/felix/fv/bpf_test.go index 31f3f65db8b..c43fdb1b9df 100644 --- a/felix/fv/bpf_test.go +++ b/felix/fv/bpf_test.go @@ -408,6 +408,7 @@ func describeBPFTests(opts ...bpfTestOpt) bool { options.ExtraEnvVars["FELIX_BPFConntrackCleanupMode"] = testOpts.conntrackCleanupMode options.ExtraEnvVars["FELIX_BPFLogLevel"] = fmt.Sprint(testOpts.bpfLogLevel) options.ExtraEnvVars["FELIX_BPFConntrackLogLevel"] = fmt.Sprint(testOpts.bpfLogLevel) + options.ExtraEnvVars["FELIX_BPFProfiling"] = "Enabled" if testOpts.dsr { options.ExtraEnvVars["FELIX_BPFExternalServiceMode"] = "dsr" } diff --git a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml index 2a1fa1a4460..39d05ea0393 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_felixconfigurations.yaml @@ -296,6 +296,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 8568bdead49..6891d04ebd9 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -1304,6 +1304,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 092b1bfb4e5..e947921f8e3 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -1314,6 +1314,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 46e527e2812..82377f839cd 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -1315,6 +1315,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 53b941141cd..28323699471 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -1299,6 +1299,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/calico.yaml b/manifests/calico.yaml index b6fa2d939f4..d9e6f31c6e6 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -1299,6 +1299,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 7ff0418d403..9f2034a8b7a 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -1316,6 +1316,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/crds.yaml b/manifests/crds.yaml index adde581cc25..dba833be639 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -1209,6 +1209,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 6895ef27ac9..dd4a603f007 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -1299,6 +1299,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It diff --git a/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml new file mode 100644 index 00000000000..90ad053b7a4 --- /dev/null +++ b/manifests/ocp/crd.projectcalico.org_felixconfigurations.yaml @@ -0,0 +1,1106 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads. + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads. + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'AWSSrcDstCheck controls whether Felix will try to change + the "source/dest check" setting on the EC2 instance on which it + is running. A value of "Disable" will try to disable the source/dest + check. Disabling the check allows for sending workload traffic without + encapsulation within the same AWS subnet. [Default: DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfCTLBLogFilter: + description: 'BPFCTLBLogFilter specifies, what is logged by connect + time load balancer when BPFLogLevel is debug. Currently has to be + specified as ''all'' when BPFLogFilters is set to see CTLB logs. + [Default: unset - means logs are emitted when BPFLogLevel id debug + and BPFLogFilters not set.]' + type: string + bpfConnectTimeLoadBalancing: + description: 'BPFConnectTimeLoadBalancing when in BPF mode, controls + whether Felix installs the connect-time load balancer. The connect-time + load balancer is required for the host to be able to reach Kubernetes + services and it improves the performance of pod-to-service connections.When + set to TCP, connect time load balancing is available only for services + with TCP ports. [Default: TCP]' + enum: + - TCP + - Enabled + - Disabled + type: string + bpfConnectTimeLoadBalancingEnabled: + description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. + \ The connect-time load balancer is required for the host to be + able to reach Kubernetes services and it improves the performance + of pod-to-service connections. The only reason to disable it is + for debugging purposes. \n Deprecated: Use BPFConnectTimeLoadBalancing + [Default: true]" + type: boolean + bpfConntrackLogLevel: + description: 'BPFConntrackLogLevel controls the log level of the BPF + conntrack cleanup program, which runs periodically to clean up expired + BPF conntrack entries. [Default: Off].' + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: 'BPFConntrackCleanupMode controls how BPF conntrack entries + are cleaned up. `Auto` will use a BPF program if supported, falling + back to userspace if not. `Userspace` will always use the userspace + cleanup code. `BPFProgram` will always use the BPF program (failing + if not supported). [Default: Auto]' + enum: + - Auto + - Userspace + - BPFProgram + type: string + bpfDSROptoutCIDRs: + description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded + from DSR. That is, clients in those CIDRs will access service node + ports as if BPFExternalServiceMode was set to Tunnel. + items: + type: string + type: array + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...) or any other special device managed by Calico itself + (e.g., tunnels). + type: string + bpfDisableGROForIfaces: + description: BPFDisableGROForIfaces is a regular expression that controls + which interfaces Felix should disable the Generic Receive Offload + [GRO] option. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all host interfaces + with BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled, Strict or Loose. [Default: + Loose]' + pattern: ^(?i)(Disabled|Strict|Loose)?$ + type: string + bpfExcludeCIDRsFromNAT: + description: BPFExcludeCIDRsFromNAT is a list of CIDRs that are to + be excluded from NAT resolution so that host can handle them. A + typical usecase is node local DNS cache. + items: + type: string + type: array + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, controls a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing interpreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + pattern: ^(?i)(Tunnel|DSR)?$ + type: string + bpfForceTrackPacketsFromIfaces: + description: 'BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic + from these interfaces to skip Calico''s iptables NOTRACK rule, allowing + traffic from those interfaces to be tracked by Linux conntrack. Should + only be used for interfaces that are not used for the Calico fabric. For + example, a docker bridge device for non-Calico-networked containers. + [Default: docker+]' + items: + type: string + type: array + bpfHostConntrackBypass: + description: 'BPFHostConntrackBypass Controls whether to bypass Linux + conntrack in BPF mode for workloads and services. [Default: true + - bypass Linux conntrack]' + type: boolean + bpfHostNetworkedNATWithoutCTLB: + description: 'BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls + whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing + determines the CTLB behavior. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled is deprecated and has + no effect. BPF kube-proxy always accepts endpoint slices. This option + will be removed in the next release. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + bpfL3IfacePattern: + description: BPFL3IfacePattern is a regular expression that allows + to list tunnel devices like wireguard or vxlan (i.e., L3 devices) + in addition to BPFDataIfacePattern. That is, tunnel interfaces not + created by Calico, that Calico workload traffic flows over as well + as any interfaces that handle incoming traffic to nodeports and + services from outside the cluster. + type: string + bpfLogFilters: + additionalProperties: + type: string + description: "BPFLogFilters is a map of key=values where the value + is a pcap filter expression and the key is an interface name with + 'all' denoting all interfaces, 'weps' all workload endpoints and + 'heps' all host endpoints. \n When specified as an env var, it accepts + a comma-separated list of key=values. [Default: unset - means all + debug logs are emitted]" + type: object + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + pattern: ^(?i)(Off|Info|Debug)?$ + type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeConntrackCleanupQueue: + description: BPFMapSizeConntrackCleanupQueue sets the size for the + map used to hold NAT conntrack entries that are queued for cleanup. This + should be big enough to hold all the NAT entries that expire within + one cleanup interval. + minimum: 1 + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + description: BPFMapSizeNATAffinity sets the size of the BPF map that + stores the affinity of a connection (for services that enable that + feature. + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for NAT back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for NAT front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string + bpfRedirectToPeer: + description: 'BPFRedirectToPeer controls which whether it is allowed + to forward straight to the peer side of the workload devices. It + is allowed for any host L2 devices by default (L2Only), but it breaks + TCP dump on the host side of workload device as it bypasses it on + ingress. Value of Enabled also allows redirection from L3 host devices + like IPIP tunnel or Wireguard directly to the peer side of the workload''s + device. This makes redirection faster, however, it breaks tools + like tcpdump on the peer side. Use Enabled with caution. [Default: + L2Only]' + enum: + - Enabled + - Disabled + - L2Only + type: string + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + pattern: ^(?i)(Insert|Append)?$ + type: string + dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Deprecated: replaced + by the generic HealthTimeoutOverrides.' + type: string + debugDisableLogDropping: + description: 'DebugDisableLogDropping disables the dropping of log + messages when the log buffer is full. This can significantly impact + performance if log write-out is a bottleneck. [Default: false]' + type: boolean + debugHost: + description: DebugHost is the host IP or hostname to bind the debug + port to. Only used if DebugPort is set. [Default:localhost] + type: string + debugMemoryProfilePath: + description: DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. + type: string + debugPort: + description: DebugPort if set, enables Felix's debug HTTP port, which + allows memory and CPU profiles to be retrieved. The debug port + is not secure, it should not be exposed to the internet. + type: integer + debugSimulateCalcGraphHangAfter: + description: DebugSimulateCalcGraphHangAfter is used to simulate a + hang in the calculation graph after the specified duration. This + is useful in tests of the watchdog system only! + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + debugSimulateDataplaneApplyDelay: + description: DebugSimulateDataplaneApplyDelay adds an artificial delay + to every dataplane operation. This is useful for simulating a heavily + loaded system for test purposes only. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + debugSimulateDataplaneHangAfter: + description: DebugSimulateDataplaneHangAfter is used to simulate a + hang in the dataplane after the specified duration. This is useful + in tests of the watchdog system only! + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the endpoint''s egress policy is applied). By default, Calico blocks + traffic from workload endpoints to the host itself with an iptables + "DROP" action. If you want to allow some or all traffic from endpoint + to host, set this parameter to RETURN or ACCEPT. Use RETURN if you + have your own rules in the iptables "INPUT" chain; Calico will insert + its rules at the top of that chain, then "RETURN" packets to the + "INPUT" chain once it has completed processing workload endpoint + egress policy. Use ACCEPT to unconditionally accept packets from + workloads after processing workload endpoint egress policy. [Default: + Drop]' + pattern: ^(?i)(Drop|Accept|Return)?$ + type: string + deviceRouteProtocol: + description: DeviceRouteProtocol controls the protocol to set on routes + programmed by Felix. The protocol is an 8-bit label used to identify + the owner of the route. + type: integer + deviceRouteSourceAddress: + description: DeviceRouteSourceAddress IPv4 address to set as the source + hint for routes programmed by Felix. When not set the source address + for local traffic from host to workload will be determined by the + kernel. + type: string + deviceRouteSourceAddressIPv6: + description: DeviceRouteSourceAddressIPv6 IPv6 address to set as the + source hint for routes programmed by Felix. When not set the source + address for local traffic from host to workload will be determined + by the kernel. + type: string + disableConntrackInvalidCheck: + description: DisableConntrackInvalidCheck disables the check for invalid + connections in conntrack. While the conntrack invalid check helps + to detect malicious traffic, it can also cause issues with certain + multi-NIC scenarios. + type: boolean + endpointReportingDelay: + description: 'EndpointReportingDelay is the delay before Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + endpointReportingEnabled: + description: 'EndpointReportingEnabled controls whether Felix reports + endpoint status to the datastore. This is only used by the OpenStack + integration. [Default: false]' + type: boolean + endpointStatusPathPrefix: + description: "EndpointStatusPathPrefix is the path to the directory + where endpoint status will be written. Endpoint status file reporting + is disabled if field is left empty. \n Chosen directory should match + the directory used by the CNI plugin for PodStartupDelay. [Default: + \"\"]" + type: string + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external, + non-Calico nodes from which VXLAN/IPIP overlay traffic will be allowed. By + default, external tunneled traffic is blocked to reduce attack surface. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of ProtoPort struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + incoming traffic to host endpoints on irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable + all inbound host ports, use the value "[]". The default value allows + ssh access, DHCP, BGP, etcd and the Kubernetes API. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, + tcp:6667 ]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of PortProto struct + objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow + outgoing traffic from host endpoints to irrespective of the security + policy. This is useful to avoid accidentally cutting off a host + with incorrect configuration. For backwards compatibility, if the + protocol is not specified, it defaults to "tcp". If a CIDR is not + specified, it will allow traffic from all addresses. To disable + all outbound host ports, use the value "[]". The default value opens + etcd''s standard ports to ensure that Felix does not get cut off + from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes + API. [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473, + tcp:6443, tcp:6666, tcp:6667 ]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override feature detection + based on auto-detected platform capabilities. Values are specified + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + A value of "true" or "false" will force enable/disable feature, + empty or omitted values fall back to auto-detection. + pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ + type: string + featureGates: + description: FeatureGates is used to enable or disable tech-preview + Calico features. Values are specified in a comma separated list + with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". + This is used to enable features that are not fully production ready. + pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$ + type: string + floatingIPs: + description: FloatingIPs configures whether or not Felix will program + non-OpenStack floating IP addresses. (OpenStack-derived floating + IPs are always programmed, regardless of this setting.) + enum: + - Enabled + - Disabled + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + goGCThreshold: + description: "GoGCThreshold Sets the Go runtime's garbage collection + threshold. I.e. the percentage that the heap is allowed to grow + before garbage collection is triggered. In general, doubling the + value halves the CPU time spent doing GC, but it also doubles peak + GC memory overhead. A special value of -1 can be used to disable + GC entirely; this should only be used in conjunction with the GoMemoryLimitMB + setting. \n This setting is overridden by the GOGC environment variable. + \n [Default: 40]" + type: integer + goMaxProcs: + description: "GoMaxProcs sets the maximum number of CPUs that the + Go runtime will use concurrently. A value of -1 means \"use the + system default\"; typically the number of real CPUs on the system. + \n this setting is overridden by the GOMAXPROCS environment variable. + \n [Default: -1]" + type: integer + goMemoryLimitMB: + description: "GoMemoryLimitMB sets a (soft) memory limit for the Go + runtime in MB. The Go runtime will try to keep its memory usage + under the limit by triggering GC as needed. To avoid thrashing, + it will exceed the limit if GC starts to take more than 50% of the + process's CPU time. A value of -1 disables the memory limit. \n + Note that the memory limit, if used, must be considerably less than + any hard resource limit set at the container or pod level. This + is because felix is not the only process that must run in the container + or pod. \n This setting is overridden by the GOMEMLIMIT environment + variable. \n [Default: -1]" + type: integer + healthEnabled: + description: 'HealthEnabled if set to true, enables Felix''s health + port, which provides readiness and liveness endpoints. [Default: + false]' + type: boolean + healthHost: + description: 'HealthHost is the host that the health server should + bind to. [Default: localhost]' + type: string + healthPort: + description: 'HealthPort is the TCP port that the health server should + bind to. [Default: 9099]' + type: integer + healthTimeoutOverrides: + description: HealthTimeoutOverrides allows the internal watchdog timeouts + of individual subcomponents to be overridden. This is useful for + working around "false positive" liveness timeouts that can occur + in particularly stressful workloads or if CPU is constrained. For + a list of active subcomponents, see Felix's logs. + items: + properties: + name: + type: string + timeout: + type: string + required: + - name + - timeout + type: object + type: array + interfaceExclude: + description: 'InterfaceExclude A comma-separated list of interface + names that should be excluded when Felix is resolving host endpoints. + The default value ensures that Felix ignores Kubernetes'' internal + `kube-ipvs0` device. If you want to exclude multiple interface names + using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example + having values `/^kube/,veth1` will exclude all interfaces that begin + with `kube` and also the interface `veth1`. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipForwarding: + description: 'IPForwarding controls whether Felix sets the host sysctls + to enable IP forwarding. IP forwarding is required when using Calico + for workload networking. This should be disabled only on hosts + where Calico is used solely for host protection. In BPF mode, due + to a kernel interaction, either IPForwarding must be enabled or + BPFEnforceRPF must be disabled. [Default: Enabled]' + enum: + - Enabled + - Disabled + type: string + ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' + type: boolean + ipipMTU: + description: 'IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional + as Felix auto-detects the MTU based on the MTU of the host''s interfaces. + [Default: 0 (auto-detect)]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval controls the period at which Felix + re-checks all IP sets to look for discrepancies. Set to 0 to disable + the periodic refresh. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesBackend: + description: "IptablesBackend controls which backend of iptables will + be used. The default is `Auto`. \n Warning: changing this on a running + system can leave \"orphaned\" rules in the \"other\" backend. These + should be cleaned up to avoid confusing interactions." + pattern: ^(?i)(Auto|Legacy|NFT)?$ + type: string + iptablesFilterAllowAction: + description: IptablesFilterAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables filter + table (which is used for "normal" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesFilterDenyAction: + description: IptablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default Calico blocks traffic + with an iptables "DROP" action. If you want to use "REJECT" action + instead you can configure it in here. + pattern: ^(?i)(Drop|Reject)?$ + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval when IptablesLockTimeout is + enabled: the time that Felix will wait between attempts to acquire + the iptables lock if it is not available. Lower values make Felix + more responsive when the lock is contended, but use more CPU. [Default: + 50ms]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesLockTimeout: + description: "IptablesLockTimeout is the time that Felix itself will + wait for the iptables lock (rather than delegating the lock handling + to the `iptables` command). \n Deprecated: `iptables-restore` v1.8+ + always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled]" + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesMangleAllowAction: + description: IptablesMangleAllowAction controls what happens to traffic + that is accepted by a Felix policy chain in the iptables mangle + table (which is used for "pre-DNAT" policy). The default will immediately + `Accept` the traffic. Use `Return` to send the traffic back up to + the system chains for further processing. + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + description: 'This parameter can be used to limit the host interfaces + on which Calico will apply SNAT to traffic leaving a Calico IPAM + pool with "NAT outgoing" enabled. This can be useful if you have + a main data interface, where traffic should be SNATted and a secondary + device (such as the docker bridge) which is local to the host and + doesn''t require SNAT. This parameter uses the iptables interface + matching syntax, which allows + as a wildcard. Most users will not + need to set this. Example: if your data interfaces are eth0 and + eth1 and you want to exclude the docker bridge, you could set this + to eth+' + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ + type: string + maxIpsetSize: + description: MaxIpsetSize is the maximum number of IP addresses that + can be stored in an IP set. Not applicable if using the nftables + backend. + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case-insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (i.e. it uses the iptables MASQUERADE + target). + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + description: 'NetlinkTimeout is the timeout when talking to the kernel + over the netlink protocol, used for programming routes, rules, and + other kernel objects. [Default: 10s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + nftablesFilterAllowAction: + description: NftablesFilterAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the filter + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. + pattern: ^(?i)(Accept|Return)?$ + type: string + nftablesFilterDenyAction: + description: NftablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default, Calico blocks traffic + with a "drop" action. If you want to use a "reject" action instead + you can configure it here. + pattern: ^(?i)(Drop|Reject)?$ + type: string + nftablesMangleAllowAction: + description: NftablesMangleAllowAction controls the nftables action + that Felix uses to represent the "allow" policy verdict in the mangle + table. The default is to `ACCEPT` the traffic, which is a terminal + action. Alternatively, `RETURN` can be used to return the traffic + back to the top-level chain for further processing by your rules. + pattern: ^(?i)(Accept|Return)?$ + type: string + nftablesMarkMask: + description: 'NftablesMarkMask is the mask that Felix selects its + nftables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xffff0000]' + format: int32 + type: integer + nftablesMode: + description: 'NFTablesMode configures nftables support in Felix. [Default: + Disabled]' + enum: + - Disabled + - Enabled + - Auto + type: string + nftablesRefreshInterval: + description: 'NftablesRefreshInterval controls the interval at which + Felix periodically refreshes the nftables rules. [Default: 90s]' + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusWireGuardMetricsEnabled: + description: 'PrometheusWireGuardMetricsEnabled disables wireguard + metrics collection, which the Prometheus client does by default, + when set to false. This reduces the number of metrics reported, + reducing Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: RemoveExternalRoutes Controls whether Felix will remove + unexpected routes to workload interfaces. Felix will always clean + up expected routes that use the configured DeviceRouteProtocol. To + add your own routes, you must use a distinct protocol (in addition + to setting this field to false). + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$ + type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean + routeTableRange: + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + pattern: ^(?i)(Drop|Reject|Disabled)?$ + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix + determines this based on the existing IP pools. [Default: nil (unset)]' + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. Optional as Felix auto-detects the MTU based on the MTU + of the host''s interfaces. [Default: 0 (auto-detect)]' + type: integer + vxlanPort: + description: 'VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]' + type: integer + vxlanVNI: + description: 'VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You + may need to change this if the default value is in use on your system. + [Default: 4096]' + type: integer + windowsManageFirewallRules: + description: 'WindowsManageFirewallRules configures whether or not + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]' + type: boolean + wireguardHostEncryptionEnabled: + description: 'WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' + type: string + wireguardKeepAlive: + description: 'WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + wireguardThreadingEnabled: + description: 'WireguardThreadingEnabled controls whether Wireguard + has NAPI threading enabled. [Default: false]' + type: boolean + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + pattern: ^(?i)(Disabled|Any)?$ + type: string + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index a1293be47b6..500d1d5c470 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -19835,6 +19835,13 @@ spec: information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean + bpfProfiling: + description: 'BPFProfiling controls profiling of BPF programs. At + the monent, it can be Disabled or Enabled. [Default: Disabled]' + enum: + - Enabled + - Disabled + type: string bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It