Enhance conntrack map flexibility with CPU-based scaling

Previously, BPFMapSizeConntrack was set to a fixed value, which lacked
flexibility across different spec of machines. Now, with the introduction
of BPFMapSizePerCPUConntrack, the conntrack map size can be adjusted
dynamically as BPFMapSizeConntrackPerCPU * (Number of CPUs). This allows
for larger map sizes on high-spec machines and smaller map sizes on
lower-spec machines, optimizing resource usage accordingly.

BTW, recently, we added several high-spec machines to the data center,
which led to the conntrack map size being filled up on these machines. It
could have been avoided with BPFMapSizePerCPUConntrack IMHO.

Suggested-by: Tomas Hruby <[email protected]>
Signed-off-by: Mingzhe Yang <[email protected]>
Co-Authored-By: Amaindex <[email protected]>
Co-Authored-By: Lance Yang <[email protected]>

api/pkg/apis/projectcalico/v3/felixconfig.go

@@ -724,6 +724,12 @@ type FelixConfigurationSpec struct {
 	// an entry for each active connection.  Warning: changing the size of the conntrack map can cause disruption.
 	BPFMapSizeConntrack *int `json:"bpfMapSizeConntrack,omitempty"`
 
+	// BPFMapSizePerCPUConntrack determines the size of conntrack map based on the number of CPUs. If set to a
+	// non-zero value, overrides BPFMapSizeConntrack with `BPFMapSizePerCPUConntrack * (Number of CPUs)`.
+	// This map must be large enough to hold an entry for each active connection.  Warning: changing the size of the
+	// conntrack map can cause disruption.
+	BPFMapSizePerCPUConntrack *int `json:"bpfMapSizePerCpuConntrack,omitempty"`
+
 	// BPFMapSizeConntrackCleanupQueue sets the size for the map used to hold NAT conntrack entries that are queued
 	// for cleanup.  This should be big enough to hold all the NAT entries that expire within one cleanup interval.
 	// +kubebuilder:validation:Minimum=1

felix/config/config_params.go

@@ -199,6 +199,7 @@ type Config struct {
 	BPFMapSizeNATAffinity              int               `config:"int;65536;non-zero"`
 	BPFMapSizeRoute                    int               `config:"int;262144;non-zero"`
 	BPFMapSizeConntrack                int               `config:"int;512000;non-zero"`
+	BPFMapSizePerCPUConntrack          int               `config:"int;0"`
 	BPFMapSizeConntrackCleanupQueue    int               `config:"int;100000;non-zero"`
 	BPFMapSizeIPSets                   int               `config:"int;1048576;non-zero"`
 	BPFMapSizeIfState                  int               `config:"int;1000;non-zero"`

felix/dataplane/driver.go

@@ -371,6 +371,7 @@ func StartDataplaneDriver(
 			BPFMapSizeNATBackend:               configParams.BPFMapSizeNATBackend,
 			BPFMapSizeNATAffinity:              configParams.BPFMapSizeNATAffinity,
 			BPFMapSizeConntrack:                configParams.BPFMapSizeConntrack,
+			BPFMapSizePerCPUConntrack:          configParams.BPFMapSizePerCPUConntrack,
 			BPFMapSizeConntrackCleanupQueue:    configParams.BPFMapSizeConntrackCleanupQueue,
 			BPFMapSizeIPSets:                   configParams.BPFMapSizeIPSets,
 			BPFMapSizeIfState:                  configParams.BPFMapSizeIfState,

felix/dataplane/linux/int_dataplane.go

@@ -217,6 +217,7 @@ type Config struct {
 	BPFPSNATPorts                      numorstring.Port
 	BPFMapSizeRoute                    int
 	BPFMapSizeConntrack                int
+	BPFMapSizePerCPUConntrack          int
 	BPFMapSizeConntrackCleanupQueue    int
 	BPFMapSizeNATFrontend              int
 	BPFMapSizeNATBackend               int
@@ -789,10 +790,15 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane {
 		bpfmaps.DisableRepin()
 	}
 
+	bpfMapSizeConntrack := config.BPFMapSizeConntrack
+	if config.BPFMapSizePerCPUConntrack > 0 {
+		bpfMapSizeConntrack = config.BPFMapSizePerCPUConntrack * bpfmaps.NumPossibleCPUs()
+	}
+
 	bpfipsets.SetMapSize(config.BPFMapSizeIPSets)
 	bpfnat.SetMapSizes(config.BPFMapSizeNATFrontend, config.BPFMapSizeNATBackend, config.BPFMapSizeNATAffinity)
 	bpfroutes.SetMapSize(config.BPFMapSizeRoute)
-	bpfconntrack.SetMapSize(config.BPFMapSizeConntrack)
+	bpfconntrack.SetMapSize(bpfMapSizeConntrack)
 	bpfconntrack.SetCleanupMapSize(config.BPFMapSizeConntrackCleanupQueue)
 	bpfifstate.SetMapSize(config.BPFMapSizeIfState)