Skip to content

Commit

Permalink
Create a common CNI interface and config struct
Browse files Browse the repository at this point in the history 
Signed-off-by: Manuel Buil <[email protected]>
  • Loading branch information
@manuelbuil
manuelbuil committed Jan 16, 2024
1 parent bfcf0ad commit 49ef16c
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 98 deletions.
29 changes: 16 additions & 13 deletions pkg/pebinaryexecutor/pebinary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ type PEBinaryConfig struct {
KubeConfigKubeProxy string
DisableETCD bool
IsServer bool
CNI string
CNIConfig win.Calico
CNIName string
CniPlugin win.CniPlugin
}

type CloudProviderConfig struct {
Expand Down Expand Up @@ -105,20 +105,21 @@ func (p *PEBinaryConfig) Bootstrap(ctx context.Context, nodeConfig *daemonconfig

restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)

p.CNI, err = getCniType(restConfig)
p.CNIName, err = getCniPluginName(restConfig)
if err != nil {
return err
}

switch p.CNI {
switch p.CNIName {
case "", CNICalico:
if err := p.CNIConfig.Setup(ctx, nodeConfig, restConfig, p.DataDir); err != nil {
p.CniPlugin = &win.Calico{}
if err := p.CniPlugin.Setup(ctx, nodeConfig, restConfig, p.DataDir); err != nil {
return err
}
case CNINone:
logrus.Info("Skipping CNI setup")
default:
logrus.Fatal("Unsupported CNI: ", p.CNI)
logrus.Fatal("Unsupported CNI: ", p.CNIName)
}

// required to initialize KubeProxy
Expand Down Expand Up @@ -158,9 +159,9 @@ func (p *PEBinaryConfig) Kubelet(ctx context.Context, args []string) error {
go func() {
for {
cniCtx, cancel := context.WithCancel(ctx)
if p.CNI != CNINone {
if p.CNIName != CNINone {
go func() {
if err := p.CNIConfig.Start(cniCtx); err != nil {
if err := p.CniPlugin.Start(cniCtx); err != nil {
logrus.Errorf("error in cni start: %s", err)
}
}()
Expand All @@ -181,13 +182,15 @@ func (p *PEBinaryConfig) Kubelet(ctx context.Context, args []string) error {

// KubeProxy starts the kubeproxy in a subprocess with watching goroutine.
func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error {
if p.CNI == CNINone {
if p.CNIName == CNINone {
return nil
}

CNIConfig := p.CniPlugin.GetConfig()

extraArgs := map[string]string{
"network-name": p.CNIConfig.CNICfg.OverlayNetName,
"bind-address": p.CNIConfig.CNICfg.IP,
"network-name": CNIConfig.OverlayNetName,
"bind-address": CNIConfig.NodeIP,
}

if err := hcn.DSRSupported(); err == nil {
Expand All @@ -196,7 +199,7 @@ func (p *PEBinaryConfig) KubeProxy(ctx context.Context, args []string) error {
extraArgs["enable-dsr"] = "true"
}

if p.CNIConfig.CNICfg.Name == "Calico" {
if CNIConfig.Name == "Calico" {
var vip string
for range time.Tick(time.Second * 5) {
endpoint, err := hcsshim.GetHNSEndpointByName("Calico_ep")
Expand Down Expand Up @@ -288,7 +291,7 @@ func getArgs(argsMap map[string]string) []string {
return args
}

func getCniType(restConfig *rest.Config) (string, error) {
func getCniPluginName(restConfig *rest.Config) (string, error) {
hc, err := helm.NewFactoryFromConfig(restConfig)
if err != nil {
return "", err
Expand Down
2 changes: 1 addition & 1 deletion pkg/rke2/rke2_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,6 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto
KubeletPath: cfg.KubeletPath,
DisableETCD: clx.Bool("disable-etcd"),
IsServer: isServer,
CNI: "",
CNIName: "",
}, nil
}
108 changes: 55 additions & 53 deletions pkg/windows/calico.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var (
},
}

calicoKubeConfigTemplate = template.Must(template.New("CalicoKubeconfig").Parse(`apiVersion: v1
calicoKubeConfigTemplate = template.Must(template.New("Kubeconfig").Parse(`apiVersion: v1
kind: Config
clusters:
- name: kubernetes
Expand All @@ -61,11 +61,11 @@ users:
calicoConfigTemplate = template.Must(template.New("CalicoConfig").Funcs(replaceSlashWin).Parse(`{
"name": "{{ .Name }}",
"windows_use_single_network": true,
"cniVersion": "{{ .CNI.Version }}",
"cniVersion": "{{ .CNIVersion }}",
"type": "calico",
"mode": "{{ .Mode }}",
"vxlan_mac_prefix": "{{ .Felix.MacPrefix }}",
"vxlan_vni": {{ .Felix.Vxlanvni }},
"mode": "{{ .OverlayEncap }}",
"vxlan_mac_prefix": "0E-2A",
"vxlan_vni": {{ .VxlanVNI }},
"policy": {
"type": "k8s"
},
Expand All @@ -89,7 +89,7 @@ users:
"kubeconfig": "{{ replace .KubeConfig.Path }}"
},
"ipam": {
"type": "{{ .CNI.IpamType }}",
"type": "{{ .IpamType }}",
"subnet": "usePodCidr"
},
"policies": [
Expand All @@ -107,7 +107,7 @@ users:
"Value": {
"Type": "SDNROUTE",
"DestinationPrefix": "{{ .ServiceCIDR }}",
{{- if eq .Mode "vxlan" }}
{{- if eq .OverlayEncap "vxlan" }}
"NeedEncap": true
{{- else }}
"NeedEncap": false
Expand All @@ -120,8 +120,7 @@ users:
)

type Calico struct {
CNICfg *CalicoConfig
DataDir string
CNICfg *CalicoConfig
}

const (
Expand All @@ -134,19 +133,21 @@ const (
calicoNode = "calico-node"
)

func (c *Calico) GetConfig() *CNICommonConfig {
return &c.CNICfg.CNICommonConfig
}

// Setup creates the basic configuration required by the CNI.
func (c *Calico) Setup(ctx context.Context, nodeConfig *daemonconfig.Node, restConfig *rest.Config, dataDir string) error {
c.DataDir = dataDir

if err := c.initializeConfig(ctx, nodeConfig, restConfig); err != nil {
if err := c.initializeConfig(ctx, nodeConfig, restConfig, dataDir); err != nil {
return err
}

if err := c.overrideCalicoConfigByHelm(restConfig); err != nil {
return err
}

if err := c.writeConfigFiles(nodeConfig.AgentConfig.CNIConfDir, nodeConfig.AgentConfig.NodeName); err != nil {
if err := c.writeConfigFiles(); err != nil {
return err
}

Expand All @@ -155,60 +156,61 @@ func (c *Calico) Setup(ctx context.Context, nodeConfig *daemonconfig.Node, restC
}

// initializeConfig sets the default configuration in CNIConfig
func (c *Calico) initializeConfig(ctx context.Context, nodeConfig *daemonconfig.Node, restConfig *rest.Config) error {
func (c *Calico) initializeConfig(ctx context.Context, nodeConfig *daemonconfig.Node, restConfig *rest.Config, dataDir string) error {
platformType, err := platformType()
if err != nil {
return err
}

c.CNICfg = &CalicoConfig{
Name: "Calico",
OverlayNetName: "Calico",
Hostname: nodeConfig.AgentConfig.NodeName,
NodeNameFile: filepath.Join("c:\\", c.DataDir, "agent", CalicoNodeNameFileName),
CNICommonConfig: CNICommonConfig{
Name: "Calico",
OverlayNetName: "Calico",
OverlayEncap: "vxlan",
Hostname: nodeConfig.AgentConfig.NodeName,
ConfigPath: filepath.Join("c:\\", dataDir, "agent"),
CNIConfDir: nodeConfig.AgentConfig.CNIConfDir,
CNIBinDir: nodeConfig.AgentConfig.CNIBinDir,
ClusterCIDR: nodeConfig.AgentConfig.ClusterCIDR.String(),
ServiceCIDR: nodeConfig.AgentConfig.ServiceCIDR.String(),
NodeIP: nodeConfig.AgentConfig.NodeIP,
VxlanVNI: "4096",
VxlanPort: "4789",
IpamType: "calico-ipam",
CNIVersion: "0.3.1",
},
NodeNameFile: filepath.Join("c:\\", dataDir, "agent", CalicoNodeNameFileName),
KubeNetwork: "Calico.*",
Mode: "vxlan",
ServiceCIDR: nodeConfig.AgentConfig.ServiceCIDR.String(),
DNSServers: nodeConfig.AgentConfig.ClusterDNS.String(),
DNSSearch: "svc." + nodeConfig.AgentConfig.ClusterDomain,
DatastoreType: "kubernetes",
Platform: platformType,
IP: nodeConfig.AgentConfig.NodeIP,
IPAutoDetectionMethod: "first-found",
Felix: FelixConfig{
Metadataaddr: "none",
Vxlanvni: "4096",
MacPrefix: "0E-2A",
},
CNI: CalicoCNIConfig{
BinDir: nodeConfig.AgentConfig.CNIBinDir,
ConfDir: nodeConfig.AgentConfig.CNIConfDir,
IpamType: "calico-ipam",
Version: "0.3.1",
},
}

c.CNICfg.KubeConfig, err = c.createKubeConfig(ctx, restConfig)
if err != nil {
return err
}

logrus.Debugf("Calico Config: %+v", c.CNICfg)

return nil
}

// writeConfigFiles writes the three required files by Calico
func (c *Calico) writeConfigFiles(CNIConfDir string, NodeName string) error {
func (c *Calico) writeConfigFiles() error {

// Create CalicoKubeConfig and CIPAutoDetectionMethodalicoConfig files
if err := c.renderCalicoConfig(c.CNICfg.KubeConfig.Path, calicoKubeConfigTemplate); err != nil {
return err
}

if err := c.renderCalicoConfig(filepath.Join(CNIConfDir, CalicoConfigName), calicoConfigTemplate); err != nil {
if err := c.renderCalicoConfig(filepath.Join(c.CNICfg.CNIConfDir, CalicoConfigName), calicoConfigTemplate); err != nil {
return err
}

return os.WriteFile(filepath.Join("c:\\", c.DataDir, "agent", CalicoNodeNameFileName), []byte(NodeName), 0644)
return os.WriteFile(filepath.Join(c.CNICfg.ConfigPath, CalicoNodeNameFileName), []byte(c.CNICfg.Hostname), 0644)
}

// renderCalicoConfig creates the file and then renders the template using Calico Config parameters
Expand All @@ -228,13 +230,13 @@ func (c *Calico) renderCalicoConfig(path string, toRender *template.Template) er
}

// createKubeConfig creates all needed for Calico to contact kube-api
func (c *Calico) createKubeConfig(ctx context.Context, restConfig *rest.Config) (*CalicoKubeConfig, error) {
func (c *Calico) createKubeConfig(ctx context.Context, restConfig *rest.Config) (*KubeConfig, error) {

// Fill all information except for the token
calicoKubeConfig := CalicoKubeConfig{
calicoKubeConfig := KubeConfig{
Server: "https://127.0.0.1:6443",
CertificateAuthority: filepath.Join("c:\\", c.DataDir, "agent", "server-ca.crt"),
Path: filepath.Join("c:\\", c.DataDir, "agent", CalicoKubeConfigName),
CertificateAuthority: filepath.Join(c.CNICfg.ConfigPath, "server-ca.crt"),
Path: filepath.Join(c.CNICfg.ConfigPath, CalicoKubeConfigName),
}

// Generate the token request
Expand Down Expand Up @@ -263,7 +265,7 @@ func (c *Calico) createKubeConfig(ctx context.Context, restConfig *rest.Config)

// Start starts the CNI services on the Windows node.
func (c *Calico) Start(ctx context.Context) error {
logPath := filepath.Join(c.DataDir, "agent", "logs")
logPath := filepath.Join(c.CNICfg.ConfigPath, "logs")
for {
if err := startCalico(ctx, c.CNICfg, logPath); err != nil {
time.Sleep(5 * time.Second)
Expand All @@ -273,7 +275,7 @@ func (c *Calico) Start(ctx context.Context) error {
break
}
go startFelix(ctx, c.CNICfg, logPath)
if c.CNICfg.Mode == "windows-bgp" {
if c.CNICfg.OverlayEncap == "windows-bgp" {
go startConfd(ctx, c.CNICfg, logPath)
}

Expand All @@ -297,16 +299,16 @@ func (c *Calico) generateCalicoNetworks() error {
networkAdapter = c.CNICfg.Interface
}

if c.CNICfg.Interface == "" && c.CNICfg.IP != "" {
iFace, err := findInterface(c.CNICfg.IP)
if c.CNICfg.Interface == "" && c.CNICfg.NodeIP != "" {
iFace, err := findInterface(c.CNICfg.NodeIP)
if err != nil {
return err
}
networkAdapter = iFace
}
}

mgmt, err := createHnsNetwork(c.CNICfg.Mode, networkAdapter)
mgmt, err := createHnsNetwork(c.CNICfg.OverlayEncap, networkAdapter)
if err != nil {
return err
}
Expand Down Expand Up @@ -356,7 +358,7 @@ func (c *Calico) overrideCalicoConfigByHelm(restConfig *rest.Config) error {
}
if bgpEnabled := overrides.Installation.CalicoNetwork.BGP; bgpEnabled != nil {
if *bgpEnabled == opv1.BGPEnabled {
c.CNICfg.Mode = "windows-bgp"
c.CNICfg.OverlayEncap = "windows-bgp"
}
}
return nil
Expand Down Expand Up @@ -399,15 +401,15 @@ func startConfd(ctx context.Context, config *CalicoConfig, logPath string) {

args := []string{
"-confd",
fmt.Sprintf("-confd-confdir=%s", filepath.Join(config.CNI.BinDir, "confd")),
fmt.Sprintf("-confd-confdir=%s", filepath.Join(config.CNIBinDir, "confd")),
}

logrus.Infof("Confd Envs: %s", append(generateGeneralCalicoEnvs(config), specificEnvs...))
cmd := exec.CommandContext(ctx, "calico-node.exe", args...)
cmd.Env = append(generateGeneralCalicoEnvs(config), specificEnvs...)
cmd.Stdout = outputFile
cmd.Stderr = outputFile
_ = os.Chdir(filepath.Join(config.CNI.BinDir, "confd"))
_ = os.Chdir(filepath.Join(config.CNIBinDir, "confd"))
_ = cmd.Run()
logrus.Error("Confd exited")
}
Expand All @@ -417,7 +419,7 @@ func startFelix(ctx context.Context, config *CalicoConfig, logPath string) {

specificEnvs := []string{
fmt.Sprintf("FELIX_FELIXHOSTNAME=%s", config.Hostname),
fmt.Sprintf("FELIX_VXLANVNI=%s", config.Felix.Vxlanvni),
fmt.Sprintf("FELIX_VXLANVNI=%s", config.VxlanVNI),
fmt.Sprintf("FELIX_DATASTORETYPE=%s", config.DatastoreType),
}

Expand Down Expand Up @@ -446,10 +448,10 @@ func startCalico(ctx context.Context, config *CalicoConfig, logPath string) erro

specificEnvs := []string{
fmt.Sprintf("CALICO_NODENAME_FILE=%s", config.NodeNameFile),
fmt.Sprintf("CALICO_NETWORKING_BACKEND=%s", config.Mode),
fmt.Sprintf("CALICO_NETWORKING_BACKEND=%s", config.OverlayEncap),
fmt.Sprintf("CALICO_DATASTORE_TYPE=%s", config.DatastoreType),
fmt.Sprintf("IP_AUTODETECTION_METHOD=%s", config.IPAutoDetectionMethod),
fmt.Sprintf("VXLAN_VNI=%s", config.Felix.Vxlanvni),
fmt.Sprintf("VXLAN_VNI=%s", config.VxlanVNI),
}

// Add OS variables related to Calico. As they come after, they'll overwrite the previous ones
Expand All @@ -476,11 +478,11 @@ func startCalico(ctx context.Context, config *CalicoConfig, logPath string) erro
func generateGeneralCalicoEnvs(config *CalicoConfig) []string {
return []string{
fmt.Sprintf("KUBE_NETWORK=%s", config.KubeNetwork),
fmt.Sprintf("KUBECONFIG=%s", config.KubeConfig.Path),
fmt.Sprintf("KUBECONFIG=%s", filepath.Join(config.ConfigPath, CalicoKubeConfigName)),
fmt.Sprintf("NODENAME=%s", config.Hostname),
fmt.Sprintf("CALICO_K8S_NODE_REF=%s", config.Hostname),

fmt.Sprintf("IP=%s", config.IP),
fmt.Sprintf("USE_POD_CIDR=%t", autoConfigureIpam(config.CNI.IpamType)),
fmt.Sprintf("IP=%s", config.NodeIP),
fmt.Sprintf("USE_POD_CIDR=%t", autoConfigureIpam(config.IpamType)),
}
}
Loading

0 comments on commit 49ef16c

Please sign in to comment.