allow to adjust cni config dynamically (#7)

* allow adjust cni config dynamically * inject hooks with cni_args env
projecteru2 · Dec 16, 2021 · b8a71b4 · b8a71b4
1 parent 9c5181e
commit b8a71b4
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 53 deletions.
diff --git a/app/app.go b/app/app.go
@@ -41,7 +41,7 @@ func NewApp(handler handler.Handler, printVersion func()) *cli.App {
 						Usage: "cni command",
 					},
 				},
-				Action: runCNI,
+				Action: runCNI(handler),
 			},
 		},
 	}

diff --git a/app/cnicmd.go b/app/cnicmd.go
@@ -12,58 +12,73 @@ import (
 	"github.com/pkg/errors"
 	"github.com/projecteru2/docker-cni/cni"
 	"github.com/projecteru2/docker-cni/config"
+	"github.com/projecteru2/docker-cni/handler"
 	"github.com/urfave/cli/v2"
 )
 
-func runCNI(c *cli.Context) (err error) {
-	stateBuf, err := ioutil.ReadAll(os.Stdin)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	var state specs.State
-	if err = json.Unmarshal(stateBuf, &state); err != nil {
-		return errors.WithStack(err)
-	}
+func runCNI(handler handler.Handler) func(*cli.Context) error {
+	return func(c *cli.Context) (err error) {
+		stateBuf, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		var state specs.State
+		if err = json.Unmarshal(stateBuf, &state); err != nil {
+			return errors.WithStack(err)
+		}
 
-	conf, err := config.LoadConfig(c.String("config"))
-	if err != nil {
-		return
-	}
+		conf, err := config.LoadConfig(c.String("config"))
+		if err != nil {
+			return
+		}
 
-	cniFilename, cniConfigFilename, err := cni.FindCNI(conf.CNIConfDir, conf.CNIBinDir)
-	if err != nil {
-		return
-	}
+		cniFilename, cniConfigFilename, err := cni.FindCNI(conf.CNIConfDir, conf.CNIBinDir)
+		if err != nil {
+			return
+		}
 
-	env := []string{
-		"CNI_IFNAME=" + conf.CNIIfname,
-		"CNI_PATH=" + conf.CNIBinDir,
-		"CNI_ARGS=" + os.Getenv("CNI_ARGS"),
-		"CNI_COMMAND=" + strings.ToUpper(c.String("command")),
-		"CNI_CONTAINERID=" + state.ID,
-	}
+		env := []string{
+			"CNI_IFNAME=" + conf.CNIIfname,
+			"CNI_PATH=" + conf.CNIBinDir,
+			"CNI_ARGS=" + os.Getenv("CNI_ARGS"),
+			"CNI_COMMAND=" + strings.ToUpper(c.String("command")),
+			"CNI_CONTAINERID=" + state.ID,
+		}
 
-	if state.Pid != 0 {
-		env = append(env, "CNI_NETNS="+fmt.Sprintf("/proc/%d/ns/net", state.Pid))
-	}
+		if state.Pid != 0 {
+			env = append(env, "CNI_NETNS="+fmt.Sprintf("/proc/%d/ns/net", state.Pid))
+		}
 
-	file, err := os.Open(cniConfigFilename)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	if err := syscall.Dup2(int(file.Fd()), 0); err != nil {
-		return errors.WithStack(err)
-	}
+		cniConfig, err := os.ReadFile(cniConfigFilename)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if cniConfig, err = handler.HandleCNIConfig(cniConfig); err != nil {
+			return errors.WithStack(err)
+		}
+		r, w, err := os.Pipe()
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if _, err = w.Write(cniConfig); err != nil {
+			return errors.WithStack(err)
+		}
+		defer w.Close()
+		if err := syscall.Dup2(int(r.Fd()), 0); err != nil {
+			return errors.WithStack(err)
+		}
 
-	if file, err = os.OpenFile(conf.CNILog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644); err != nil {
-		return errors.WithStack(err)
-	}
-	if err := syscall.Dup2(int(file.Fd()), 1); err != nil {
-		return errors.WithStack(err)
-	}
-	if err := syscall.Dup2(int(file.Fd()), 2); err != nil {
-		return errors.WithStack(err)
-	}
+		file, err := os.OpenFile(conf.CNILog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if err := syscall.Dup2(int(file.Fd()), 1); err != nil {
+			return errors.WithStack(err)
+		}
+		if err := syscall.Dup2(int(file.Fd()), 2); err != nil {
+			return errors.WithStack(err)
+		}
 
-	return errors.WithStack(syscall.Exec(cniFilename, []string{cniFilename}, env))
+		return errors.WithStack(syscall.Exec(cniFilename, []string{cniFilename}, env))
+	}
 }
diff --git a/handler/cni/create.go b/handler/cni/create.go
@@ -1,11 +1,13 @@
 package cni
 
 import (
+	"strings"
+
 	"github.com/projecteru2/docker-cni/config"
 	"github.com/projecteru2/docker-cni/oci"
 )
 
-func (h CNIHandler) HandleCreate(conf config.Config, containerMeta *oci.ContainerMeta) (err error) {
+func (h *CNIHandler) HandleCreate(conf config.Config, containerMeta *oci.ContainerMeta) (err error) {
 	if err = h.AddCNIStartHook(conf, containerMeta); err != nil {
 		return
 	}
@@ -15,11 +17,16 @@ func (h CNIHandler) HandleCreate(conf config.Config, containerMeta *oci.Containe
 	return containerMeta.Save()
 }
 
-func (h CNIHandler) AddCNIStartHook(conf config.Config, containerMeta *oci.ContainerMeta) (err error) {
+func (h *CNIHandler) AddCNIStartHook(conf config.Config, containerMeta *oci.ContainerMeta) (err error) {
 	env := []string{}
+	cniArgs := []string{}
+	if containerMeta.RequiresSpecificIPPool() {
+		cniArgs = append(cniArgs, "IPPOOL="+containerMeta.SpecificIPPool())
+	}
 	if containerMeta.RequiresSpecificIP() {
-		env = append(env, "CNI_ARGS=IP="+containerMeta.SpecificIP())
+		cniArgs = append(cniArgs, "IP="+containerMeta.SpecificIP())
 	}
+	env = append(env, "CNI_ARGS="+strings.Join(cniArgs, ";"))
 	containerMeta.AppendHook("prestart",
 		conf.BinPathname,
 		[]string{conf.BinPathname, "cni", "--config", conf.Filename, "--command", "add"}, // args
@@ -28,7 +35,7 @@ func (h CNIHandler) AddCNIStartHook(conf config.Config, containerMeta *oci.Conta
 	return
 }
 
-func (h CNIHandler) AddCNIStopHook(conf config.Config, containerMeta *oci.ContainerMeta) (err error) {
+func (h *CNIHandler) AddCNIStopHook(conf config.Config, containerMeta *oci.ContainerMeta) (err error) {
 	containerMeta.AppendHook("poststop",
 		conf.BinPathname,
 		[]string{conf.BinPathname, "cni", "--config", conf.Filename, "--command", "del"}, // args

diff --git a/handler/cni/handler.go b/handler/cni/handler.go
@@ -7,10 +7,14 @@ import (
 
 type CNIHandler struct{}
 
-func (a CNIHandler) HandleStart(_ config.Config, __ *oci.ContainerMeta) (err error) {
+func (a *CNIHandler) HandleStart(_ config.Config, __ *oci.ContainerMeta) (err error) {
 	return
 }
 
-func (a CNIHandler) HandleDelete(_ config.Config, __ *oci.ContainerMeta) (err error) {
+func (a *CNIHandler) HandleDelete(_ config.Config, __ *oci.ContainerMeta) (err error) {
 	return
 }
+
+func (a *CNIHandler) HandleCNIConfig(conf []byte) (newConf []byte, err error) {
+	return conf, nil
+}
diff --git a/handler/handler.go b/handler/handler.go
@@ -9,4 +9,5 @@ type Handler interface {
 	HandleCreate(config.Config, *oci.ContainerMeta) error
 	HandleStart(config.Config, *oci.ContainerMeta) error
 	HandleDelete(config.Config, *oci.ContainerMeta) error
+	HandleCNIConfig([]byte) ([]byte, error)
 }
diff --git a/main.go b/main.go
@@ -9,7 +9,7 @@ import (
 )
 
 func main() {
-	app := app.NewApp(cni.CNIHandler{}, printVersion)
+	app := app.NewApp(&cni.CNIHandler{}, printVersion)
 	if err := app.Run(os.Args); err != nil {
 		fmt.Printf("Error running docker-cni: %+v\n", err)
 		os.Exit(-1)

diff --git a/oci/container.go b/oci/container.go
@@ -37,7 +37,7 @@ func (c *ContainerMeta) Save() (err error) {
 func (c ContainerMeta) SpecificIP() string {
 	for _, env := range c.Process.Env {
 		parts := strings.Split(env, "=")
-		if len(parts) == 2 && parts[0] == "ipv4" && parts[1] != "" {
+		if len(parts) == 2 && parts[0] == "IPV4" && parts[1] != "" {
 			return parts[1]
 		}
 	}
@@ -47,3 +47,17 @@ func (c ContainerMeta) SpecificIP() string {
 func (c ContainerMeta) RequiresSpecificIP() bool {
 	return c.SpecificIP() != ""
 }
+
+func (c ContainerMeta) SpecificIPPool() string {
+	for _, env := range c.Process.Env {
+		parts := strings.Split(env, "=")
+		if len(parts) == 2 && parts[0] == "IPPOOL" && parts[1] != "" {
+			return parts[1]
+		}
+	}
+	return ""
+}
+
+func (c ContainerMeta) RequiresSpecificIPPool() bool {
+	return c.SpecificIPPool() != ""
+}
-Original file line number
+Diff line change
@@ Expand Up @@
     						Usage: "cni command",
     					},
     				},
-    				Action: runCNI,
+    				Action: runCNI(handler),
     			},
     		},
     	}
@@ Expand Down @@