Merge pull request #13 from yunify/log

add logger and  fix netlink.AddrAdd file exist error.

hostnic/hostnic.go

@@ -37,9 +37,9 @@ import (
 	"github.com/yunify/hostnic-cni/pkg"
 	"github.com/yunify/hostnic-cni/provider"
 	_ "github.com/yunify/hostnic-cni/provider/qingcloud"
+	"github.com/yunify/qingcloud-sdk-go/logger"
 )
 
-
 const processLockFile = pkg.DefaultDataDir + "/lock"
 
 func saveScratchNetConf(containerID, dataDir string, nic *pkg.HostNic) error {
@@ -82,6 +82,12 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
+	netns, err := ns.GetNS(args.Netns)
+	if err != nil {
+		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+	}
+	defer netns.Close()
+
 	nicProvider, err := provider.New(n.Provider, n.Args)
 	if err != nil {
 		return err
@@ -96,6 +102,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 			if route.GW != nil && route.GW.Equal(net.IPv4(0, 0, 0, 0)) {
 				gateway, err := getOrAllocateNicAsGateway(nicProvider, nic)
 				if err != nil {
+					logger.Error("getOrAllocateNicAsGateway err %s, delete Nic %s", err.Error(), nic.ID)
 					deleteNic(nic.ID, nicProvider)
 					return err
 				}
@@ -104,27 +111,23 @@ func cmdAdd(args *skel.CmdArgs) error {
 		}
 	}
 
-	netns, err := ns.GetNS(args.Netns)
-	if err != nil {
-		deleteNic(nic.ID, nicProvider)
-		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
-	}
-	defer netns.Close()
-
 	iface, err := pkg.LinkByMacAddr(nic.HardwareAddr)
 	if err != nil {
+		logger.Error("LinkByMacAddr err %s, delete Nic %s", err.Error(), nic.ID)
 		deleteNic(nic.ID, nicProvider)
 		return fmt.Errorf("failed to get link by MacAddr %q: %v", nic.HardwareAddr, err)
 	}
 
 	if err = netlink.LinkSetNsFd(iface, int(netns.Fd())); err != nil {
+		logger.Error("LinkSetNsFd err %s, delete Nic %s", err.Error(), nic.ID)
 		deleteNic(nic.ID, nicProvider)
 		return fmt.Errorf("failed to set namespace on link %q: %v", nic.HardwareAddr, err)
 	}
 
 	srcName := iface.Attrs().Name
 	_, ipNet, err := net.ParseCIDR(nic.VxNet.Network)
 	if err != nil {
+		logger.Error("ParseCIDR err %s, delete Nic %s", err.Error(), nic.ID)
 		deleteNic(nic.ID, nicProvider)
 		return fmt.Errorf("failed to parse vxnet %q network %q: %v", nic.VxNet.ID, nic.VxNet.Network, err)
 	}
@@ -155,6 +158,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 	err = saveScratchNetConf(args.ContainerID, n.DataDir, nic)
 	if err != nil {
+		logger.Error("saveScratchNetConf err %s, delete Nic %s", err.Error(), nic.ID)
 		deleteNic(nic.ID, nicProvider)
 		return err
 	}
@@ -200,31 +204,39 @@ func getOrAllocateNicAsGateway(nicProvider provider.NicProvider, containernic *p
 	}
 	gateway, err := nicProvider.CreateNicInVxnet(containernic.VxNet.ID)
 	if err != nil {
+		logger.Error("CreateNicInVxnet err %s, delete Nic %s", err.Error(), gateway.ID)
 		deleteNic(gateway.ID, nicProvider)
 		return nil, err
 	}
+	//TODO refactor to  use pkg.ConfigureIface
 	iface, err := pkg.LinkByMacAddr(gateway.HardwareAddr)
 	if err != nil {
+		logger.Error("LinkByMacAddr err %s, delete Nic %s", err.Error(), gateway.ID)
 		deleteNic(gateway.ID, nicProvider)
 		return nil, err
 	}
 	_, ipNet, err := net.ParseCIDR(gateway.VxNet.Network)
 	if err != nil {
+		logger.Error("ParseCIDR err %s, delete Nic %s", err.Error(), gateway.ID)
 		deleteNic(gateway.ID, nicProvider)
 		return nil, err
 	}
 	if err := netlink.LinkSetDown(iface); err != nil {
+		logger.Error("LinkSetDown err %s, delete Nic %s", err.Error(), gateway.ID)
 		deleteNic(gateway.ID, nicProvider)
 		return nil, err
 	}
 	//start to configure ip
+	pkg.ClearLinkAddr(iface)
 	addr := &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(gateway.Address), Mask: ipNet.Mask}, Label: ""}
 	if err := netlink.AddrAdd(iface, addr); err != nil {
+		logger.Error("AddrAdd err %s, delete Nic %s", err.Error(), gateway.ID)
 		deleteNic(gateway.ID, nicProvider)
 		return nil, err
 	}
 	//bring up interface
 	if err := netlink.LinkSetUp(iface); err != nil {
+		logger.Error("LinkSetUp err %s, delete Nic %s", err.Error(), gateway.ID)
 		deleteNic(gateway.ID, nicProvider)
 		return nil, err
 	}

pkg/util.go

@@ -29,6 +29,8 @@ import (
 	"encoding/json"
 	"errors"
 	"io/ioutil"
+	"syscall"
+	"github.com/yunify/qingcloud-sdk-go/logger"
 )
 
 func StringPtr(str string) *string {
@@ -48,7 +50,7 @@ func ConfigureIface(ifName string, res *current.Result) error {
 	if err := netlink.LinkSetDown(link); err != nil {
 		return fmt.Errorf("failed to set link down: %v", err)
 	}
-
+	ClearLinkAddr(link)
 	var v4gw, v6gw net.IP
 	for _, ipc := range res.IPs {
 		if int(ipc.Interface) >= len(res.Interfaces) || res.Interfaces[ipc.Interface].Name != ifName {
@@ -128,4 +130,18 @@ func LoadNetConfFromFile(file string) (*NetConf, error){
 		return nil, err
 	}
 	return LoadNetConf(b)
+}
+
+func ClearLinkAddr(iface netlink.Link){
+	//clear old addr. os possible bind ip to nic before hostnic.
+	addrs, err := netlink.AddrList(iface, syscall.AF_INET)
+	if err == nil && len(addrs) >0 {
+		logger.Info("Del Nic exists addrs: %+v, Nic %s", addrs, iface.Attrs().HardwareAddr)
+		for _,addr := range addrs {
+			err := netlink.AddrDel(iface, &addr)
+			if err != nil {
+				logger.Error("AddrDel err %s addr:%+v, Nic %s", err.Error(), addr, iface.Attrs().HardwareAddr)
+			}
+		}
+	}
 }

provider/qingcloud/provider.go

@@ -23,7 +23,6 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"net"
-	"os"
 	"time"
 
 	"github.com/yunify/hostnic-cni/pkg"
@@ -32,6 +31,7 @@ import (
 	"github.com/yunify/qingcloud-sdk-go/config"
 	"github.com/yunify/qingcloud-sdk-go/service"
 	qcutil "github.com/yunify/qingcloud-sdk-go/utils"
+	"github.com/yunify/qingcloud-sdk-go/logger"
 )
 
 const (
@@ -155,7 +155,7 @@ func (p *QCNicProvider) attachNic(hostNic *pkg.HostNic, instanceID string) error
 	}
 	if *output.RetCode == 0 {
 		jobID := *output.JobID
-		err := p.waitNic(hostNic.ID, jobID, true)
+		err := p.waitNic(hostNic.ID, jobID)
 		if err != nil {
 			return err
 		}
@@ -164,17 +164,18 @@ func (p *QCNicProvider) attachNic(hostNic *pkg.HostNic, instanceID string) error
 	return fmt.Errorf("AttachNics output [%+v] error", *output)
 }
 
-func (p *QCNicProvider) waitNic(nicID string, jobID string, attach bool) error {
-	fmt.Fprintf(os.Stderr, "wait for nic %s %s %v\n", nicID, "attach:", attach)
+func (p *QCNicProvider) waitNic(nicID string, jobID string) error {
+	logger.Debug("Wait for nic %v", nicID)
 	err := qcutil.WaitForSpecific(func() bool {
 		link, err := pkg.LinkByMacAddr(nicID)
 		if err != nil {
-			return !attach
+			return false
 		}
-		fmt.Fprintln(os.Stderr, "find link", link.Attrs().Name, nicID)
-		return attach
+		logger.Debug("Find link %s %s", link.Attrs().Name, nicID)
+		return true
 	}, waitNicLocalTimeout, waitNicLocalInterval)
 	if _, ok := err.(*qcutil.TimeoutError); ok {
+		logger.Info("Wait nic %s by local timeout", nicID)
 		err = client.WaitJob(p.jobService, jobID, defaultOpTimeout, defaultWaitInterval)
 	}
 	return err