Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed logging issues #56

Merged
merged 1 commit into from
Oct 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions pkg/anycast/anycast.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ package anycast

import (
"bytes"
"fmt"
"net"
"time"

"github.com/go-logr/logr"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
ctrl "sigs.k8s.io/controller-runtime"
)

var (
Expand All @@ -23,14 +24,15 @@ type Tracker struct {
// TODO: Anycast Support is currently highly experimental.

func (t *Tracker) checkTrackedInterfaces() {
logger := ctrl.Log.WithName("anycast")
for _, intfIdx := range t.TrackedBridges {
intf, err := netlink.LinkByIndex(intfIdx)
if err != nil {
fmt.Printf("Couldn't load interface idx %d: %v\n", intfIdx, err)
logger.Error(err, "couldn't load interface", "index", intfIdx)
continue
}

syncInterface(intf.(*netlink.Bridge))
syncInterface(intf.(*netlink.Bridge), logger)
}
}

Expand Down Expand Up @@ -79,10 +81,10 @@ func filterNeighbors(neighIn []netlink.Neigh) (neighOut []netlink.Neigh) {
return neighOut
}

func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32) {
func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32, logger logr.Logger) {
bridgeNeighbors, err := netlink.NeighList(intf.Attrs().Index, family)
if err != nil {
fmt.Printf("Error getting v4 neighbors of interface %s: %v\n", intf.Attrs().Name, err)
logger.Error(err, "error getting v4 neighbors of interface", "interface", intf.Attrs().Name)
return
}
bridgeNeighbors = filterNeighbors(bridgeNeighbors)
Expand All @@ -94,15 +96,15 @@ func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32
}
routes, err := netlink.RouteListFiltered(family, routeFilterV4, netlink.RT_FILTER_OIF|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_PROTOCOL)
if err != nil {
fmt.Printf("Error getting v4 routes of interface %s: %v\n", intf.Attrs().Name, err)
logger.Error(err, "error getting v4 routes of interface", "interface", intf.Attrs().Name)
return
}

alreadyV4Existing := []*net.IPNet{}
for i := range routes {
if !containsIPAddress(bridgeNeighbors, routes[i].Dst) {
if err := netlink.RouteDel(&routes[i]); err != nil {
fmt.Printf("Error deleting route %v: %v\n", routes[i], err)
logger.Error(err, "error deleting route", "route", routes[i])
}
} else {
alreadyV4Existing = append(alreadyV4Existing, routes[i].Dst)
Expand All @@ -114,29 +116,29 @@ func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32
if !containsIPNetwork(alreadyV4Existing, ipnet) {
route := buildRoute(family, intf, ipnet, routingTable)
if err := netlink.RouteAdd(route); err != nil {
fmt.Printf("Error adding route %v: %v\n", route, err)
logger.Error(err, "error adding route", "route", routes[i])
}
}
}
}

func syncInterface(intf *netlink.Bridge) {
func syncInterface(intf *netlink.Bridge, logger logr.Logger) {
routingTable := uint32(defaultVrfAnycastTable)
if intf.Attrs().MasterIndex > 0 {
nl, err := netlink.LinkByIndex(intf.Attrs().MasterIndex)
if err != nil {
fmt.Printf("Error getting VRF parent of interface %s: %v\n", intf.Attrs().Name, err)
logger.Error(err, "error getting VRF parent of interface", "interface", intf.Attrs().Name)
return
}
if nl.Type() != "vrf" {
fmt.Printf("Parent interface of %s is not a VRF: %v\n", intf.Attrs().Name, err)
logger.Info("parent of the interface is not a VRF", "interface", intf.Attrs().Name)
return
}
routingTable = nl.(*netlink.Vrf).Table
}

syncInterfaceByFamily(intf, unix.AF_INET, routingTable)
syncInterfaceByFamily(intf, unix.AF_INET6, routingTable)
syncInterfaceByFamily(intf, unix.AF_INET, routingTable, logger)
syncInterfaceByFamily(intf, unix.AF_INET6, routingTable, logger)
}

func (t *Tracker) RunAnycastSync() {
Expand Down
21 changes: 11 additions & 10 deletions pkg/bpf/monitoring.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package bpf

import (
"fmt"

"github.com/cilium/ebpf"
"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

Expand All @@ -18,22 +18,23 @@ var (
)

func initMonitoring() {
registerMap(router.routerMaps.EbpfRetStatsMap, "ebpf_return_reasons", epbfReturnReasons)
registerMap(router.routerMaps.EbpfFibLkupStatsMap, "ebpf_fib_lookup", ebpfFibLookuPResult)
logger := ctrl.Log.WithName("bpf-monitoring")
registerMap(router.routerMaps.EbpfRetStatsMap, "ebpf_return_reasons", epbfReturnReasons, logger)
registerMap(router.routerMaps.EbpfFibLkupStatsMap, "ebpf_fib_lookup", ebpfFibLookuPResult, logger)
}

type StatsRecord struct {
RXPackets uint64
RXBytes uint64
}

func registerMap(m *ebpf.Map, prefix string, keys []string) {
func registerMap(m *ebpf.Map, prefix string, keys []string, logger logr.Logger) {
mapInfo, err := m.Info()
if err != nil {
panic(err)
}
mapID, _ := mapInfo.ID()
fmt.Printf("adding monitoring for map %s (id: %d) with prefix %s\n", mapInfo.Name, mapID, prefix)
logger.Info("adding monitoring", "map", mapInfo.Name, "id", mapID, "prefix", prefix)
for idx, name := range keys {
// eBPF map by packets
metrics.Registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Expand All @@ -42,7 +43,7 @@ func registerMap(m *ebpf.Map, prefix string, keys []string) {
"key": name,
},
}, func() float64 {
stats := fetchEbpfStatistics(m, uint32(idx))
stats := fetchEbpfStatistics(m, uint32(idx), logger)
return float64(stats.RXPackets)
}))

Expand All @@ -53,17 +54,17 @@ func registerMap(m *ebpf.Map, prefix string, keys []string) {
"key": name,
},
}, func() float64 {
stats := fetchEbpfStatistics(m, uint32(idx))
stats := fetchEbpfStatistics(m, uint32(idx), logger)
return float64(stats.RXBytes)
}))
}
}

func fetchEbpfStatistics(m *ebpf.Map, key uint32) *StatsRecord {
func fetchEbpfStatistics(m *ebpf.Map, key uint32, logger logr.Logger) *StatsRecord {
var perCPUStats []*StatsRecord
err := m.Lookup(key, &perCPUStats)
if err != nil {
fmt.Printf("Error reading eBPF statistics from map: %v\n", err)
logger.Error(err, "error reading eBPF statistics from map")
return nil
}
var aggregatedStats StatsRecord
Expand Down
45 changes: 24 additions & 21 deletions pkg/macvlan/macvlan.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,28 @@ import (
"strings"
"time"

"github.com/go-logr/logr"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
ctrl "sigs.k8s.io/controller-runtime"
)

const checkInterval = 5 * time.Second

var trackedBridges []int

func checkTrackedInterfaces() {
func checkTrackedInterfaces(logger logr.Logger) {
for _, intfIdx := range trackedBridges {
intf, err := netlink.LinkByIndex(intfIdx)
if err != nil {
fmt.Printf("Couldn't load interface idx %d: %v\n", intfIdx, err)
logger.Error(err, "couldn't load interface", "index", intfIdx)
}

syncInterface(intf.(*netlink.Bridge))
syncInterface(intf.(*netlink.Bridge), logger)
}
}

func ensureMACDummyIntf(intf *netlink.Bridge) (netlink.Link, error) {
func ensureMACDummyIntf(intf *netlink.Bridge, logger logr.Logger) (netlink.Link, error) {
name := fmt.Sprintf("mvd.%s", intf.Attrs().Name)
macDummy, err := netlink.LinkByName(name)
if err != nil {
Expand All @@ -52,7 +54,7 @@ func ensureMACDummyIntf(intf *netlink.Bridge) (netlink.Link, error) {
}
}
if macDummy.Attrs().OperState != netlink.OperDown {
fmt.Printf("Interface %s not down, setting down - otherwise it would route traffic\n", name)
logger.Info("Interface not down, setting down - otherwise it would route traffic", "interface", name)
err = netlink.LinkSetDown(macDummy)
if err != nil {
return nil, fmt.Errorf("error setting link %s down: %w", macDummy.Attrs().Name, err)
Expand Down Expand Up @@ -84,22 +86,22 @@ func containsMACAddress(list []net.HardwareAddr, mac net.HardwareAddr) bool {
return false
}

func syncInterface(intf *netlink.Bridge) {
func syncInterface(intf *netlink.Bridge, logger logr.Logger) {
// First ensure that we have a dummy interface
dummy, err := ensureMACDummyIntf(intf)
dummy, err := ensureMACDummyIntf(intf, logger)
if err != nil {
fmt.Printf("Error syncing interface %s: %v\n", intf.Attrs().Name, err)
logger.Error(err, "error syncing interface", "interface", intf.Attrs().Name)
return
}

configureNeighbors(intf, dummy)
configureNeighbors(intf, dummy, logger)
}

func configureNeighbors(intf *netlink.Bridge, dummy netlink.Link) {
func configureNeighbors(intf *netlink.Bridge, dummy netlink.Link, logger logr.Logger) {
// Get neighbors of bridge
bridgeNeighbors, err := netlink.NeighList(intf.Attrs().Index, unix.AF_BRIDGE)
if err != nil {
fmt.Printf("Error syncing interface %s: %v\n", intf.Attrs().Name, err)
logger.Error(err, "error getting neighbors", "interface", intf.Attrs().Name)
return
}
requiredMACAddresses := []net.HardwareAddr{}
Expand All @@ -114,25 +116,25 @@ func configureNeighbors(intf *netlink.Bridge, dummy netlink.Link) {
// Get neighbors of dummy
dummyNeighbors, err := netlink.NeighList(dummy.Attrs().Index, unix.AF_BRIDGE)
if err != nil {
fmt.Printf("Error syncing interface %s: %v\n", intf.Attrs().Name, err)
logger.Error(err, "error getting neighbors", "interface", intf.Attrs().Name)
return
}

alreadyExisting := getAlreadyExistingNeighbors(dummyNeighbors, requiredMACAddresses, dummy.Attrs().Name, intf.Attrs().Name)
alreadyExisting := getAlreadyExistingNeighbors(dummyNeighbors, requiredMACAddresses, dummy.Attrs().Name, intf.Attrs().Name, logger)

// Add required MAC addresses when they are not yet existing (aka in alreadyExisting slice)
for _, neigh := range requiredMACAddresses {
if !containsMACAddress(alreadyExisting, neigh) {
fmt.Printf("Adding MAC address %s on dummy interface %s of bridge %s\n", neigh, dummy.Attrs().Name, intf.Attrs().Name)
logger.Info("adding MAC address on dummy interface of bridge", "address", neigh, "interface", dummy.Attrs().Name, "bridge", intf.Attrs().Name)
err = netlink.NeighSet(createNeighborEntry(neigh, dummy.Attrs().Index, intf.Attrs().Index))
if err != nil {
fmt.Printf("Error adding neighbor %s to intf %s (br %s): %v\n", neigh, dummy.Attrs().Name, intf.Attrs().Name, err)
logger.Error(err, "error adding neighbor to interface", "neighbor", neigh, "interface", dummy.Attrs().Name, "bridge", intf.Attrs().Name)
}
}
}
}

func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddresses []net.HardwareAddr, dummyName, intfName string) []net.HardwareAddr {
func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddresses []net.HardwareAddr, dummyName, intfName string, logger logr.Logger) []net.HardwareAddr {
alreadyExisting := []net.HardwareAddr{}
for i := range dummyNeighbors {
neigh := &dummyNeighbors[i]
Expand All @@ -141,9 +143,9 @@ func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddr
if !containsMACAddress(requiredMACAddresses, neigh.HardwareAddr) {
// If MAC Address is not in required MAC addresses, delete neighbor
if err := netlink.NeighDel(neigh); err != nil {
fmt.Printf("Error deleting neighbor %v: %v\n", neigh, err)
logger.Error(err, "error deleting neighbor", "neighbor", neigh)
}
fmt.Printf("Removed MAC address %s from dummy interface %s of bridge %s\n", neigh.HardwareAddr, dummyName, intfName)
logger.Info("removed MAC address", "address", neigh.HardwareAddr.String(), "interface", dummyName, "bridge", intfName)
} else {
// Add MAC address to alreadyExisting table
alreadyExisting = append(alreadyExisting, neigh.HardwareAddr)
Expand All @@ -154,22 +156,23 @@ func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddr
}

func RunMACSync(interfacePrefix string) {
logger := ctrl.Log.WithName("macvlan")
links, err := netlink.LinkList()
if err != nil {
fmt.Printf("Couldn't load interfaces: %v\n", err)
logger.Error(err, "error loading interfaces")
return
}
for _, link := range links {
if strings.HasPrefix(link.Attrs().Name, interfacePrefix) && link.Type() == "bridge" {
fmt.Printf("Tracking interface %s (bridge and Prefix '%s')\n", link.Attrs().Name, interfacePrefix)
logger.Info("tracking interface", "interface", link.Attrs().Name, "prefix", interfacePrefix)
trackedBridges = append(trackedBridges, link.Attrs().Index)
}
}

if len(trackedBridges) > 0 {
go func() {
for {
checkTrackedInterfaces()
checkTrackedInterfaces(logger)
time.Sleep(checkInterval)
}
}()
Expand Down