Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "Improved logging" #55

Merged
merged 1 commit into from
Sep 12, 2023
Merged

Revert "Improved logging" #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 13 additions & 15 deletions pkg/anycast/anycast.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@ package anycast

import (
"bytes"
"fmt"
"net"
"time"

"github.com/go-logr/logr"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
ctrl "sigs.k8s.io/controller-runtime"
)

var (
Expand All @@ -24,15 +23,14 @@ type Tracker struct {
// TODO: Anycast Support is currently highly experimental.

func (t *Tracker) checkTrackedInterfaces() {
logger := ctrl.Log.WithName("anycast")
for _, intfIdx := range t.TrackedBridges {
intf, err := netlink.LinkByIndex(intfIdx)
if err != nil {
logger.Error(err, "couldn't load interface idx %d", intfIdx)
fmt.Printf("Couldn't load interface idx %d: %v\n", intfIdx, err)
continue
}

syncInterface(intf.(*netlink.Bridge), logger)
syncInterface(intf.(*netlink.Bridge))
}
}

Expand Down Expand Up @@ -81,10 +79,10 @@ func filterNeighbors(neighIn []netlink.Neigh) (neighOut []netlink.Neigh) {
return neighOut
}

func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32, logger logr.Logger) {
func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32) {
bridgeNeighbors, err := netlink.NeighList(intf.Attrs().Index, family)
if err != nil {
logger.Error(err, "error getting v4 neighbors of interface %s", intf.Attrs().Name)
fmt.Printf("Error getting v4 neighbors of interface %s: %v\n", intf.Attrs().Name, err)
return
}
bridgeNeighbors = filterNeighbors(bridgeNeighbors)
Expand All @@ -96,15 +94,15 @@ func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32
}
routes, err := netlink.RouteListFiltered(family, routeFilterV4, netlink.RT_FILTER_OIF|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_PROTOCOL)
if err != nil {
logger.Error(err, "error getting v4 routes of interface %s", intf.Attrs().Name)
fmt.Printf("Error getting v4 routes of interface %s: %v\n", intf.Attrs().Name, err)
return
}

alreadyV4Existing := []*net.IPNet{}
for i := range routes {
if !containsIPAddress(bridgeNeighbors, routes[i].Dst) {
if err := netlink.RouteDel(&routes[i]); err != nil {
logger.Error(err, "error deleting route %v", routes[i])
fmt.Printf("Error deleting route %v: %v\n", routes[i], err)
}
} else {
alreadyV4Existing = append(alreadyV4Existing, routes[i].Dst)
Expand All @@ -116,29 +114,29 @@ func syncInterfaceByFamily(intf *netlink.Bridge, family int, routingTable uint32
if !containsIPNetwork(alreadyV4Existing, ipnet) {
route := buildRoute(family, intf, ipnet, routingTable)
if err := netlink.RouteAdd(route); err != nil {
logger.Error(err, "error adding route %v", route)
fmt.Printf("Error adding route %v: %v\n", route, err)
}
}
}
}

func syncInterface(intf *netlink.Bridge, logger logr.Logger) {
func syncInterface(intf *netlink.Bridge) {
routingTable := uint32(defaultVrfAnycastTable)
if intf.Attrs().MasterIndex > 0 {
nl, err := netlink.LinkByIndex(intf.Attrs().MasterIndex)
if err != nil {
logger.Error(err, "error getting VRF parent of interface %s", intf.Attrs().Name)
fmt.Printf("Error getting VRF parent of interface %s: %v\n", intf.Attrs().Name, err)
return
}
if nl.Type() != "vrf" {
logger.Info("parent interface of %s is not a VRF: %v\n", intf.Attrs().Name, err)
fmt.Printf("Parent interface of %s is not a VRF: %v\n", intf.Attrs().Name, err)
return
}
routingTable = nl.(*netlink.Vrf).Table
}

syncInterfaceByFamily(intf, unix.AF_INET, routingTable, logger)
syncInterfaceByFamily(intf, unix.AF_INET6, routingTable, logger)
syncInterfaceByFamily(intf, unix.AF_INET, routingTable)
syncInterfaceByFamily(intf, unix.AF_INET6, routingTable)
}

func (t *Tracker) RunAnycastSync() {
Expand Down
21 changes: 10 additions & 11 deletions pkg/bpf/monitoring.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package bpf

import (
"fmt"

"github.com/cilium/ebpf"
"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

Expand All @@ -18,23 +18,22 @@ var (
)

func initMonitoring() {
logger := ctrl.Log.WithName("bpf-monitoring")
registerMap(router.routerMaps.EbpfRetStatsMap, "ebpf_return_reasons", epbfReturnReasons, logger)
registerMap(router.routerMaps.EbpfFibLkupStatsMap, "ebpf_fib_lookup", ebpfFibLookuPResult, logger)
registerMap(router.routerMaps.EbpfRetStatsMap, "ebpf_return_reasons", epbfReturnReasons)
registerMap(router.routerMaps.EbpfFibLkupStatsMap, "ebpf_fib_lookup", ebpfFibLookuPResult)
}

type StatsRecord struct {
RXPackets uint64
RXBytes uint64
}

func registerMap(m *ebpf.Map, prefix string, keys []string, logger logr.Logger) {
func registerMap(m *ebpf.Map, prefix string, keys []string) {
mapInfo, err := m.Info()
if err != nil {
panic(err)
}
mapID, _ := mapInfo.ID()
logger.Info("adding monitoring for map %s (id: %d) with prefix %s", mapInfo.Name, mapID, prefix)
fmt.Printf("adding monitoring for map %s (id: %d) with prefix %s\n", mapInfo.Name, mapID, prefix)
for idx, name := range keys {
// eBPF map by packets
metrics.Registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Expand All @@ -43,7 +42,7 @@ func registerMap(m *ebpf.Map, prefix string, keys []string, logger logr.Logger)
"key": name,
},
}, func() float64 {
stats := fetchEbpfStatistics(m, uint32(idx), logger)
stats := fetchEbpfStatistics(m, uint32(idx))
return float64(stats.RXPackets)
}))

Expand All @@ -54,17 +53,17 @@ func registerMap(m *ebpf.Map, prefix string, keys []string, logger logr.Logger)
"key": name,
},
}, func() float64 {
stats := fetchEbpfStatistics(m, uint32(idx), logger)
stats := fetchEbpfStatistics(m, uint32(idx))
return float64(stats.RXBytes)
}))
}
}

func fetchEbpfStatistics(m *ebpf.Map, key uint32, logger logr.Logger) *StatsRecord {
func fetchEbpfStatistics(m *ebpf.Map, key uint32) *StatsRecord {
var perCPUStats []*StatsRecord
err := m.Lookup(key, &perCPUStats)
if err != nil {
logger.Error(err, "error reading eBPF statistics from map")
fmt.Printf("Error reading eBPF statistics from map: %v\n", err)
return nil
}
var aggregatedStats StatsRecord
Expand Down
45 changes: 21 additions & 24 deletions pkg/macvlan/macvlan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,28 +8,26 @@ import (
"strings"
"time"

"github.com/go-logr/logr"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
ctrl "sigs.k8s.io/controller-runtime"
)

const checkInterval = 5 * time.Second

var trackedBridges []int

func checkTrackedInterfaces(logger logr.Logger) {
func checkTrackedInterfaces() {
for _, intfIdx := range trackedBridges {
intf, err := netlink.LinkByIndex(intfIdx)
if err != nil {
logger.Error(err, "couldn't load interface idx %d", intfIdx)
fmt.Printf("Couldn't load interface idx %d: %v\n", intfIdx, err)
}

syncInterface(intf.(*netlink.Bridge), logger)
syncInterface(intf.(*netlink.Bridge))
}
}

func ensureMACDummyIntf(intf *netlink.Bridge, logger logr.Logger) (netlink.Link, error) {
func ensureMACDummyIntf(intf *netlink.Bridge) (netlink.Link, error) {
name := fmt.Sprintf("mvd.%s", intf.Attrs().Name)
macDummy, err := netlink.LinkByName(name)
if err != nil {
Expand All @@ -54,7 +52,7 @@ func ensureMACDummyIntf(intf *netlink.Bridge, logger logr.Logger) (netlink.Link,
}
}
if macDummy.Attrs().OperState != netlink.OperDown {
logger.Info("interface %s not down, setting down - otherwise it would route traffic", name)
fmt.Printf("Interface %s not down, setting down - otherwise it would route traffic\n", name)
err = netlink.LinkSetDown(macDummy)
if err != nil {
return nil, fmt.Errorf("error setting link %s down: %w", macDummy.Attrs().Name, err)
Expand Down Expand Up @@ -86,22 +84,22 @@ func containsMACAddress(list []net.HardwareAddr, mac net.HardwareAddr) bool {
return false
}

func syncInterface(intf *netlink.Bridge, logger logr.Logger) {
func syncInterface(intf *netlink.Bridge) {
// First ensure that we have a dummy interface
dummy, err := ensureMACDummyIntf(intf, logger)
dummy, err := ensureMACDummyIntf(intf)
if err != nil {
logger.Error(err, "error syncing interface %s", intf.Attrs().Name)
fmt.Printf("Error syncing interface %s: %v\n", intf.Attrs().Name, err)
return
}

configureNeighbors(intf, dummy, logger)
configureNeighbors(intf, dummy)
}

func configureNeighbors(intf *netlink.Bridge, dummy netlink.Link, logger logr.Logger) {
func configureNeighbors(intf *netlink.Bridge, dummy netlink.Link) {
// Get neighbors of bridge
bridgeNeighbors, err := netlink.NeighList(intf.Attrs().Index, unix.AF_BRIDGE)
if err != nil {
logger.Error(err, "error syncing interface %s", intf.Attrs().Name)
fmt.Printf("Error syncing interface %s: %v\n", intf.Attrs().Name, err)
return
}
requiredMACAddresses := []net.HardwareAddr{}
Expand All @@ -116,25 +114,25 @@ func configureNeighbors(intf *netlink.Bridge, dummy netlink.Link, logger logr.Lo
// Get neighbors of dummy
dummyNeighbors, err := netlink.NeighList(dummy.Attrs().Index, unix.AF_BRIDGE)
if err != nil {
logger.Error(err, "error syncing interface %s", intf.Attrs().Name)
fmt.Printf("Error syncing interface %s: %v\n", intf.Attrs().Name, err)
return
}

alreadyExisting := getAlreadyExistingNeighbors(dummyNeighbors, requiredMACAddresses, dummy.Attrs().Name, intf.Attrs().Name, logger)
alreadyExisting := getAlreadyExistingNeighbors(dummyNeighbors, requiredMACAddresses, dummy.Attrs().Name, intf.Attrs().Name)

// Add required MAC addresses when they are not yet existing (aka in alreadyExisting slice)
for _, neigh := range requiredMACAddresses {
if !containsMACAddress(alreadyExisting, neigh) {
logger.Info("adding MAC address %s on dummy interface %s of bridge %s", neigh, dummy.Attrs().Name, intf.Attrs().Name)
fmt.Printf("Adding MAC address %s on dummy interface %s of bridge %s\n", neigh, dummy.Attrs().Name, intf.Attrs().Name)
err = netlink.NeighSet(createNeighborEntry(neigh, dummy.Attrs().Index, intf.Attrs().Index))
if err != nil {
logger.Error(err, "error adding neighbor %s to intf %s (br %s)", neigh, dummy.Attrs().Name, intf.Attrs().Name)
fmt.Printf("Error adding neighbor %s to intf %s (br %s): %v\n", neigh, dummy.Attrs().Name, intf.Attrs().Name, err)
}
}
}
}

func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddresses []net.HardwareAddr, dummyName, intfName string, logger logr.Logger) []net.HardwareAddr {
func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddresses []net.HardwareAddr, dummyName, intfName string) []net.HardwareAddr {
alreadyExisting := []net.HardwareAddr{}
for i := range dummyNeighbors {
neigh := &dummyNeighbors[i]
Expand All @@ -143,9 +141,9 @@ func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddr
if !containsMACAddress(requiredMACAddresses, neigh.HardwareAddr) {
// If MAC Address is not in required MAC addresses, delete neighbor
if err := netlink.NeighDel(neigh); err != nil {
logger.Error(err, "error deleting neighbor %v", neigh)
fmt.Printf("Error deleting neighbor %v: %v\n", neigh, err)
}
logger.Info("removed MAC address %s from dummy interface %s of bridge %s", neigh.HardwareAddr, dummyName, intfName)
fmt.Printf("Removed MAC address %s from dummy interface %s of bridge %s\n", neigh.HardwareAddr, dummyName, intfName)
} else {
// Add MAC address to alreadyExisting table
alreadyExisting = append(alreadyExisting, neigh.HardwareAddr)
Expand All @@ -156,23 +154,22 @@ func getAlreadyExistingNeighbors(dummyNeighbors []netlink.Neigh, requiredMACAddr
}

func RunMACSync(interfacePrefix string) {
logger := ctrl.Log.WithName("macvlan")
links, err := netlink.LinkList()
if err != nil {
logger.Error(err, "couldn't load interfaces")
fmt.Printf("Couldn't load interfaces: %v\n", err)
return
}
for _, link := range links {
if strings.HasPrefix(link.Attrs().Name, interfacePrefix) && link.Type() == "bridge" {
logger.Info("tracking interface %s (bridge and Prefix '%s')", link.Attrs().Name, interfacePrefix)
fmt.Printf("Tracking interface %s (bridge and Prefix '%s')\n", link.Attrs().Name, interfacePrefix)
trackedBridges = append(trackedBridges, link.Attrs().Index)
}
}

if len(trackedBridges) > 0 {
go func() {
for {
checkTrackedInterfaces(logger)
checkTrackedInterfaces()
time.Sleep(checkInterval)
}
}()
Expand Down