Added separate go entrypoints for frr-monitoring and the manager

.github/workflows/container-image.yaml

@@ -41,3 +41,10 @@ jobs:
           context: .
           push: true
           tags: ghcr.io/telekom/das-schiff-network-operator:main
+      - name: Build and push sidecar Docker image
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: frr-monitoring.Dockerfile
+          push: true
+          tags: ghcr.io/telekom/frr-monitoring:main

.github/workflows/draft_release.yaml

@@ -79,3 +79,18 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Extract Metadata (tags, labels) for Docker
+        id: monitor_meta
+        uses: docker/[email protected]
+        with:
+          images: ghcr.io/telekom/frr-monitoring
+
+      - name: Build and Push Docker Image
+        uses: docker/[email protected]
+        with:
+          context: .
+          push: true
+          file: frr-monitoring.Dockerfile
+          tags: ${{ steps.monitor_meta.outputs.tags }}
+          labels: ${{ steps.monitor_meta.outputs.labels }}

.golangci.yml

@@ -145,9 +145,9 @@ linters-settings:
 issues:
   exclude-rules:
     # As of now we ignore the statements in main
-    - path: 'main.go'
+    - path: 'cmd/manager/main.go'
       text: "Function 'main' has too many statements"
-    - path: 'main.go'
+    - path: 'cmd/manager/main.go'
       text: "function-length: maximum number of statements per function exceeded; max \\d+ but got \\d+"
     # This is very C near and those const values are easier to google.
     - path: '(.+)/unix/(coil|frr)\.go'

Dockerfile

@@ -14,7 +14,7 @@ RUN go mod download
 RUN apk add llvm clang linux-headers libbpf-dev musl-dev
 
 # Copy the go source
-COPY main.go main.go
+COPY cmd/manager/main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
 COPY pkg/ pkg/

Makefile

@@ -67,11 +67,15 @@ test: manifests generate fmt vet envtest ## Run tests.
 
 .PHONY: build
 build: generate fmt vet ## Build manager binary.
-	go build -o bin/manager main.go
+	go build -o bin/manager cmd/manager/main.go
+
+.PHONY: build
+sidecar-build: build
+	go build -o bin/frr-monitoring cmd/frr-monitoring/main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./main.go
+	go run ./cmd/manager/main.go
 
 .PHONY: docker-build
 docker-build: test ## Build docker image with the manager.

README.md

@@ -205,7 +205,20 @@ Now we can setup the network namespace for frr and network-operator to run in it
 sudo -E bash testdata/test-netns-setup.sh
 
 ## This finally starts the Operator for development testing in the network namespace called test.
-OPERATOR_CONFIG=$(pwd)/testdata/config.yaml sudo -E ip netns exec test go run main.go --config $(pwd)/testdata/manager-config.yaml
+OPERATOR_CONFIG=$(pwd)/testdata/config.yaml sudo -E ip netns exec test go run cmd/manager/main.go --config $(pwd)/testdata/manager-config.yaml
+```
+
+#### Building and running the frr monitoring container
+
+This container needs a lot of rights as it needs to be able to connect to frr.
+We currently enforce that the vtysh instantiated by the container is not able to write any configuration to the main system.
+
+```bash
+## Building the image for frr-monitoring
+sudo podman build --network=host -t frr-monitoring:latest -f frr-monitoring.Dockerfile .
+
+## Run the container as root as its required.
+sudo podman run --net=host -v /var/run/frr:/var/run/frr -v ./testdata/vtysh.conf:/etc/frr/vtysh.conf localhost/frr-monitoring:latest
 ```
 ### Networking healthcheck
 

cmd/frr-monitoring/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/telekom/das-schiff-network-operator/pkg/monitoring"
+)
+
+const (
+	twenty = 20
+)
+
+var (
+	addr = flag.String("listen-address", ":7082", "The address to listen on for HTTP requests.")
+)
+
+func main() {
+	flag.Parse()
+
+	// Create a new registry.
+	reg := prometheus.NewRegistry()
+
+	// Add Go module build info.
+	reg.MustRegister(collectors.NewBuildInfoCollector())
+	reg.MustRegister(collectors.NewGoCollector())
+	collector, err := monitoring.NewDasSchiffNetworkOperatorCollector(
+		map[string]bool{
+			"frr":     true,
+			"netlink": false,
+		})
+	if err != nil {
+		log.Fatal(fmt.Errorf("failed to create collector %w", err))
+	}
+	reg.MustRegister(collector)
+
+	// Expose the registered metrics via HTTP.
+	http.Handle("/metrics", promhttp.HandlerFor(
+		reg,
+		promhttp.HandlerOpts{
+			// Opt into OpenMetrics to support exemplars.
+			EnableOpenMetrics: true,
+			Timeout:           time.Minute,
+		},
+	))
+	server := http.Server{
+		Addr:              *addr,
+		ReadHeaderTimeout: twenty * time.Second,
+		ReadTimeout:       time.Minute,
+	}
+	log.Fatal(server.ListenAndServe())
+}

main.go → cmd/manager/main.go

@@ -67,7 +67,7 @@ func init() {
 
 func initCollectors() error {
 	var err error
-	collector, err := monitoring.NewDasSchiffNetworkOperatorCollector()
+	collector, err := monitoring.NewDasSchiffNetworkOperatorCollector(map[string]bool{})
 	if err != nil {
 		return fmt.Errorf("failed to create collector: %w", err)
 	}

config/crd/kustomization.yaml

@@ -6,11 +6,8 @@ resources:
 - bases/network.schiff.telekom.de_layer2networkconfigurations.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
-patchesStrategicMerge:
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
 # patches here are for enabling the conversion webhook for each CRD
-- patches/webhook_in_vrfrouteconfigurations.yaml
-- patches/webhook_in_layer2networkconfigurations.yaml
 #+kubebuilder:scaffold:crdkustomizewebhookpatch
 
 # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
@@ -22,3 +19,8 @@ patchesStrategicMerge:
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 configurations:
 - kustomizeconfig.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+patches:
+- path: patches/webhook_in_vrfrouteconfigurations.yaml
+- path: patches/webhook_in_layer2networkconfigurations.yaml

config/default/kustomization.yaml

@@ -9,74 +9,50 @@ namespace: kube-system
 namePrefix: network-operator-
 
 # Labels to add to all resources and selectors.
-commonLabels:
-  app.kubernetes.io/name: network-operator
 #  someName: someValue
 
-bases:
-- ../crd
-- ../rbac
-- ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
-- ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-- ../prometheus
 
-patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
 # - manager_auth_proxy_patch.yaml
 # - manager_master_auth_proxy_patch.yaml
 
 # Expose the metrics port
-- manager_metrics_patch.yaml
-- manager_master_metrics_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type
-- manager_config_patch.yaml
-- manager_master_config_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
-- manager_webhook_patch.yaml
-- manager_master_webhook_patch.yaml
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
 #- webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
-vars:
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: CERTIFICATE_NAME
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#- name: SERVICE_NAMESPACE # namespace of the service
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: SERVICE_NAME
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../crd
+- ../rbac
+- ../manager
+- ../webhook
+- ../prometheus
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/name: network-operator
+patches:
+- path: manager_metrics_patch.yaml
+- path: manager_master_metrics_patch.yaml
+- path: manager_config_patch.yaml
+- path: manager_master_config_patch.yaml
+- path: manager_webhook_patch.yaml
+- path: manager_master_webhook_patch.yaml

config/default/manager_master_metrics_patch.yaml

@@ -12,4 +12,6 @@ spec:
         - containerPort: 7080
           name: metrics
           protocol: TCP
-
+        - containerPort: 7082
+          name: frr-metrics
+          protocol: TCP

config/default/manager_metrics_patch.yaml

@@ -10,5 +10,8 @@ spec:
       - name: manager
         ports:
         - containerPort: 7080
-          name: mertics
+          name: metrics
           protocol: TCP
+        - containerPort: 7082
+          name: frr-metrics
+          protocol: TCP

config/manager/kustomization.yaml

@@ -19,3 +19,6 @@ images:
 - name: controller
   newName: ghcr.io/telekom/das-schiff-network-operator
   newTag: latest
+- name: frr-monitoring
+  newName: ghcr.io/telekom/frr-monitoring
+  newTag: latest

config/manager/manager.yaml

@@ -16,6 +16,13 @@ spec:
       labels:
         app.kubernetes.io/component: worker
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: DoesNotExist
       tolerations:
       - effect: NoSchedule
         key: node.schiff.telekom.de/uninitialized
@@ -71,13 +78,43 @@ spec:
           name: frr-config
         - mountPath: /var/run/dbus/system_bus_socket
           name: dbus-socket
+      - command:
+        - /frr-monitoring
+        args:
+        - --listen-address=":7082"
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: frr-monitoring:latest
+        name: frr-monitoring
+        securityContext:
+          privileged: true
+          runAsUser: 0
+        resources:
+          limits:
+            cpu: 100m
+            memory: 64Mi
+          requests:
+            cpu: 10m
+            memory: 64Mi
+        volumeMounts:
+        - mountPath: /etc/frr
+          name: frr-config
+        - mountPath: /var/run/frr
+          name: frr-run
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
       - name: frr-config
         hostPath:
           path: /etc/frr
           type: Directory
+      - name: frr-run
+        hostPath:
+          path: /var/run/frr
+          type: Directory
       - name: dbus-socket
         hostPath:
           path: /var/run/dbus/system_bus_socket