[test] Add Cilium BGP LB e2e test

.github/workflows/ci.yml

@@ -156,6 +156,9 @@ jobs:
       - name: Run E2E Tests
         run: devbox run e2e-test
 
+      - name: Run Cilium BGP e2e test
+        run: devbox run e2e-test-bgp
+
       - name: Cleanup Resources
         if: always()
         run: devbox run cleanup-cluster

Makefile

@@ -195,6 +195,17 @@ e2e-test:
 	LINODE_TOKEN=$(LINODE_TOKEN) \
 	chainsaw test e2e/test --parallel 2
 
+.PHONY: e2e-test-bgp
+e2e-test-bgp:
+	KUBECONFIG=$(KUBECONFIG_PATH) CLUSTER_SUFFIX=$(CLUSTER_NAME) ./e2e/setup/cilium-setup.sh
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl -n kube-system rollout status daemonset/ccm-linode --timeout=300s
+	CLUSTER_NAME=$(CLUSTER_NAME) \
+		MGMT_KUBECONFIG=$(MGMT_KUBECONFIG_PATH) \
+		KUBECONFIG=$(KUBECONFIG_PATH) \
+		REGION=$(LINODE_REGION) \
+		LINODE_TOKEN=$(LINODE_TOKEN) \
+		chainsaw test e2e/lb-cilium-bgp
+
 #####################################################################
 # OS / ARCH
 #####################################################################

devbox.json

@@ -21,6 +21,7 @@
     "scripts": {
       "mgmt-and-capl-cluster": "make mgmt-and-capl-cluster",
       "e2e-test": "make e2e-test",
+      "e2e-test-bgp": "make e2e-test-bgp",
       "cleanup-cluster": "make cleanup-cluster"
     }
   },

e2e/lb-cilium-bgp/chainsaw-test.yaml

@@ -0,0 +1,139 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: cilium-bgp-test
+spec:
+  namespace: "cilium-bgp-test"
+  steps:
+    - name: Check if CCM is deployed
+      try:
+        - assert:
+            file: ../test/assert-ccm-resources.yaml
+    - name: Create a pod and service with load balancer type cilium-bgp
+      try:
+        - apply:
+            file: create-pod-service.yaml
+      catch:
+        - describe:
+            apiVersion: v1
+            kind: Pod
+        - describe:
+            apiVersion: v1
+            kind: Service
+    - name: Verify CiliumLoadBalancerIPPool creation
+      try:
+        - assert:
+            resource:
+              apiVersion: cilium.io/v2alpha1
+              kind: CiliumLoadBalancerIPPool
+              metadata:
+                name: cilium-bgp-test-test-bgp-svc-pool
+              spec:
+                disabled: false
+    - name: Verify CiliumBGPPeeringPolicy
+      try:
+        - assert:
+            resource:
+              apiVersion: cilium.io/v2alpha1
+              kind: CiliumBGPPeeringPolicy
+              metadata:
+                name: linode-ccm-bgp-peering
+              spec:
+                nodeSelector:
+                  matchLabels:
+                    cilium-bgp-peering: "true"
+    - name: Check LoadBalancer IP assignment
+      try:
+        - assert:
+            resource:
+              apiVersion: v1
+              kind: Service
+              metadata:
+                name: test-bgp-svc
+              status:
+                conditions:
+                - status: "True"
+                  type: cilium.io/IPAMRequestSatisfied
+    - name: Verify IP sharing on labeled nodes
+      try:
+        - script:
+            content: |
+              set -e
+
+              delete_nanode() {
+                local NANODE_RESPONSE=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \
+                  "https://api.linode.com/v4/linode/instances")
+
+                local NANODE_ID=$(echo "$NANODE_RESPONSE" | \
+                  jq -r --arg cluster "$CLUSTER_NAME" '.data[] | select(.label | endswith($cluster)) | .id')
+
+                if [ -n "$NANODE_ID" ]; then
+                  curl -s -X DELETE -H "Authorization: Bearer $LINODE_TOKEN" \
+                    "https://api.linode.com/v4/linode/instances/$NANODE_ID" || true
+                fi
+              }
+
+              # Get the LoadBalancer IP
+              LB_IP=$(kubectl get svc test-bgp-svc -n cilium-bgp-test -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+
+              # Get nodes with BGP label
+              BGP_NODES=$(kubectl get nodes -l cilium-bgp-peering=true -o name)
+
+              if [ -z "$BGP_NODES" ]; then
+                echo "No nodes found with label cilium-bgp-peering=true"
+                delete_nanode
+                exit 1
+              fi
+
+              # Check if IP is shared on each BGP node
+              for node in $BGP_NODES; do
+                NODE_ID=$(kubectl get $node -o jsonpath='{.spec.providerID}' | sed 's|linode://||')
+                echo "Node ID: $NODE_ID"
+
+                NODE_IP_RESPONSE=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \
+                                 "https://api.linode.com/v4/linode/instances/$NODE_ID/ips")
+
+                SHARED_IPS=$(echo "$NODE_IP_RESPONSE" | jq -r '.ipv4.shared[]?.address // empty')
+                echo "shared IPs: $SHARED_IPS"
+
+                if [ -n "$SHARED_IPS" ] && ! echo "$SHARED_IPS" | grep -q "$LB_IP"; then
+                  echo "LoadBalancer IP $LB_IP not found in shared IPs of node $node"
+                  delete_nanode
+                  exit 1
+                fi
+              done
+
+              # Check if the nanode has the shared IP
+              NANODE_RESPONSE=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \
+                "https://api.linode.com/v4/linode/instances")
+
+              NANODE_ID=$(echo "$NANODE_RESPONSE" | \
+                jq -r --arg cluster "$CLUSTER_NAME" '.data[] | select(.label | endswith($cluster)) | .id')
+
+              if [ -z "$NANODE_ID" ]; then
+                echo "No nanode found for cluster $CLUSTER_NAME"
+                exit 0
+              fi
+
+              NANODE_IP_RESPONSE=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \
+                "https://api.linode.com/v4/linode/instances/$NANODE_ID/ips")
+
+              NANODE_IPS=$(echo "$NANODE_IP_RESPONSE" | jq -r '.ipv4.public[]?.address // empty')
+
+              if [ -n "$NANODE_IPS" ] && ! echo "$NANODE_IPS" | grep -q "$LB_IP"; then
+                echo "LoadBalancer IP not found in nanode IPs"
+                delete_nanode
+                exit 1
+              fi
+
+              echo "Successfully found LoadBalancer IP in nanode IPs"
+
+              # Delete the nanode on success
+              delete_nanode
+            check:
+              ($error == null): true
+              (contains($stdout, 'LoadBalancer IP not found in shared IPs of node')): false
+              (contains($stdout, 'LoadBalancer IP not found in nanode IPs')): false
+              (contains($stdout, 'Successfully found LoadBalancer IP in nanode IPs')): true
+

e2e/lb-cilium-bgp/create-pod-service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-pod-1
+  labels:
+    app: test-bgp
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+      ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-bgp-svc
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 80
+      targetPort: 80
+  selector:
+    app: test-bgp

e2e/setup/cilium-setup.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+
+# Add bgp peering label to non control plane nodes. Needed to update the shared IP on the nodes
+kubectl get nodes --no-headers | grep -v control-plane |\
+ awk '{print $1}' | xargs -I {} kubectl label nodes {} cilium-bgp-peering=true --overwrite
+
+# Add RBAC permissions
+kubectl patch clusterrole ccm-linode-clusterrole --type='json' -p='[{
+  "op": "add",
+  "path": "/rules/-",
+  "value": {
+    "apiGroups": ["cilium.io"],
+    "resources": ["ciliumloadbalancerippools", "ciliumbgppeeringpolicies"],
+    "verbs": ["get", "list", "watch", "create", "update", "patch", "delete"]
+  }
+}]'
+
+# Patch DaemonSet
+kubectl patch daemonset ccm-linode -n kube-system --type='json' -p='[{
+  "op": "add",
+  "path": "/spec/template/spec/containers/0/args/-",
+  "value": "--bgp-node-selector=cilium-bgp-peering=true"
+}, {
+  "op": "add",
+  "path": "/spec/template/spec/containers/0/args/-",
+  "value": "--load-balancer-type=cilium-bgp"
+}, {
+  "op": "add",
+  "path": "/spec/template/spec/containers/0/args/-",
+  "value": "--ip-holder-suffix='"${CLUSTER_SUFFIX}"'"
+}]'