Pi kube-vip daemonset (#1925)

* change config

* feat: remove extra args

* fix: bootstrap

ansible/pi/inventory/group_vars/kubernetes/main.yaml

@@ -1,3 +1,4 @@
+---
 k3s_become: true
 k3s_etcd_datastore: true
 k3s_install_hard_links: true
@@ -16,15 +17,9 @@ k3s_registries:
     registry.k8s.io:
 # renovate: datasource=github-releases depName=k3s-io/k3s
 k3s_release_version: v1.29.2+k3s1
-k3s_server_manifests_urls:
-  # Kube-vip RBAC
-  - url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml
-    filename: kube-vip-rbac.yaml
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2
-  - custom-cilium-l2.yaml.j2
   - custom-coredns-helmchart.yaml.j2
-# /var/lib/rancher/k3s/agent/pod-manifests
-k3s_server_pod_manifests_templates:
-  - kube-vip-static-pod.yaml.j2
+  - custom-kube-vip-daemonset.yaml.j2
+  - custom-kube-vip-rbac.yaml.j2

ansible/pi/playbooks/cluster-installation.yaml

@@ -21,6 +21,10 @@
         k3s_server_manifests_templates: []
         k3s_server_manifests_urls: []
 
+    - name: Prevent downgrades
+      when: k3s_installed.stat.exists
+      ansible.builtin.include_tasks: tasks/version-check.yaml
+
     - name: Install Kubernetes
       ansible.builtin.include_role:
         name: xanmanning.k3s
@@ -48,9 +52,8 @@
         wait_timeout: 360
       loop:
         - { name: cilium, kind: HelmChart, namespace: kube-system }
+        - { name: kube-vip, kind: DaemonSet, namespace: kube-system }
         - { name: coredns, kind: HelmChart, namespace: kube-system }
-        - { name: policy, kind: CiliumL2AnnouncementPolicy }
-        - { name: pool, kind: CiliumLoadBalancerIPPool }
 
     - name: Coredns
       when: k3s_primary_control_node

ansible/pi/playbooks/tasks/version-check.yaml

@@ -0,0 +1,17 @@
+---
+- name: Version Check
+  block:
+    - name: Get deployed k3s version
+      ansible.builtin.command: k3s --version
+      register: k3s_version
+      changed_when: false
+      failed_when: false
+
+    - name: Extract k3s version
+      ansible.builtin.set_fact:
+        current_k3s_version: "{{ k3s_version.stdout | regex_replace('(?im)k3s version (?P<version>[a-z0-9\\.\\+]+).*\n.*', '\\g<version>') }}"
+
+    - name: Check if upgrades are allowed
+      ansible.builtin.assert:
+        that: ["k3s_release_version is version(current_k3s_version, '>=')"]
+        fail_msg: "Unable to upgrade k3s because the deployed version is higher than the one specified in the configuration"

ansible/pi/playbooks/templates/custom-cilium-helmchart.yaml.j2

@@ -9,15 +9,15 @@ spec:
   # renovate: datasource=helm
   repo: https://helm.cilium.io/
   chart: cilium
-  version: 1.15.0-rc.1
+  version: 1.15.1
   targetNamespace: kube-system
   bootstrap: true
   valuesContent: |-
     autoDirectNodeRoutes: true
     bpf:
       masquerade: true
     cluster:
-      name: home-cluster
+      name: pi
       id: 1
     containerRuntime:
       integration: containerd
@@ -29,16 +29,12 @@ spec:
     ipam:
       mode: kubernetes
     ipv4NativeRoutingCIDR: "{{ cluster_cidr }}"
-    k8sServiceHost: "{{ kube_vip_addr }}"
+    k8sServiceHost: 127.0.0.1
     k8sServicePort: 6443
     kubeProxyReplacement: true
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
     l2announcements:
       enabled: true
-      # https://github.com/cilium/cilium/issues/26586
-      leaseDuration: 120s
-      leaseRenewDeadline: 60s
-      leaseRetryPeriod: 1s
     loadBalancer:
       algorithm: maglev
       mode: dsr

ansible/pi/playbooks/templates/custom-kube-vip-daemonset.yaml.j2

@@ -0,0 +1,72 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kube-vip
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-vip
+    spec:
+      containers:
+        - name: kube-vip
+          image: ghcr.io/kube-vip/kube-vip:v0.7.2
+          imagePullPolicy: IfNotPresent
+          args: ["manager"]
+          env:
+            - name: address
+              value: "{{ k3s_registration_address }}"
+            - name: vip_arp
+              value: "true"
+            - name: lb_enable
+              value: "true"
+            - name: port
+              value: "6443"
+            - name: vip_cidr
+              value: "32"
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: vip_ddns
+              value: "false"
+            - name: svc_enable
+              value: "false"
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            - name: prometheus_server
+              value: :2112
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
+      hostAliases:
+        - hostnames:
+            - kubernetes
+          ip: 127.0.0.1
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists

ansible/pi/playbooks/templates/custom-kube-vip-rbac.yaml.j2

@@ -0,0 +1,42 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["services", "endpoints"]
+    verbs: ["list","get","watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list","get","watch", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["list","get","watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system

kubernetes/pi/apps/kube-system/cilium/app/helmrelease.yaml

@@ -31,7 +31,7 @@ spec:
     bpf:
       masquerade: true
     cluster:
-      name: kubernetes
+      name: pi
       id: 1
     containerRuntime:
       integration: containerd
@@ -67,8 +67,6 @@ spec:
         ingress:
           enabled: true
           className: internal
-          annotations:
-            hajimari.io/icon: simple-icons:cilium
           hosts:
             - &host hubble.${SECRET_DOMAIN}
           tls:
@@ -83,9 +81,6 @@ spec:
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
     l2announcements:
       enabled: true
-      leaseDuration: 120s
-      leaseRenewDeadline: 60s
-      leaseRetryPeriod: 1s
     loadBalancer:
       algorithm: maglev
       mode: dsr