From b8f1c1c9be6cb63ca15a9d9b6c561f9f5a3fa84b Mon Sep 17 00:00:00 2001 From: Jory Irving <46251616+joryirving@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:16:54 -0600 Subject: [PATCH] Pi kube-vip daemonset (#1925) * change config * feat: remove extra args * fix: bootstrap --- .../inventory/group_vars/kubernetes/main.yaml | 11 +-- .../pi/playbooks/cluster-installation.yaml | 7 +- ansible/pi/playbooks/cluster-kube-vip.yaml | 24 ------ ansible/pi/playbooks/tasks/version-check.yaml | 17 +++++ .../templates/custom-cilium-helmchart.yaml.j2 | 10 +-- .../templates/custom-cilium-l2.yaml.j2 | 22 ------ .../custom-kube-vip-daemonset.yaml.j2 | 72 ++++++++++++++++++ .../templates/custom-kube-vip-rbac.yaml.j2 | 42 +++++++++++ .../templates/kube-vip-static-pod.yaml.j2 | 59 --------------- .../kube-system/cilium/app/helmrelease.yaml | 7 +- .../kube-system/kube-vip/app/daemonset.yaml | 75 +++++++++++++++++++ .../kube-vip/app/kustomization.yaml | 7 ++ .../apps/kube-system/kube-vip/app/rbac.yaml | 42 +++++++++++ .../pi/apps/kube-system/kube-vip/ks.yaml | 21 ++++++ .../pi/apps/kube-system/kustomization.yaml | 1 + 15 files changed, 289 insertions(+), 128 deletions(-) delete mode 100644 ansible/pi/playbooks/cluster-kube-vip.yaml create mode 100644 ansible/pi/playbooks/tasks/version-check.yaml delete mode 100644 ansible/pi/playbooks/templates/custom-cilium-l2.yaml.j2 create mode 100644 ansible/pi/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 create mode 100644 ansible/pi/playbooks/templates/custom-kube-vip-rbac.yaml.j2 delete mode 100644 ansible/pi/playbooks/templates/kube-vip-static-pod.yaml.j2 create mode 100644 kubernetes/pi/apps/kube-system/kube-vip/app/daemonset.yaml create mode 100644 kubernetes/pi/apps/kube-system/kube-vip/app/kustomization.yaml create mode 100644 kubernetes/pi/apps/kube-system/kube-vip/app/rbac.yaml create mode 100644 kubernetes/pi/apps/kube-system/kube-vip/ks.yaml diff --git a/ansible/pi/inventory/group_vars/kubernetes/main.yaml b/ansible/pi/inventory/group_vars/kubernetes/main.yaml index 2d5036f8e5..84cb2cb80b 100644 --- a/ansible/pi/inventory/group_vars/kubernetes/main.yaml +++ b/ansible/pi/inventory/group_vars/kubernetes/main.yaml @@ -1,3 +1,4 @@ +--- k3s_become: true k3s_etcd_datastore: true k3s_install_hard_links: true @@ -16,15 +17,9 @@ k3s_registries: registry.k8s.io: # renovate: datasource=github-releases depName=k3s-io/k3s k3s_release_version: v1.29.2+k3s1 -k3s_server_manifests_urls: - # Kube-vip RBAC - - url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml - filename: kube-vip-rbac.yaml # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - custom-cilium-helmchart.yaml.j2 - - custom-cilium-l2.yaml.j2 - custom-coredns-helmchart.yaml.j2 -# /var/lib/rancher/k3s/agent/pod-manifests -k3s_server_pod_manifests_templates: - - kube-vip-static-pod.yaml.j2 + - custom-kube-vip-daemonset.yaml.j2 + - custom-kube-vip-rbac.yaml.j2 diff --git a/ansible/pi/playbooks/cluster-installation.yaml b/ansible/pi/playbooks/cluster-installation.yaml index 95b6159533..6915e0339e 100644 --- a/ansible/pi/playbooks/cluster-installation.yaml +++ b/ansible/pi/playbooks/cluster-installation.yaml @@ -21,6 +21,10 @@ k3s_server_manifests_templates: [] k3s_server_manifests_urls: [] + - name: Prevent downgrades + when: k3s_installed.stat.exists + ansible.builtin.include_tasks: tasks/version-check.yaml + - name: Install Kubernetes ansible.builtin.include_role: name: xanmanning.k3s @@ -48,9 +52,8 @@ wait_timeout: 360 loop: - { name: cilium, kind: HelmChart, namespace: kube-system } + - { name: kube-vip, kind: DaemonSet, namespace: kube-system } - { name: coredns, kind: HelmChart, namespace: kube-system } - - { name: policy, kind: CiliumL2AnnouncementPolicy } - - { name: pool, kind: CiliumLoadBalancerIPPool } - name: Coredns when: k3s_primary_control_node diff --git a/ansible/pi/playbooks/cluster-kube-vip.yaml b/ansible/pi/playbooks/cluster-kube-vip.yaml deleted file mode 100644 index f16ff60f07..0000000000 --- a/ansible/pi/playbooks/cluster-kube-vip.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Cluster kube-vip - hosts: master - serial: 1 - become: true - gather_facts: true - any_errors_fatal: true - pre_tasks: - - name: Pausing for 2 seconds... - ansible.builtin.pause: - seconds: 2 - tasks: - - name: Ensure Kubernetes is running - ansible.builtin.include_role: - name: xanmanning.k3s - public: true - vars: - k3s_state: started - - - name: Upgrade kube-vip - ansible.builtin.template: - src: templates/kube-vip-static-pod.yaml.j2 - dest: "{{ k3s_server_pod_manifests_dir }}/kube-vip-static-pod.yaml" - mode: preserve diff --git a/ansible/pi/playbooks/tasks/version-check.yaml b/ansible/pi/playbooks/tasks/version-check.yaml new file mode 100644 index 0000000000..56e5670262 --- /dev/null +++ b/ansible/pi/playbooks/tasks/version-check.yaml @@ -0,0 +1,17 @@ +--- +- name: Version Check + block: + - name: Get deployed k3s version + ansible.builtin.command: k3s --version + register: k3s_version + changed_when: false + failed_when: false + + - name: Extract k3s version + ansible.builtin.set_fact: + current_k3s_version: "{{ k3s_version.stdout | regex_replace('(?im)k3s version (?P[a-z0-9\\.\\+]+).*\n.*', '\\g') }}" + + - name: Check if upgrades are allowed + ansible.builtin.assert: + that: ["k3s_release_version is version(current_k3s_version, '>=')"] + fail_msg: "Unable to upgrade k3s because the deployed version is higher than the one specified in the configuration" diff --git a/ansible/pi/playbooks/templates/custom-cilium-helmchart.yaml.j2 b/ansible/pi/playbooks/templates/custom-cilium-helmchart.yaml.j2 index cc59a6948e..1476911380 100644 --- a/ansible/pi/playbooks/templates/custom-cilium-helmchart.yaml.j2 +++ b/ansible/pi/playbooks/templates/custom-cilium-helmchart.yaml.j2 @@ -9,7 +9,7 @@ spec: # renovate: datasource=helm repo: https://helm.cilium.io/ chart: cilium - version: 1.15.0-rc.1 + version: 1.15.1 targetNamespace: kube-system bootstrap: true valuesContent: |- @@ -17,7 +17,7 @@ spec: bpf: masquerade: true cluster: - name: home-cluster + name: pi id: 1 containerRuntime: integration: containerd @@ -29,16 +29,12 @@ spec: ipam: mode: kubernetes ipv4NativeRoutingCIDR: "{{ cluster_cidr }}" - k8sServiceHost: "{{ kube_vip_addr }}" + k8sServiceHost: 127.0.0.1 k8sServicePort: 6443 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: enabled: true - # https://github.com/cilium/cilium/issues/26586 - leaseDuration: 120s - leaseRenewDeadline: 60s - leaseRetryPeriod: 1s loadBalancer: algorithm: maglev mode: dsr diff --git a/ansible/pi/playbooks/templates/custom-cilium-l2.yaml.j2 b/ansible/pi/playbooks/templates/custom-cilium-l2.yaml.j2 deleted file mode 100644 index 4c889e6283..0000000000 --- a/ansible/pi/playbooks/templates/custom-cilium-l2.yaml.j2 +++ /dev/null @@ -1,22 +0,0 @@ ---- -# https://docs.cilium.io/en/latest/network/l2-announcements -apiVersion: cilium.io/v2alpha1 -kind: CiliumL2AnnouncementPolicy -metadata: - name: policy -spec: - loadBalancerIPs: true - # NOTE: This might need to be set if you have more than one active NIC on your nodes - # interfaces: - # - ^eno[0-9]+ - nodeSelector: - matchLabels: - kubernetes.io/os: linux ---- -apiVersion: cilium.io/v2alpha1 -kind: CiliumLoadBalancerIPPool -metadata: - name: pool -spec: - cidrs: - - cidr: "{{ node_cidr }}" diff --git a/ansible/pi/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 b/ansible/pi/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 new file mode 100644 index 0000000000..e5ae933b62 --- /dev/null +++ b/ansible/pi/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 @@ -0,0 +1,72 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/name: kube-vip +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.7.2 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "{{ k3s_registration_address }}" + - name: vip_arp + value: "true" + - name: lb_enable + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists diff --git a/ansible/pi/playbooks/templates/custom-kube-vip-rbac.yaml.j2 b/ansible/pi/playbooks/templates/custom-kube-vip-rbac.yaml.j2 new file mode 100644 index 0000000000..b3cc6d4d60 --- /dev/null +++ b/ansible/pi/playbooks/templates/custom-kube-vip-rbac.yaml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/ansible/pi/playbooks/templates/kube-vip-static-pod.yaml.j2 b/ansible/pi/playbooks/templates/kube-vip-static-pod.yaml.j2 deleted file mode 100644 index 1c7db334fb..0000000000 --- a/ansible/pi/playbooks/templates/kube-vip-static-pod.yaml.j2 +++ /dev/null @@ -1,59 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip -spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.7.2 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: address - value: "{{ kube_vip_addr }}" - - name: vip_arp - value: "true" - - name: lb_enable - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: prometheus_server - value: :2112 - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - name: kubeconfig - hostPath: - path: /etc/rancher/k3s/k3s.yaml diff --git a/kubernetes/pi/apps/kube-system/cilium/app/helmrelease.yaml b/kubernetes/pi/apps/kube-system/cilium/app/helmrelease.yaml index e6209e1921..71120cd157 100644 --- a/kubernetes/pi/apps/kube-system/cilium/app/helmrelease.yaml +++ b/kubernetes/pi/apps/kube-system/cilium/app/helmrelease.yaml @@ -31,7 +31,7 @@ spec: bpf: masquerade: true cluster: - name: kubernetes + name: pi id: 1 containerRuntime: integration: containerd @@ -67,8 +67,6 @@ spec: ingress: enabled: true className: internal - annotations: - hajimari.io/icon: simple-icons:cilium hosts: - &host hubble.${SECRET_DOMAIN} tls: @@ -83,9 +81,6 @@ spec: kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: enabled: true - leaseDuration: 120s - leaseRenewDeadline: 60s - leaseRetryPeriod: 1s loadBalancer: algorithm: maglev mode: dsr diff --git a/kubernetes/pi/apps/kube-system/kube-vip/app/daemonset.yaml b/kubernetes/pi/apps/kube-system/kube-vip/app/daemonset.yaml new file mode 100644 index 0000000000..9b64fe0ea0 --- /dev/null +++ b/kubernetes/pi/apps/kube-system/kube-vip/app/daemonset.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/name: kube-vip +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.7.1 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "192.168.2.1" + - name: vip_arp + value: "true" + - name: lb_enable + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists diff --git a/kubernetes/pi/apps/kube-system/kube-vip/app/kustomization.yaml b/kubernetes/pi/apps/kube-system/kube-vip/app/kustomization.yaml new file mode 100644 index 0000000000..1217ed4e76 --- /dev/null +++ b/kubernetes/pi/apps/kube-system/kube-vip/app/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./rbac.yaml + - ./daemonset.yaml diff --git a/kubernetes/pi/apps/kube-system/kube-vip/app/rbac.yaml b/kubernetes/pi/apps/kube-system/kube-vip/app/rbac.yaml new file mode 100644 index 0000000000..b3cc6d4d60 --- /dev/null +++ b/kubernetes/pi/apps/kube-system/kube-vip/app/rbac.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/kubernetes/pi/apps/kube-system/kube-vip/ks.yaml b/kubernetes/pi/apps/kube-system/kube-vip/ks.yaml new file mode 100644 index 0000000000..f2b174e652 --- /dev/null +++ b/kubernetes/pi/apps/kube-system/kube-vip/ks.yaml @@ -0,0 +1,21 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app kube-vip + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/pi/apps/kube-system/kube-vip/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/pi/apps/kube-system/kustomization.yaml b/kubernetes/pi/apps/kube-system/kustomization.yaml index 41a295d015..f9108ea667 100644 --- a/kubernetes/pi/apps/kube-system/kustomization.yaml +++ b/kubernetes/pi/apps/kube-system/kustomization.yaml @@ -8,4 +8,5 @@ resources: # Flux-Kustomizations - ./cilium/ks.yaml - ./coredns/ks.yaml + - ./kube-vip/ks.yaml - ./metrics-server/ks.yaml