Enable HA deployments

README.md

@@ -66,6 +66,16 @@ The variables that can be passed to this role and a brief description about them
 	kube_docker_compatible_versions: ['17.03.', '18.06.', '18.09.', '19.03.']
 	# Install docker with pip
 	kube_install_docker_pip
+	# Endpoint for the control plane in case of HA mode with multiple master
+	kube_control_plane_ip: ""
+	kube_control_plane_port: 8443
+	kube_control_plane_peer_ip: "{{ ansible_default_ipv4.address }}"
+	kube_control_plane_peer_iface: "{{ ansible_default_ipv4.interface }}"
+	kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"]
+	# ETCD Peer adress
+	kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}"
+	kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"}
+	kube_etcd_peer_name: "kubeserver.localdomain"
 
 Example Playbook
 ----------------

defaults/main.yml

@@ -1,6 +1,6 @@
 # Version to install or latest
 kube_version: 1.19.14
-# Type of node front or wn
+# Type of node front, control_plane or wn
 kube_type_of_node: front
 # IP address or name of the Kube front node
 kube_server: "{{ ansible_default_ipv4.address }}"
@@ -82,6 +82,19 @@ docker_nvidia_options:
 # Install docker with pip
 kube_install_docker_pip: false
 
+
+# Endpoint for the control plane in case of HA mode with multiple master
+kube_control_plane_ip: ""
+kube_control_plane_port: 8443
+kube_control_plane_peer_ip: "{{ ansible_default_ipv4.address }}"
+kube_control_plane_peer_iface: "{{ ansible_default_ipv4.interface }}"
+kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"]
+# ETCD Peer adress
+kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}"
+kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"}
+kube_etcd_peer_name: "kubeserver.localdomain"
+
+# Kyverno options
 kube_install_kyverno: false
 kyverno_crds_helm_chart_version: "2.0.3"
 kyverno_helm_chart_version: "2.0.3"
@@ -98,4 +111,5 @@ kyverno_podSecurityStandard: baseline
 # Set desired pod security severity low, medium, high. Used severity level in PolicyReportResults for the selected pod security policie
 kyverno_podSecuritySeverity: high
 # Set to get response in failed validation check. Supported values are audit and enforce. See: https://kyverno.io/docs/writing-policies/validate/       
-kyverno_validationFailureAction: ensure 
+kyverno_validationFailureAction: ensure 
+

files/kube-vip.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: kube-vip
+  namespace: kube-system
+spec:
+  containers:
+  - command:
+    - /kube-vip
+    - start
+    - -c
+    - /vip.yaml
+    image: 'ghcr.io/kube-vip/kube-vip:v0.3.8'
+    name: kube-vip
+    resources: {}
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - SYS_TIME
+    volumeMounts:
+    - mountPath: /vip.yaml
+      name: config
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/kube-vip/config.yaml
+    name: config
+status: {}

tasks/control_plane.yaml

@@ -0,0 +1,24 @@
+---
+- name: Configure VIP LB
+  import_tasks: lb.yaml
+
+- name: Configure HA ETDC
+  import_tasks: etcd.yaml
+
+- name: Wait for Kube master
+  wait_for:
+    path: /etc/environment
+    search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf"
+  delegate_to: "{{kube_server}}"
+
+- name: Add node to kube cluster control_plane
+  command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf
+
+- name: Add Kube API server options
+  lineinfile:
+    dest: /etc/kubernetes/manifests/kube-apiserver.yaml
+    line: '    - {{item.option}}={{item.value}}'
+    regexp: '^    - {{item.option}}='
+    insertafter: '    - kube-apiserver'
+  notify: restart kubeapi
+  with_items: "{{ kube_apiserver_options }}"

tasks/etcd.yaml

@@ -0,0 +1,75 @@
+- name: Create kubeadm-config-etcd file
+  template: src=kubeadm-config-etcd.j2 dest=/tmp/kubeadm-config-etcd.yaml
+
+- name: Create /etc/kubernetes/pki/etcd file dir
+  file: path=/etc/kubernetes/pki/etcd state=directory mode=755 recurse=yes
+
+- block:
+
+  - name: Create k8s ca cert
+    command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/ca.crt
+
+  - name: Create k8s sa cert
+    command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/sa.pub
+
+  - name: Create k8s front-proxy-ca cert
+    command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/front-proxy-ca.crt
+
+  - name: Create etcd CA cert
+    command: kubeadm init phase certs etcd-ca creates=/etc/kubernetes/pki/etcd/ca.crt
+
+  when: kube_type_of_node == "front"
+
+- block:
+
+  - name: Wait for certs
+    wait_for:
+      path: "{{item}}"
+    delegate_to: "{{kube_server}}"
+    with_items:
+      - "/etc/kubernetes/pki/etcd/ca.key"
+      - "/etc/kubernetes/pki/etcd/ca.crt"
+      - "/etc/kubernetes/pki/ca.crt"
+      - "/etc/kubernetes/pki/ca.key"
+      - "/etc/kubernetes/pki/sa.pub"
+      - "/etc/kubernetes/pki/sa.key"
+      - "/etc/kubernetes/pki/front-proxy-ca.crt"
+      - "/etc/kubernetes/pki/front-proxy-ca.key"
+
+  - name: Copy cert files from master
+    synchronize:
+      src: "{{item}}"
+      dest: "{{item}}"
+    delegate_to: "{{kube_server}}"
+    with_items:
+      - "/etc/kubernetes/pki/etcd/ca.key"
+      - "/etc/kubernetes/pki/etcd/ca.crt"
+      - "/etc/kubernetes/pki/ca.crt"
+      - "/etc/kubernetes/pki/ca.key"
+      - "/etc/kubernetes/pki/sa.pub"
+      - "/etc/kubernetes/pki/sa.key"
+      - "/etc/kubernetes/pki/front-proxy-ca.crt"
+      - "/etc/kubernetes/pki/front-proxy-ca.key"
+
+  when: kube_type_of_node == "control_plane"
+
+- name: Create etcd server cert
+  command: kubeadm init phase certs etcd-server --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/server.crt
+
+- name: Create etcd peer cert
+  command: kubeadm init phase certs etcd-peer --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/peer.crt
+
+- name: Create etcd healthcheck client cert
+  command: kubeadm init phase certs etcd-healthcheck-client --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/healthcheck-client.crt
+
+- name: Create apiserver-etcd-client server cert
+  command: kubeadm init phase certs apiserver-etcd-client --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/apiserver-etcd-client.crt
+
+- name: Create etcd manifest
+  command: kubeadm init phase etcd local --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/manifests/etcd.yaml
+
+- name: Configure kubelet
+  command: kubeadm init phase kubeconfig kubelet --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/kubelet.conf
+
+- name: Configure kubelet
+  command: kubeadm init phase kubelet-start --config=/tmp/kubeadm-config-etcd.yaml creates=/var/lib/kubelet/config.yaml

tasks/front.yaml

@@ -5,11 +5,26 @@
 - name: force handlers
   meta: flush_handlers
 
+- set_fact:
+    init_params: ""
+
+- block:
+  - name: Configure VIP LB
+    import_tasks: lb.yaml
+
+  - name: Configure HA ETDC
+    import_tasks: etcd.yaml
+
+  - set_fact:
+      init_params: "--ignore-preflight-errors=all"
+
+  when: kube_control_plane_ip != ""
+
 - name: Create kubeadm-config file
   template: src=kubeadm-config.j2 dest=/tmp/kubeadm-config.yml
 
 - name: Kubeadm init
-  command: kubeadm init --config /tmp/kubeadm-config.yml creates=/etc/kubernetes/admin.conf
+  command: kubeadm init --config /tmp/kubeadm-config.yml {{init_params}} creates=/etc/kubernetes/admin.conf 
 
 - name: Set kube_wait_api_server_ip
   set_fact:

tasks/lb.yaml

@@ -0,0 +1,10 @@
+- name: Create /etc/kube-vip file dir
+  file: path=/etc/kube-vip state=directory mode=755
+
+- name: Create kube-vip config file
+  template: src=kube-vip-config.j2 dest=/etc/kube-vip/config.yaml
+
+- name: Create /etc/kubernetes/manifests file dir
+  file: path=/etc/kubernetes/manifests state=directory mode=755 recurse=yes
+
+- copy: src=kube-vip.yaml dest=/etc/kubernetes/manifests/kube-vip.yaml

tasks/main.yaml

@@ -50,5 +50,17 @@
 - name: Include "{{ansible_os_family}}" Kubernetes recipe
   include_tasks: "{{ansible_os_family}}.yaml"
 
+- name: Add KUBELET_EXTRA_ARGS
+  lineinfile:
+    dest: "{{item}}/kubelet"
+    line: 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd {{kubelet_extra_args}}'
+    regexp: 'KUBELET_EXTRA_ARGS='
+    create: yes
+  notify: restart kubelet
+  with_first_found:
+    - files:
+       - /etc/sysconfig/
+       - /etc/default/
+
 - name: Include "{{kube_type_of_node}}" tasks
-  include_tasks: "{{kube_type_of_node}}.yaml"
+  include_tasks: "{{kube_type_of_node}}.yaml"

tasks/wn.yaml

@@ -5,17 +5,10 @@
     search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf"
   delegate_to: "{{kube_server}}"
 
-- name: Add KUBELET_EXTRA_ARGS
-  lineinfile:
-    dest: "{{item}}/kubelet"
-    line: 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd {{kubelet_extra_args}}'
-    regexp: 'KUBELET_EXTRA_ARGS='
-    create: yes
-  notify: restart kubelet
-  with_first_found:
-    - files:
-       - /etc/sysconfig/
-       - /etc/default/
-
-- name: Add node to kube cluster
+- name: Add WN to kube cluster to master
   command: kubeadm join --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf
+  when: kube_control_plane_ip == ""
+
+- name: Add WN to kube cluster control plane
+  command: kubeadm join --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf
+  when: kube_control_plane_ip != ""

templates/kube-vip-config.j2

@@ -0,0 +1,27 @@
+localPeer:
+  id: 0
+  address: {{kube_control_plane_peer_ip}}
+  port: 10000
+remotePeers:
+{% for peer_ip in kube_control_plane_remote_peer_list %}
+- id: {{loop.index + 1}}
+  address: {{peer_ip}}
+  port: 10000
+{% endfor %}
+vip: {{kube_control_plane_ip}}
+gratuitousARP: true
+singleNode: false
+startAsLeader: true
+interface: {{kube_control_plane_peer_iface}}
+loadBalancers:
+- name: API Server Load Balancer
+  type: tcp
+  port: {{kube_control_plane_port}}
+  bindToVip: false
+  backends:
+  - port: 6443
+    address: {{kube_control_plane_peer_ip}}
+{% for peer_ip in kube_control_plane_remote_peer_list %}
+  - port: 6443
+    address: {{peer_ip}}
+{% endfor %}

templates/kubeadm-config-etcd.j2

@@ -0,0 +1,30 @@
+---
+kind: ClusterConfiguration
+apiVersion: kubeadm.k8s.io/v1beta2
+networking:
+  podSubnet: "{{kube_pod_network_cidr}}"
+apiServer:
+  extraArgs:
+    advertise-address: "{{kube_api_server}}"
+controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}}
+etcd:
+  local:
+    extraArgs:
+      listen-client-urls: "https://{{kube_etcd_peer_address}}:2379"
+      listen-peer-urls: "https://{{kube_etcd_peer_address}}:2380"
+      advertise-client-urls: "https://{{kube_etcd_peer_address}}:2379"
+      initial-advertise-peer-urls: "https://{{kube_etcd_peer_address}}:2380"
+      initial-cluster: "{%- for peer_name, peer_ip in kube_etcd_peer_list.items() -%}
+        {%- if loop.index > 1 -%} , {%- endif -%}
+        {{peer_name}}=https://{{peer_ip}}:2380
+      {%- endfor -%}"
+      name: "{{kube_etcd_peer_name}}"
+      initial-cluster-state: "new"
+    peerCertSANs:
+      - "{{kube_etcd_peer_address}}"
+    serverCertSANs:
+      - "{{kube_etcd_peer_address}}"
+---
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: systemd

templates/kubeadm-config.j2

@@ -6,6 +6,18 @@ networking:
 apiServer:
   extraArgs:
     advertise-address: "{{kube_api_server}}"        # --apiserver-advertise-address
+{% if kube_control_plane_ip != "" %}
+controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}}
+etcd:
+    external:
+        endpoints:
+        {% for peer_name, peer_ip in kube_etcd_peer_list.items() %}
+          - https://{{peer_ip}}:2379
+        {% endfor -%}
+        caFile: /etc/kubernetes/pki/etcd/ca.crt
+        certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
+        keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
+{% endif %}
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1