Merge pull request #172 from grycap/cri-dockerd

Add support to cri-dockerd #168

.github/workflows/main.yaml

@@ -117,5 +117,40 @@ jobs:
       - name: Basic role check in wn
         run: sudo ansible-playbook tests/test-crio.yml -i tests/inventory -e kube_type_of_node=wn -e kube_server=localhost
 
+      - name: Test nodes
+        run: sudo kubectl -s https://localhost:6443 --insecure-skip-tls-verify --kubeconfig /etc/kubernetes/admin.conf  get nodes
+
+  test-docker:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Install python
+        run: sudo apt update && sudo apt install -y python3 python3-pip python3-setuptools
+
+      - name: Remove pre-installed kubectl to avoid errors
+        run: sudo apt remove buildah podman -y
+
+      - name: Install Ansible
+        run: sudo pip3 install dnspython ansible==2.9.22
+
+      - name: Create ansible.cfg with correct roles_path
+        run: sudo printf '[defaults]\nhost_key_checking = False\nroles_path=../' > ansible.cfg
+
+      - name: Install geerlingguy.ntp
+        run: sudo ansible-galaxy install geerlingguy.ntp grycap.docker grycap.cri_o
+
+      - name: Basic role syntax check
+        run: sudo ansible-playbook tests/test-docker.yml -i tests/inventory --syntax-check
+
+      - name: Basic role check in front
+        run: sudo ansible-playbook tests/test-docker.yml -i tests/inventory
+
+      - name: Basic role check in wn
+        run: sudo ansible-playbook tests/test-docker.yml -i tests/inventory -e kube_type_of_node=wn -e kube_server=localhost
+
       - name: Test nodes
         run: sudo kubectl -s https://localhost:6443 --insecure-skip-tls-verify --kubeconfig /etc/kubernetes/admin.conf  get nodes

README.md

@@ -61,12 +61,9 @@ The variables that can be passed to this role and a brief description about them
 	# Email to be used in the Let's Encrypt issuer
 	kube_cert_user_email: [email protected]
 	# Override default docker version
-	# (installed when not in kube_docker_compatible_versions)
 	kube_docker_version: ""
 	# Options to add in the docker.json file
 	kube_docker_options: {}
-	# Compatible docker versions
-	kube_docker_compatible_versions: ['17.03.', '18.06.', '18.09.', '19.03.']
 	# Install docker with pip
 	kube_install_docker_pip
 	# Command flags to use for launching k3s in the systemd service

defaults/main.yml

@@ -17,11 +17,18 @@ kube_pod_network_cidr: 10.244.0.0/16
 # Type of network to install: currently supported: flannel, kube-router, calico, weave
 kube_network: flannel
 # Kubelet extra args
-kubelet_extra_args: ''
+kubelet_extra_args: '' # deprecated move to kubelet_extra_args_dict
+# dict of kubelet extra args, if set kubelet_extra_args is ignored
+# A key in this map is the flag name as it appears on the command line except without leading dash(es).
+kubelet_extra_args_dict: {}
 # Kube API server options
 kube_apiserver_options: []
 # CRI runtime
 kube_cri_runtime: docker # docker, containerd or crio
+# Install CRI runtime
+kube_cri_runtime_install: true
+# CRI dockerd version
+kube_cri_dockerd_version: "0.3.4"
 # Flag to set HELM to be installed
 kube_install_helm: true
 # Helm version
@@ -79,12 +86,9 @@ kube_cert_manager_challenge_dns01_sk: ''
 # Optionally a wildcard dns certificate name can be set
 kube_cert_manager_wildcard_cert_dns_name: ''
 # Override default docker version
-# (installed when not in kube_docker_compatible_versions)
 kube_docker_version: ""
 # Options to add in the docker.json file
 kube_docker_options: {}
-# Compatible docker versions
-kube_docker_compatible_versions: ['17.03.', '18.06.', '18.09.', '19.03.']
 # Nvidia docker options to add in the docker.json file
 docker_nvidia_options:
   default-runtime: nvidia

meta/main.yml

@@ -20,27 +20,25 @@ dependencies:
     ntp_servers: "{{ kube_ntp_servers }}"
     when: kube_ntp_servers != []
   - role: 'grycap.docker'
-    docker_version: "{{ kube_docker_version | default('19.03.15', true) }}"
-    docker_compatible_versions: "{{kube_docker_compatible_versions}}"
+    docker_version: "{{ kube_docker_version | default('latest', true) }}"
     docker_config_values: "{{ {'exec-opts': ['native.cgroupdriver=systemd'], 'log-driver': 'json-file', 'log-opts': {'max-size': '100m'}, 'storage-driver': 'devicemapper'} | combine(kube_docker_options) }}"
     docker_nvidia_support: '{{ kube_nvidia_support and kube_type_of_node == "wn" }}'
     docker_install_criu: false
     docker_install_pip: "{{kube_install_docker_pip}}"
     docker_compose_version: ""
     docker_containerd_only: "{{ (kube_cri_runtime == 'containerd') | bool }}"
     docker_nvidia_driver_version: "{{ kube_nvidia_driver_version }}"
-    when: ansible_os_family == "RedHat" and kube_install_method == 'kubeadm' and kube_cri_runtime != 'crio'
+    when: ansible_os_family == "RedHat" and kube_install_method == 'kubeadm' and kube_cri_runtime != 'crio' and kube_cri_runtime_install
   - role: 'grycap.docker'
-    docker_version: "{{ kube_docker_version | default('5:19.03.11~3-0~' + (ansible_distribution | lower) + '-' + ansible_distribution_release, true) }}"
-    docker_compatible_versions: "{{kube_docker_compatible_versions}}"
+    docker_version: "{{ kube_docker_version | default('latest', true) }}"
     docker_config_values: "{{ {'exec-opts': ['native.cgroupdriver=systemd'], 'log-driver': 'json-file', 'log-opts': {'max-size': '100m'}, 'storage-driver': 'overlay2'} | combine(kube_docker_options) }}"
     docker_nvidia_support: '{{ kube_nvidia_support and kube_type_of_node == "wn" }}'
     docker_install_criu: false
     docker_install_pip: "{{kube_install_docker_pip}}"
     docker_compose_version: ""
     docker_containerd_only: "{{ (kube_cri_runtime == 'containerd') | bool }}"
     docker_nvidia_driver_version: "{{ kube_nvidia_driver_version }}"
-    when: ansible_os_family == "Debian" and kube_install_method == 'kubeadm' and kube_cri_runtime != 'crio'
+    when: ansible_os_family == "Debian" and kube_install_method == 'kubeadm' and kube_cri_runtime != 'crio' and kube_cri_runtime_install
   - role: 'grycap.cri_o'
-    when: ansible_os_family == "Debian" and kube_install_method == 'kubeadm' and kube_cri_runtime == 'crio'
+    when: ansible_os_family == "Debian" and kube_install_method == 'kubeadm' and kube_cri_runtime == 'crio' and kube_cri_runtime_install
 

tasks/cert-manager.yaml

@@ -27,6 +27,7 @@
     command: kubectl apply -f /tmp/cert-manager.yaml
     environment:
       KUBECONFIG: "{{KUBECONFIG}}"
+    register: cert_manager
 
   - name: Wait for cert-manager ready status
     command: kubectl rollout status deployment/{{ item }} -n cert-manager
@@ -37,11 +38,17 @@
       - cert-manager-cainjector
       - cert-manager-webhook
 
-  - template: src=dns01_secret.j2 dest=/tmp/dns01_secret.yaml
-  - name: Create DNS01 secret
-    command: kubectl apply -f /tmp/dns01_secret.yaml
-    environment:
-      KUBECONFIG: "{{KUBECONFIG}}"
+  - name: Pause for 5 seconds to wait cert-manager to fully start
+    pause:
+      seconds: 5
+    when: cert_manager is changed
+
+  - block:
+    - template: src=dns01_secret.j2 dest=/tmp/dns01_secret.yaml
+    - name: Create DNS01 secret
+      command: kubectl apply -f /tmp/dns01_secret.yaml
+      environment:
+        KUBECONFIG: "{{KUBECONFIG}}"
     when: kube_cert_manager_challenge == "dns01"
 
   - template: src=prod_issuer.j2 dest=/tmp/prod_issuer.yaml
@@ -50,11 +57,12 @@
     environment:
       KUBECONFIG: "{{KUBECONFIG}}"
 
-  - template: src=wildcard_cert.j2 dest=/tmp/wildcard_cert.yaml
-  - name: Create Let's encrypt Wildcard Certificate for '{{ kube_cert_manager_wildcard_cert_dns_name }}'
-    command: kubectl apply -f /tmp/wildcard_cert.yaml
-    environment:
-      KUBECONFIG: "{{KUBECONFIG}}"
+  - block:
+    - template: src=wildcard_cert.j2 dest=/tmp/wildcard_cert.yaml
+    - name: Create Let's encrypt Wildcard Certificate for '{{ kube_cert_manager_wildcard_cert_dns_name }}'
+      command: kubectl apply -f /tmp/wildcard_cert.yaml
+      environment:
+        KUBECONFIG: "{{KUBECONFIG}}"
     when: kube_cert_manager_wildcard_cert_dns_name != '' and kube_cert_manager_challenge == 'dns01'
 
   when: kube_cert_manager | bool

tasks/cri-dockerd.yaml

@@ -0,0 +1,36 @@
+---
+- name: Download cri-dockerd tarball
+  get_url:
+    url: https://github.com/Mirantis/cri-dockerd/releases/download/v{{ kube_cri_dockerd_version }}/cri-dockerd-{{ kube_cri_dockerd_version }}.amd64.tgz
+    dest: /tmp/cri-dockerd-{{ kube_cri_dockerd_version }}.amd64.tgz
+
+- name: Extract cri-dockerd tarball
+  unarchive:
+    src: /tmp/cri-dockerd-{{ kube_cri_dockerd_version }}.amd64.tgz
+    dest: /tmp
+    remote_src: yes
+
+- name: Copy cri-dockerd binary
+  copy:
+    src: /tmp/cri-dockerd/cri-dockerd
+    dest: /usr/bin/cri-dockerd
+    mode: '0755'
+
+- name: Download cri-docker service and socket
+  get_url:
+    url: https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/{{ item }}
+    dest: /etc/systemd/system/{{ item }}
+    mode: '0644'
+  loop: 
+    - cri-docker.service
+    - cri-docker.socket
+
+- name: Enable cri-dockerd service
+  systemd:
+    name: "{{ item }}"
+    enabled: yes
+    daemon_reload: yes
+    state: started
+  loop: 
+    - cri-docker.service
+    - cri-docker.socket

tasks/kubeadm.yaml

@@ -1,3 +1,7 @@
+- name: Include cri-docker tasks
+  include_tasks: "cri-dockerd.yaml"
+  when: kube_cri_runtime == "docker"
+
 - name: Check kube version
   shell: kubeadm version -o short | cut -d 'v' -f 2
   register: kubeadm_output

tasks/wn.yaml

@@ -42,6 +42,7 @@
 
 - block:
 
+  # to deprecate and move to kubelet_extra_args_dict
   - name: Add KUBELET_EXTRA_ARGS
     lineinfile:
       dest: "{{item}}/kubelet"
@@ -54,9 +55,13 @@
         - /etc/sysconfig/
         - /etc/default/
     ignore_errors: true
+    when: kubelet_extra_args != "" and kubelet_extra_args_dict == {}
+
+  - name: Create kubeadm-config file
+    template: src=kubeadm-config-join.j2 dest=/tmp/kubeadm-config.yml
 
   - name: Add node to kube cluster
-    command: kubeadm join --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf
+    command: kubeadm join --config /tmp/kubeadm-config.yml creates=/etc/kubernetes/kubelet.conf
 
   when: kube_install_method == "kubeadm"
 

templates/kubeadm-config-join.j2

@@ -0,0 +1,21 @@
+---
+kind: JoinConfiguration
+{% if kube_version is version_compare('1.22.0', '<') %}
+apiVersion: kubeadm.k8s.io/v1beta2
+{% else %}
+apiVersion: kubeadm.k8s.io/v1beta3
+{% endif %}
+nodeRegistration:
+  kubeletExtraArgs:
+    cgroup-driver: systemd
+{% for key, value in kubelet_extra_args_dict.items() %}
+    {{key}}: {{value}}
+{% endfor %}
+{% if kube_cri_runtime == "docker" %}
+  criSocket: "/run/cri-dockerd.sock"
+{% endif %}
+discovery:
+  bootstrapToken:
+    token: "{{kube_token}}"
+    apiServerEndpoint: {{kube_server}}:6443
+    unsafeSkipCAVerification: true

templates/kubeadm-config.j2

@@ -41,3 +41,7 @@ bootstrapTokens:
 - token: "{{kube_token}}"                          # --token
   description: "kubeadm bootstrap token"
   ttl: "{{kube_token_ttl}}"                         # --token-ttl
+{% if kube_cri_runtime == "docker" %}
+nodeRegistration:
+  criSocket: "/run/cri-dockerd.sock"
+{% endif %}

tests/test-crio.yml

@@ -11,4 +11,5 @@
       kube_public_dns_name: test.domain.com
       kube_version: 1.25.3
       kube_cri_runtime: crio
-      kubelet_extra_args:  '-node-labels=somelabel'
+      kubelet_extra_args_dict:
+        node-labels: somelabel

tests/test-docker.yml

@@ -0,0 +1,17 @@
+---
+- hosts: localhost
+  roles:
+    - role: ansible-role-kubernetes
+      kube_install_metrics: true
+      kube_cert_manager: true
+      kube_install_kubeapps: false
+      kube_install_kyverno: false
+      kube_deploy_dashboard: true
+      kube_install_ingress: true
+      kube_public_dns_name: test.domain.com
+      kube_version: 1.25.3
+      kube_cri_runtime: docker
+      kube_cri_runtime_install: false
+      kube_install_docker_pip: true
+      kubelet_extra_args_dict: 
+        node-labels: somelabel

tests/test.yml

@@ -9,9 +9,8 @@
       kube_deploy_dashboard: true
       kube_install_ingress: true
       kube_public_dns_name: test.domain.com
-      kube_docker_options:
-        data-root: /var/lib/docker
       kube_version: 1.25.3
       kube_cri_runtime: containerd
       kube_install_docker_pip: true
-      kubelet_extra_args:  '-node-labels=somelabel'
+      kubelet_extra_args_dict: 
+        node-labels: somelabel