From 85de71b8e3eead9364a0f91a3236add5fbcbf03b Mon Sep 17 00:00:00 2001 From: Dean Taylor Date: Wed, 4 May 2022 11:47:29 +0800 Subject: [PATCH] add cert-manager base configuration add kadalu client for external glusterfs service - works with glusterfs role created add openstack tasks to configure for microk8s, especially metallb setup simplified join to cluster operation --- defaults/main.yml | 32 +++++++++-- handlers/main.yml | 5 ++ tasks/configure-cert-manager.yml | 94 +++++++++++++++++++++++++++++++ tasks/configure-kadalu.yml | 68 ++++++++++++++++++++++ tasks/configure-openstack.yml | 83 +++++++++++++++++++++++++++ tasks/join.yml | 2 +- tasks/main.yml | 90 +++++++++++++++++++++++++---- templates/openstack-cloud.conf.j2 | 32 +++++++++++ vars/main.yml | 7 +++ 9 files changed, 397 insertions(+), 16 deletions(-) create mode 100644 tasks/configure-cert-manager.yml create mode 100644 tasks/configure-kadalu.yml create mode 100644 tasks/configure-openstack.yml create mode 100644 templates/openstack-cloud.conf.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 8468c3d..e6a7f5d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,10 +1,34 @@ --- -# defaults file for microk8s -microk8s_snap_channel: "" +# roles/microk8s/default/main.yml +# +microk8s_alias_enabled: false +microk8s_cert_manager_enabled: true +microk8s_cert_manager_version: v1.8.0 +microk8s_cert_manager_rootCA_issuer: true +microk8s_cloud_service: "" # openstack +microk8s_csr_extra: [] +microk8s_features: [] +microk8s_firewall_enabled: "{{microk8s_cloud_service |length ==0}}" microk8s_group_members: - "{{ ansible_user }}" -microk8s_features: [] -microk8s_csr_extra: [] + +microk8s_kadalu_enabled: false +microk8s_kadalu_gluster_hosts: [] +microk8s_kadalu_version: "0.8.14" +microk8s_kadalu_ssh_priv: "" + +microk8s_openstack_cloud_conf: + auth_url: "" + region: RegionOne + application_credential_id: "" + application_credential_name: "" + application_credential_secret: "" + public_network_name: [] + internal_network_name: [] + ca_bundle_crt: "" +microk8s_openstack_cinder_enabled: true + +microk8s_snap_channel: "" # MetalLB settings (optional) # Only used if microk8s_features contains metallb diff --git a/handlers/main.yml b/handlers/main.yml index 7719e79..bf0f68d 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -4,3 +4,8 @@ - name: csr refresh ansible.builtin.command: cmd: /usr/bin/snap set microk8s csr-refresh=true + +- name: reload facts + ansible.builtin.setup: + filter: + - ansible_local diff --git a/tasks/configure-cert-manager.yml b/tasks/configure-cert-manager.yml new file mode 100644 index 0000000..07be79c --- /dev/null +++ b/tasks/configure-cert-manager.yml @@ -0,0 +1,94 @@ +--- +# roles/microk8s/tasks/configure-cert-manager.yml +# +- name: configure cert-manager + become: false + run_once: "{{run_once}}" + block: + - name: jetstack helm chart + kubernetes.core.helm_repository: + binary_path: "{{helm}}" + repo_name: jetstack + repo_state: present + repo_url: https://charts.jetstack.io + delegate_to: "{{delegate_to}}" + + - name: cert-manager CRDs + ansible.builtin.shell: + cmd: > + {{kubectl}} apply + -f https://github.com/cert-manager/cert-manager/releases/download/{{microk8s_cert_manager_version}}/cert-manager.crds.yaml + register: kubectl_apply_cert_manager_crds + delegate_to: "{{delegate_to}}" + changed_when: kubectl_apply_cert_manager_crds.stdout_lines |reject('search','unchanged') |list |length >0 + + - name: cert-manager + kubernetes.core.helm: + binary_path: "{{helm}}" + chart_ref: jetstack/cert-manager + chart_version: "{{microk8s_cert_manager_version}}" + create_namespace: true + release_name: cert-manager + release_namespace: cert-manager + release_state: present + release_values: + ingressShim.defaultIssuerName: selfsign + ingressShim.defaultIssuerKind: ClusterIssuer + ingressShim.defaultIssuerGroup: cert-manager.io + skip_crds: true + update_repo_cache: false + wait: true + delegate_to: "{{delegate_to}}" + + - name: cert-manager issuer selfsign + ansible.builtin.shell: + cmd: | + cat <0 + + - name: cert-manager root CA + ansible.builtin.shell: + cmd: | + cat <0 + when: + - microk8s_cert_manager_rootCA_issuer diff --git a/tasks/configure-kadalu.yml b/tasks/configure-kadalu.yml new file mode 100644 index 0000000..707b5f1 --- /dev/null +++ b/tasks/configure-kadalu.yml @@ -0,0 +1,68 @@ +--- +# roles/microk8s/tasks/configure-kadalu.yml +# +- ansible.builtin.debug: + var: microk8s_kadalu_gluster_hosts + +- name: kadalu configuration + become: false + run_once: true + block: + - name: kadalu external auth + ansible.builtin.shell: + cmd: | + cat <0 + + - name: kadalu install --type microk8s + ansible.builtin.shell: + cmd: > + {{kubectl}} apply -f + https://github.com/kadalu/kadalu/releases/download/0.8.14/{{item}}.yaml + register: kubectl_apply_kadalu + changed_when: kubectl_apply_kadalu.stdout_lines |reject('search','unchanged') |list |length >0 + delegate_to: "{{delegate_to}}" + with_items: + - kadalu-operator-microk8s + - csi-nodeplugin-microk8s + + - name: kadalu storage class + ansible.builtin.shell: + cmd: | + cat <0 + diff --git a/tasks/configure-openstack.yml b/tasks/configure-openstack.yml new file mode 100644 index 0000000..4ef6e2b --- /dev/null +++ b/tasks/configure-openstack.yml @@ -0,0 +1,83 @@ +--- +# roles/microk8s/tasks/configure-openstack.yml +# https://github.com/kubernetes/cloud-provider-openstack/tree/master/charts/cinder-csi-plugin +# https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md +# +- name: openstack cloud provider + become: false + run_once: "{{run_once}}" + block: + - name: openstack cloud provider settings + ansible.builtin.shell: + cmd: | + cat <0 + + - name: openstack cloud provider charts + kubernetes.core.helm_repository: + binary_path: "{{helm}}" + repo_name: cpo + repo_state: present + repo_url: https://kubernetes.github.io/cloud-provider-openstack + delegate_to: "{{delegate_to}}" + + - name: openstack cinder csi + kubernetes.core.helm: + binary_path: "{{helm}}" + chart_ref: cpo/openstack-cinder-csi + #chart_version: "" + create_namespace: true + release_name: cinder-csi + release_namespace: cloud-provider-openstack + release_state: present + release_values: + csi: + plugin: + volumes: + - name: cacert + configMap: + name: cacert + secret: + enabled: true + name: cloud-config + storageClass: + custom: |- + --- + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: csi-cinder-sc-retain-wffc + provisioner: cinder.csi.openstack.org + reclaimPolicy: Retain + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + delegate_to: "{{delegate_to}}" + when: + - microk8s_openstack_cinder_enabled diff --git a/tasks/join.yml b/tasks/join.yml index 536a9ed..784397f 100644 --- a/tasks/join.yml +++ b/tasks/join.yml @@ -14,7 +14,7 @@ |string }} {{ is_worker |ternary(' --worker','') }}" -- name: reload ansible facts +- name: reload facts ansible.builtin.setup: filter: ansible_local diff --git a/tasks/main.yml b/tasks/main.yml index 060fd86..3ea48a7 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,19 +1,21 @@ --- -# tasks file for microk8s +# roles/microk8s/tasks/main.yml # -- name: microk8s info +- name: microk8s snap info ansible.builtin.shell: cmd: /usr/bin/snap info microk8s changed_when: false - register: microk8s_info + register: microk8s_snap - name: set facts ansible.builtin.set_fact: - channel: "{{ (microk8s_info.stdout |from_yaml).tracking |default('') }}" - install: "{{ (((microk8s_info.stdout |from_yaml).installed is defined) + channel: "{{ (microk8s_snap.stdout |from_yaml).tracking |default('') }}" + helm: "/snap/bin/microk8s.helm3" + install: "{{ (((microk8s_snap.stdout |from_yaml).installed is defined) and (microk8s_snap_channel |length >0) - and ((microk8s_info.stdout |from_yaml).tracking != microk8s_snap_channel)) + and ((microk8s_snap.stdout |from_yaml).tracking != microk8s_snap_channel)) |ternary('upgrade','install') }}" + kubectl: "/snap/bin/microk8s.kubectl" primary_node: "{{ ansible_play_hosts |sort |first }}" is_worker: false @@ -26,7 +28,7 @@ # DNS.5 = kubernetes.default.svc.cluster.local # IP.2 = 10.152.183.1 -- name: Cert DNS +- name: CSR DNS ansible.builtin.blockinfile: block: | {% for csr in microk8s_csr_extra %} @@ -48,6 +50,22 @@ - ansible_local['microk8s']['high-availability']['enabled'] == true - ansible_local['microk8s']['high-availability']['nodes'] | length <= 1 +- name: set fact delegate_to + ansible.builtin.set_fact: + delegate_to: "{{ (ansible_local['microk8s']['high-availability']['nodes'] |length >1) + |ternary(primary_node,'') }}" + run_once: "{{ ansible_local['microk8s']['high-availability']['nodes'] |length >1 }}" + +- name: cloud integration + ansible.builtin.include_tasks: "configure-{{microk8s_cloud_service}}.yml" + when: + - microk8s_cloud_service == "openstack" + +- name: kadalu integration - glusterfs + ansible.builtin.include_tasks: configure-kadalu.yml + when: + - microk8s_kadalu_enabled + - name: microk8s enable features ansible.builtin.shell: cmd: | @@ -57,13 +75,63 @@ else /snap/bin/microk8s enable $FEATURE fi - shell: /bin/bash - delegate_to: "{{ (ansible_local['microk8s']['high-availability']['nodes'] |length >1) - |ternary(primary_node,'') }}" - run_once: "{{ ansible_local['microk8s']['high-availability']['nodes'] |length >1 }}" + executable: /bin/bash + delegate_to: "{{delegate_to}}" + run_once: "{{run_once}}" with_items: - "{{ microk8s_features }}" + - helm3 when: - ansible_local.microk8s is defined - ansible_local.microk8s.addons is defined - (ansible_local.microk8s.addons |selectattr('name','contains', item.split(':')[0] ) |first).status == "disabled" + notify: + - reload facts + +- name: alias to microk8s commands such as kubectl + ansible.builtin.copy: + content: | + alias kubectl='/snap/bin/microk8s.kubectl' + alias helm='/snap/bin/microk8s.helm3' + dest: /etc/profile.d/microk8s-alias.sh + mode: 0644 + when: + - microk8s_alias_enabled + +- name: configure cert-manager + ansible.builtin.include_tasks: configure-cert-manager.yml + when: + - microk8s_cert_manager_enabled + +- name: configure load balancer port for nginx ingress + ansible.builtin.shell: + cmd: | + cat <0 + when: + - microk8s_features is search("metallb") + - microk8s_features is search("ingress") diff --git a/templates/openstack-cloud.conf.j2 b/templates/openstack-cloud.conf.j2 new file mode 100644 index 0000000..8c2331d --- /dev/null +++ b/templates/openstack-cloud.conf.j2 @@ -0,0 +1,32 @@ +{% set cloud_conf = (microk8s_openstack_cloud_conf |combine(microk8s_openstack_cloud_conf_default)) %} +[Global] +auth-url={{cloud_conf.auth_url}} +ca-file={{cloud_conf.ca_file}} +region={{cloud_conf.region}} +application-credential-id={{cloud_conf.application_credential_id}} +application-credential-name={{cloud_conf.application_credential_name}} +application-credential-secret={{cloud_conf.application_credential_secret}} +[BlockStorage] +node-volume-attach-limit=256 +rescan-on-resize=false +ignore-volume-az=false +[Networking] +ipv6-support-disabled={{cloud_conf.ipv6_support_disabled |ternary('true','false')}} +{% if cloud_conf.public_network_name is string %} +public-network-name={{cloud_conf.public_network_name}} +{% else %} +{% for public_network_name in cloud_conf.public_network_name %} +public-network-name={{public_network_name}} +{% endfor %} +{% endif %} +{% if cloud_conf.internal_network_name is string %} +internal-network-name={{cloud_conf.internal_network_name}} +{% else %} +{% for internal_network_name in cloud_conf.internal_network_name %} +internal-network-name={{internal_network_name}} +{% endfor %} +{% endif %} +[LoadBalancer] +enabled={{cloud_conf.loadbalancer_enabled |ternary('true','false')}} +[Metadata] +search-order={{cloud_conf.metadata_search_order}} diff --git a/vars/main.yml b/vars/main.yml index 1f0ddf4..478da09 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,2 +1,9 @@ --- # vars file for microk8s +# +microk8s_openstack_cloud_conf_default: + ca_file: /etc/cacert/ca-bundle.crt + region: RegionOne + ipv6_support_disabled: true + loadbalancer_enabled: false + metadata_search_order: metadataService,configDrive