Merge pull request #1353 from zexi/install-k3s-defaultly

feat: use k3s as cluster defaultly

lib/cluster.py

@@ -36,10 +36,13 @@ class OnecloudCluster(object):
     def use_sudo(self):
         return getuser() != 'root'
 
+    def is_using_k3s(self):
+        return is_using_k3s(self.ssh_client, self.use_sudo())
+
     def __init__(self, ssh_client):
 
         self.ssh_client = ssh_client
-        k3s_cmd_placeholder = 'k3s' if is_using_k3s() else ''
+        k3s_cmd_placeholder = 'k3s' if self.is_using_k3s() else ''
         ret = ssh_client.exec_command(
             f'{k3s_cmd_placeholder} kubectl -n onecloud get onecloudclusters default -o json', self.use_sudo())
         print(GB(f'{k3s_cmd_placeholder} kubectl -n onecloud get onecloudclusters default -o json'))
@@ -95,7 +98,7 @@ def get_current_version(self):
         return self.get_spec().get('version')
 
     def _construct_nodes(self):
-        k3s_cmd_placeholder = 'k3s' if is_using_k3s() else ''
+        k3s_cmd_placeholder = 'k3s' if self.is_using_k3s() else ''
         print(GB(f'{k3s_cmd_placeholder} kubectl get nodes -o json'))
         k8s_nodes = json.loads(self.ssh_client.exec_command(f'{k3s_cmd_placeholder} kubectl get nodes -o json', self.use_sudo())).get('items')
         self.k8s_nodes = [k8s.Node(obj) for obj in k8s_nodes]
@@ -135,7 +138,7 @@ def add_i(node):
         return inventory.generate_content()
 
     def set_current_version(self, version):
-        k3s_cmd_placeholder = 'k3s' if is_using_k3s() else ''
+        k3s_cmd_placeholder = 'k3s' if self.is_using_k3s() else ''
         cmd = f'{k3s_cmd_placeholder} kubectl -n onecloud annotate --overwrite=true onecloudclusters default {A_OCBOOT_UPGRADE_CURRENT_VERSION}={version}'
         print(GB(cmd))
         self.ssh_client.exec_command(cmd, self.use_sudo())

lib/consts.py

@@ -6,3 +6,6 @@
 
 RUNTIME_QEMU = "qemu"
 RUNTIME_CONTAINERD = "containerd"
+ENV_K3S = "K3S"
+ENV_K8S_V115 = "K8S_V115"
+ENV_VAL_TRUE = 'TRUE'

lib/k3s.py

@@ -1,7 +1,9 @@
 import os
 import hashlib
 
+from lib import consts
 from lib.cmd import run_cmd
+from lib.ssh import StderrException
 
 def GET_AIRGAP_DIR():
     default_dir = os.path.join(os.getcwd(), "airgap_assets")
@@ -50,9 +52,27 @@ def download_asset(dest_dir, k3s_version, asset_url):
             print(f"{target_path}'s sha256 checksum {exists_checksum} != {expect_checksum}, redownload it.")
     run_cmd(f'curl -L {asset_url} > {target_path}', no_strip=True, realtime_output=True)
 
-def is_using_k3s():
-    ret = os.environ.get('K3S') == 'TRUE'
-    return ret
+
+def is_using_k3s(ssh_client=None, use_sudo=False):
+    if ssh_client is None:
+        if os.environ.get(consts.ENV_K8S_V115) == consts.ENV_VAL_TRUE:
+            return False
+        return True
+    else:
+        try:
+            kubelet_config = '/etc/kubernetes/kubelet.conf'
+            ret = ssh_client.exec_command(f'ls -alh {kubelet_config}', use_sudo)
+            if kubelet_config in ret:
+                return False
+            return True
+        except StderrException as e:
+            if f'No such file or directory' in str(e):
+                return True
+            else:
+                raise e
+        except Exception as e:
+            raise e
+
 
 def init_airgap_assets(dest_dir, k3s_version):
     # usage: K3S_URL_PREFIX=http://LOCAL_k3s_host_url

lib/ocboot.py

@@ -57,8 +57,9 @@ def load_config(config_file):
         return OcbootConfig(config)
 
 
-def get_ansible_global_vars(version):
-    _is_using_k3s = is_using_k3s()
+def get_ansible_global_vars(version, _is_using_k3s=None):
+    if _is_using_k3s is None:
+        _is_using_k3s = is_using_k3s()
     major_version = utils.get_major_version(version)
     vars = {
         KEY_ONECLOUD_VERSION: version,

lib/service.py

@@ -193,8 +193,8 @@ def __init__(self, cluster, target_nodes, ssh_user, ssh_private_file,
             woker_config_dict['insecure_registries'] = [repo]
         self.worker_config = WorkerConfig(Config(woker_config_dict))
         self.image_repository = cluster.get_image_repository()
-        print("Get current cluster %s version: %s" %
-              (controlplane_host, self.current_version))
+        self.is_using_k3s = cluster.is_using_k3s()
+        utils.pr_green(f"Get current cluster: {controlplane_host}, version: {self.current_version}, is_using_k3s: {self.is_using_k3s}")
 
     def run(self):
         inventory_content = ansible.get_inventory_config(self.worker_config)
@@ -210,7 +210,7 @@ def run(self):
         )
 
     def get_vars(self):
-        vars = get_ansible_global_vars(self.current_version)
+        vars = get_ansible_global_vars(self.current_version, self.is_using_k3s)
         vars[KEY_IMAGE_REPOSITORY] = self.image_repository
         return vars
 

lib/upgrade.py

@@ -145,7 +145,7 @@ def do_upgrade(args):
     with open(inventory_f, 'w') as f:
         f.write(inventory_content)
 
-    vars = config.to_ansible_vars()
+    vars = config.to_ansible_vars(cluster)
     if args.image_repository:
         if args.image_repository == consts.REGISTRY_ALI_YUNION:
             if utils.is_below_v3_9(args.version):
@@ -196,7 +196,7 @@ def get_yunion_yum_repo(self):
         ver = ver[1:]
         return "https://iso.yunion.cn/centos/7/%s/x86_64/yunion.repo" % (ver)
 
-    def to_ansible_vars(self):
+    def to_ansible_vars(self, cluster):
         ret = {
             "current_onecloud_version": self.current_onecloud_version,
             "current_onecloud_major_version": self.current_onecloud_major_version,
@@ -206,6 +206,6 @@ def to_ansible_vars(self):
             "yunion_yum_repo": self.get_yunion_yum_repo(),
             "yunion_qemu_package": "yunion-qemu-4.2.0",
         }
-        g_var = get_ansible_global_vars(self.current_onecloud_version)
+        g_var = get_ansible_global_vars(self.current_onecloud_version, cluster.is_using_k3s())
         ret.update(g_var)
         return ret

ocboot.sh

@@ -2,13 +2,21 @@
 
 set -e
 
-IMAGE_REPOSITORY=${IMAGE_REPOSITORY:-registry.cn-beijing.aliyuncs.com/yunionio}
+DEFAULT_REPO=registry.cn-beijing.aliyuncs.com/yunionio
+IMAGE_REPOSITORY=${IMAGE_REPOSITORY:-$DEFAULT_REPO}
 VERSION=${VERSION:-v4-k3s.4}
 OCBOOT_IMAGE="$IMAGE_REPOSITORY/ocboot:$VERSION"
 
 CUR_DIR="$(pwd)"
 CONTAINER_NAME="buildah-ocboot"
 
+ensure_buildah() {
+    if ! [ -x "$(command -v buildah)" ]; then
+        echo "Installing buildah ..."
+        ./scripts/install-buildah.sh
+    fi
+}
+
 buildah_from_image() {
     if buildah ps | grep $CONTAINER_NAME; then
         buildah rm $CONTAINER_NAME
@@ -18,14 +26,16 @@ buildah_from_image() {
     buildah from --name $CONTAINER_NAME "$img"
 }
 
+ensure_buildah
+
 buildah_from_image "$OCBOOT_IMAGE"
 
 mkdir -p "$HOME/.ssh"
 
 CMD=""
 
 is_ocboot_subcmd() {
-    local subcmds="install upgrade add-node add-lbagent backup restore"
+    local subcmds="install upgrade add-node add-lbagent backup restore setup-container-env"
     for subcmd in $subcmds; do
         if [[ "$1" == "$subcmd" ]]; then
             return 0
@@ -50,9 +60,17 @@ if [[ $buildah_version_major -eq 1 ]] && [[ "$buildah_version_minor" -gt 23 ]];
     buildah_extra_args+=(-e ANSIBLE_VERBOSITY="${ANSIBLE_VERBOSITY:-0}")
 fi
 
+cmd_extra_args=""
+
+if [[ "$1" == "run.py" ]]; then
+    if [[ "$IMAGE_REPOSITORY" != "$DEFAULT_REPO" ]]; then
+        cmd_extra_args="$cmd_extra_args -i $IMAGE_REPOSITORY"
+    fi
+fi
+
 buildah run -t "${buildah_extra_args[@]}" \
     --net=host \
     -v "$HOME/.ssh:/root/.ssh" \
     -v "$(pwd):/ocboot" \
     -v "$(pwd)/airgap_assets/k3s-install.sh:/airgap_assets/k3s-install.sh:ro" \
-    "$CONTAINER_NAME" $CMD "$@"
+    "$CONTAINER_NAME" $CMD "$@" $cmd_extra_args

onecloud/roles/k3s/k3s_server/tasks/main.yml

@@ -68,8 +68,8 @@
     - name: Add K3s autocomplete to user bashrc
       ansible.builtin.lineinfile:
         path: "~{{ ansible_user }}/.bashrc"
-        regexp: '\.\s+<\(k3s completion bash\)'
-        line: ". <(k3s completion bash)  # Added by k3s-ansible"
+        regexp: '\.\s+<\(/usr/local/bin/k3s completion bash\)'
+        line: ". <(/usr/local/bin/k3s completion bash)  # Added by k3s-ansible"
 
     # - name: Change server to API endpoint instead of localhost
     #   ansible.builtin.command: >-

onecloud/roles/upgrade/primary_master_node/tasks/main.yml

@@ -84,7 +84,7 @@
     name: utils/onecloud-waiter
 
 # use async task to upgrade, ref: https://docs.ansible.com/ansible/latest/user_guide/playbooks_async.html
-- name: primary master node | Use ocadm upgrade version "{{ current_onecloud_version }}" to "{{ upgrade_onecloud_version }}"
+- name: primary master node | Upgrade version "{{ current_onecloud_version }}" to "{{ upgrade_onecloud_version }}"
   environment:
     KUBECONFIG: "{{ ENV_KUBECONFIG }}"
     PATH: /opt/yunion/bin:{{ ansible_env.PATH }}

onecloud/roles/utils/detect-os/tasks/main.yml

@@ -22,7 +22,7 @@
 - name: set k3s cmdline env
   set_fact:
     k8s_or_k3s : "k3s"
-    K3S_CMDLINE_PREFIX: 'k3s'
+    K3S_CMDLINE_PREFIX: '/usr/local/bin/k3s'
     ENV_KUBECONFIG: "{{ '~/.kube/config' | expanduser }}"
   when:
   - env_k8s_or_k3s | default('k8s')  == 'k3s'

run.py

@@ -329,7 +329,7 @@ def update_config(yaml_conf, produc_stack, runtime):
     return yaml_conf
 
 
-def generate_config(ipv4, produc_stack, dns_list=[], runtime=consts.RUNTIME_QEMU):
+def generate_config(ipv4, produc_stack, dns_list=[], runtime=consts.RUNTIME_QEMU, image_repository=None):
     global conf
     import os.path
     import os
@@ -338,7 +338,6 @@ def generate_config(ipv4, produc_stack, dns_list=[], runtime=consts.RUNTIME_QEMU
     from lib.get_interface_by_ip import get_interface_by_ip
 
     config_dir = os.getenv("OCBOOT_CONFIG_DIR")
-    image_repository = os.getenv('IMAGE_REPOSITORY')
 
     cur_path = os.path.abspath(os.path.dirname(__file__))
     if not config_dir:
@@ -375,7 +374,7 @@ def generate_config(ipv4, produc_stack, dns_list=[], runtime=consts.RUNTIME_QEMU
     # else set to 'yunionio' namespace if it is daily build.
     # default is 'yunion', for the official and public release.
     elif re.search(r'\b\d{8}\.\d$', ver):
-        yaml_data[ocboot.GROUP_PRIMARY_MASTER_NODE]['image_repository'] = 'registry.cn-beijing.aliyuncs.com/yunionio'
+        yaml_data[ocboot.GROUP_PRIMARY_MASTER_NODE]['image_repository'] = consts.REGISTRY_ALI_YUNIONIO
 
     if image_repository and '5000' in image_repository:
         r = image_repository
@@ -442,8 +441,11 @@ def get_args():
     pip_mirror_suggest = "https://mirrors.aliyun.com/pypi/simple/"
     parser.add_argument('--pip-mirror', '-m', type=str, dest='pip_mirror',
                         help=f"{pip_mirror_help}, e.g.: {pip_mirror_suggest}")
-    parser.add_argument('--k3s', action='store_true', default=False,
-                        help="Using k3s rather than k8s to manage the cluster. Default: False (using k8s)")
+    parser.add_argument('--k8s-v115', action='store_true', default=False,
+                        help="Using old k8s v1.15 rather than k3s to manage the cluster. Default: False (using k3s)")
+    parser.add_argument('--image-repository', '-i', type=str, dest='image_repository',
+                        default=consts.REGISTRY_ALI_YUNIONIO,
+                        help=f"Image repository for container images, e.g.: docker.io/yunion. Default: {consts.REGISTRY_ALI_YUNIONIO}")
     inject_add_hostagent_options(parser)
     inject_add_nodes_runtime_options(parser)
     return parser.parse_args()
@@ -519,8 +521,10 @@ def main():
         'virt': ocboot.KEY_STACK_EDGE,
     }
 
-    if args.k3s:
-        os.environ['K3S'] = 'TRUE'
+    if not args.k8s_v115:
+        os.environ[consts.ENV_K3S] = consts.ENV_VAL_TRUE
+    else:
+        os.environ[consts.ENV_K8S_V115] = consts.ENV_VAL_TRUE
 
     # 1. try to get offline data path from optional args
     # 2. if not exist, try to get from os env
@@ -543,7 +547,7 @@ def main():
             os.system('python3 -m pip install pyyaml')
             ensure_python3_yaml('redhat')
     if match_ip4addr(ip_conf):
-        conf = generate_config(ip_conf, stackDict.get(stack), user_dns, args.runtime)
+        conf = generate_config(ip_conf, stackDict.get(stack), user_dns, args.runtime, args.image_repository)
     elif path.isfile(ip_conf) and path.getsize(ip_conf) > 0:
         conf = update_config(ip_conf, stackDict.get(stack), args.runtime)
     else: