From 784ee4696e30a6c476cb686a23ab23a13a26f41c Mon Sep 17 00:00:00 2001 From: llhuii Date: Fri, 10 Sep 2021 15:42:41 +0800 Subject: [PATCH 1/7] Add all-in-one installation script Signed-off-by: llhuii --- scripts/installation/all-in-one.sh | 494 +++++++++++++++++++ scripts/installation/build-allinone-image.sh | 155 ++++++ 2 files changed, 649 insertions(+) create mode 100755 scripts/installation/all-in-one.sh create mode 100755 scripts/installation/build-allinone-image.sh diff --git a/scripts/installation/all-in-one.sh b/scripts/installation/all-in-one.sh new file mode 100755 index 000000000..2cf3b10b0 --- /dev/null +++ b/scripts/installation/all-in-one.sh @@ -0,0 +1,494 @@ +#!/bin/bash + +# Copyright 2021 The KubeEdge Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script installs a all-in-one Sedna environment, including: +# - A Kubernetes v1.21 cluster with multi worker nodes, default none worker node. +# - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. +# - Sedna, default is latest release version. +# +# It requires you: +# - 2 CPUs or more +# - 2GB+ free memory, depends on node number setting +# - 10GB+ free disk space +# - Internet connection(docker hub, github etc.) +# - Linux platform, such as ubuntu/centos +# - Docker 17.06+ +# +# Advanced options, influential env vars: +# +# NUM_CLOUD_WORKERS | optional | The cloud workers, default 0 +# NUM_EDGE_WORKERS | optional | The KubeEdge workers, default 1 +# KUBEEDGE_VERSION | optional | The KubeEdge version to be installed. +# if not specified, it will get latest release version. +# CLUSTER_NAME | optional | The all-in-one cluster name, default 'sedna-mini' +# FORCE_INSTALL_SEDNA | optional | If 'true', force reinstall Sedna, default false. +# NODE_IMAGE | optional | Custom node image +# REUSE_EDGE_CONTAINER | optional | Whether reuse edge node containers or not, default is true + +set -o errexit +set -o nounset +set -o pipefail + + +function prepare_env() { + : ${CLUSTER_NAME:=sedna-mini} + + # here not use := because it ignore the error of get_latest_version command + if [ -z "${KUBEEDGE_VERSION:-}" ]; then + KUBEEDGE_VERSION=$(get_latest_version kubeedge/kubeedge v1.8.0) + fi + + : ${NUM_CLOUD_WORKERS:=0} + : ${NUM_EDGE_WORKERS:=1} + + # just reuse kind base image + # https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/pkg/build/nodeimage/defaults.go#L23 + + : ${ALLINONE_NODE_IMAGE:=kubeedge/sedna-allinone-node:v1.21.1} + + readonly MAX_CLOUD_WORKERS=2 + readonly MAX_EDGE_WORKERS=3 + + # TODO: find a better way to figure this kind control plane + readonly CONTROL_PLANE_NAME=${CLUSTER_NAME}-control-plane + + # cloudcore default websocket port + : ${CLOUDCORE_WS_PORT:=10000} + # cloudcore default cert port + : ${CLOUDCORE_CERT_PORT:=10002} + + # for debug purpose + : ${RETAIN_CONTAINER:=} + + # use existing edge node containers + # default is true + : ${REUSE_EDGE_CONTAINER:=true} + + # force install sedna control plane + # default is false + : ${FORCE_INSTALL_SEDNA:=false} + + # The docker network for edge nodes to separate the network of control plane. + # Since `kind` CNI doesn't support edge node, here just use the network 'kind'. + # TODO(llhuii): find a way to use the default docker network 'bridge'. + : ${EDGE_NODE_NETWORK:=kind} + + + validate_env +} + +function validate_env() { + + ((NUM_CLOUD_WORKERS<=MAX_CLOUD_WORKERS)) || { + log_fault "Only support NUM_CLOUD_WORKERS at most $MAX_CLOUD_WORKERS" + } + + ((NUM_EDGE_WORKERS<=MAX_EDGE_WORKERS)) || { + log_fault "Only support NUM_EDGE_WORKERS at most $MAX_EDGE_WORKERS" + } +} + + +function _log() { + local level=$1 + shift + timestamp=$(date +"[$level%m%d %H:%M:%S.%3N]") + echo "$timestamp $@" +} + +function log_fault() { + _log E "$@" >&2 + exit 2 +} + +function log_error() { + _log E "$@" >&2 +} + +function log_info() { + _log I "$@" +} + +function gen_kind_config() { + cat </dev/null || keadm init --kubeedge-version=$version --advertise-address=$CLOUDCORE_ADVERTISE_ADDRESSES"' + + # wait token to be created + exit_code=1 + TIMEOUT=30 # in seconds + for((i=1;i<=TIMEOUT; i++)); do + keadm gettoken >/dev/null 2>&1 && exit_code=0 && break + echo -ne "Waiting cloudcore to generate token, $i seconds...\r" + sleep 1 + done + echo + if [ $exit_code -gt 0 ]; then + log_lines=50 + tail -$log_lines /var/log/kubeedge/cloudcore.log | sed "s/^/ /" + echo "Timeout to wait cloudcore, above are the last $log_lines log of cloudcore." + fi + exit $exit_code + ' + KUBEEDGE_TOKEN=$(run_in_control_plane keadm gettoken) +} + +function gen_cni_config() { + cat </dev/null || { + keadm join \ + --cloudcore-ipport=${CLOUDCORE_EXPOSED_ADDR} \ + --certport=${CLOUDCORE_EXPOSED_CERT_PORT} \ + --token=$KUBEEDGE_TOKEN \ + --kubeedge-version '$version' \ + --edgenode-name '$hostname' \ + --remote-runtime-endpoint unix:///var/run/containerd/containerd.sock \ + --runtimetype remote + + # set imageGCHighThreshold to 100% for no image gc + sed -i 's/imageGCHighThreshold:.*/imageGCHighThreshold: 100/' /etc/kubeedge/config/edgecore.yaml && + systemctl restart edgecore || + true # ignore the error + } + + " + gen_cni_config | docker exec -i $containername tee /etc/cni/net.d/10-edgecni.conflist >/dev/null + + done +} + +function clean_edgenodes() { + for cid in $(docker ps -a --filter label=sedna.io=sedna-mini-edge -q); do + docker stop $cid; docker rm $cid + done +} + +function get_docker_network_gw() { + docker network inspect ${1-bridge} --format='{{(index .IPAM.Config 0).Gateway}}' +} + +function setup_cloud() { + create_k8s_cluster + + patch_kindnet + + setup_control_kubeconfig + + setup_cloudcore +} + +function clean_cloud() { + clean_k8s_cluster +} + +function setup_edge() { + create_and_setup_edgenodes +} + +function clean_edge() { + clean_edgenodes +} + +function install_sedna() { + local gm_node=$CONTROL_PLANE_NAME + if run_in_control_plane kubectl get ns sedna; then + if [ "$FORCE_INSTALL_SEDNA" != true ]; then + log_info '"sedna" namespace already exists, no install Sedna control components.' + log_info 'If want to reinstall them, you can remove it `kubectl delete ns sedna` or set FORCE_INSTALL_SEDNA=true!' + log_info + return + fi + run_in_control_plane bash -ec " + curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=clean bash - + " + fi + + log_info "Installing Sedna Control Components..." + + run_in_control_plane bash -ec " + curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=create bash - + " +} + +function get_latest_version() { + # get the latest version of specified gh repo + local repo=${1} default_version=${2} + # output of this latest page: + # ... + # "tag_name": "v1.0.0", + # ... + + # Sometimes this will reach rate limit + # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting + local url=https://api.github.com/repos/$repo/releases/latest + if ! curl --fail -s $url | awk '/"tag_name":/&&$0=$2' | sed 's/[",]//g'; then + log_error "Error to get latest version of $repo: $(curl -s $url | head)" + log_error "Fall back to default version: $default_version" + echo $default_version + fi +} + +function arch() { + local arch=$(uname -m) + case "$arch" in + x86_64) arch=amd64;; + *);; + esac + echo "$arch" +} + +function _download_tool() { + local name=$1 url=$2 + local file=/usr/local/bin/$name + curl -Lo $file $url + chmod +x $file +} + +function check_command_exists() { + type $1 >/dev/null 2>&1 +} + +function ensure_tool() { + local command=$1 download_url=$2 + if check_command_exists $command; then + return + fi + + _download_tool $command $download_url + +} + +function ensure_kind() { + local version=${KIND_VERSION:-0.11.1} + ensure_tool kind https://kind.sigs.k8s.io/dl/v${version/v}/kind-linux-$(arch) +} + +function ensure_kubectl() { + + local version=${KUBECTL_VERSION:-1.21.0} + ensure_tool kubectl https://dl.k8s.io/release/v${version/v}/bin/linux/$(arch)/kubectl +} + +function ensure_tools() { + ensure_kind + ensure_kubectl +} + +function main() { + ensure_tools + prepare_env + action=${1-create} + + case "$action" in + create) + # llh + setup_cloud + setup_edge + install_sedna + log_info "Mini Sedna is created successfully" + ;; + + delete|clean) + clean_edge + clean_cloud + log_info "Mini Sedna is uninstalled successfully" + ;; + + # As a source file, noop + __source__) + ;; + + *) + log_fault "Unknown action $action" + ;; + esac +} + +main "$@" diff --git a/scripts/installation/build-allinone-image.sh b/scripts/installation/build-allinone-image.sh new file mode 100755 index 000000000..e2da7fbed --- /dev/null +++ b/scripts/installation/build-allinone-image.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +# Copyright 2021 The KubeEdge Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script builds the node image for all-in-one Sedna. + +set -o errexit +set -o nounset +set -o pipefail + + +# just reuse kind image +# https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/pkg/build/nodeimage/defaults.go#L23 +# +# Note: here use v1.21.1 of kindest/node, because kubeedge-1.8.0 still uses apiextensions.k8s.io/v1beta1 of CRD, which will be removed in k8s 1.22.0 +readonly BASE_IMAGE=kindest/node:v1.21.1 + +readonly BUILD_IMAGE_NAME=sedna-edge-build-$RANDOM + +function get_latest_version() { + # get the latest version of specified gh repo + local repo=${1} + # output of this latest page: + # ... + # "tag_name": "v1.0.0", + # ... + curl -s https://api.github.com/repos/$repo/releases/latest | awk '/"tag_name":/&&$0=$2' | sed 's/[",]//g' +} + +function create_build_container() { + docker run --rm --name $BUILD_IMAGE_NAME -d --entrypoint sleep $BASE_IMAGE inf || true + + if [ -z "$RETAIN_BUILD_CONTAINER" ]; then + trap clean_build_container EXIT + fi +} + +function clean_build_container() { + docker stop $BUILD_IMAGE_NAME +} + +function run_in_build_container() { + docker exec -i $BUILD_IMAGE_NAME "$@" +} + +function commit_build_container() { + local commit_args=( + docker commit + # put entrypoint back + # https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/images/base/Dockerfile#L203 + --change 'ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]' + $BUILD_IMAGE_NAME $ALLINONE_NODE_IMAGE + ) + "${commit_args[@]}" +} + +function gen_edgecore_config_template() { + run_in_build_container mkdir -p /etc/kubeedge + cat | run_in_build_container mkdir -p /etc/kubeedge +} + +function arch() { + local arch=$(uname -m) + case "$arch" in + x86_64) arch=amd64;; + *);; + esac + echo "$arch" +} + +function install_keadm() { + # download the specified keadm binary + local arch=$(arch) version=${KUBEEDGE_VERSION} + local tarfile=keadm-$version-linux-$arch.tar.gz + local path=keadm-$version-linux-$arch/keadm/keadm + local configdir=/etc/kubeedge + + run_in_build_container bash -euc " + # copy kube config file + # install keadm + curl --fail -LSO https://github.com/kubeedge/kubeedge/releases/download/$version/$tarfile + tar -xvf $tarfile $path + mv $path /usr/local/bin/ + rm $tarfile + + # install dependencies of keadm init/join + apt update -y + apt install -y wget sudo mosquitto + systemctl enable mosquitto + + # install debug tools + apt install -y less sqlite3 + + # add convenient command + echo 'alias docker=crictl' > ~/.bash_aliases + " + + # download the kubeedge into the docker image in advance + download_kubeedge +} + +function download_kubeedge() { + # download the specified kubeedge package for keadm + local arch=$(arch) version=${KUBEEDGE_VERSION} + local tarfile=kubeedge-$version-linux-$arch.tar.gz + local configdir=/etc/kubeedge + + run_in_build_container bash -euc " + mkdir -p $configdir; cd $configdir + curl --fail -LSO https://github.com/kubeedge/kubeedge/releases/download/$version/$tarfile + " +} + +function install_edgecore() { + # download the specified edgecore binary + local arch=$(arch) version=${KUBEEDGE_VERSION} + local tarfile=kubeedge-$version-linux-$arch.tar.gz + local edgecorepath=kubeedge-$version-linux-$arch/edge/edgecore + local configdir=/etc/kubeedge + + run_in_build_container bash -euc " + mkdir -p $configdir; cd $configdir + curl --fail -LSO https://github.com/kubeedge/kubeedge/releases/download/$version/$tarfile + + tar -xvf $tarfile $edgecorepath + mv $edgecorepath /usr/local/bin/ + rm $tarfile + + # install service + curl --fail -LSO https://raw.githubusercontent.com/kubeedge/kubeedge/$version/build/tools/edgecore.service + systemctl enable $configdir/edgecore.service + " +} + +: ${KUBEEDGE_VERSION:=$(get_latest_version kubeedge/kubeedge)} + +: ${NODE_IMAGE:=kubeedge/sedna-allinone-node:v1.21.1} +: ${RETAIN_BUILD_CONTAINER:=} + +create_build_container +install_keadm +commit_build_container + From 8d515fb2e663e2466c8d9c1d62750059c448ce86 Mon Sep 17 00:00:00 2001 From: llhuii Date: Fri, 17 Sep 2021 11:30:52 +0800 Subject: [PATCH 2/7] Add all-in-one docs Signed-off-by: llhuii --- README.md | 2 +- docs/setup/all-in-one.md | 37 +++++++++++++++++++++++++++++++++++++ docs/setup/install.md | 6 +++++- docs/setup/quick-start.md | 6 ++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 docs/setup/all-in-one.md create mode 100644 docs/setup/quick-start.md diff --git a/README.md b/README.md index d7733c678..fbc11abff 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Documentation is located on [readthedoc.io](https://sedna.readthedocs.io/). Thes ### Installation -Follow the [Sedna installation document](docs/setup/install.md) to install Sedna. +Follow the [Sedna installation document](docs/setup/quick-start.md) to install Sedna. ### Examples Example1:[Using Joint Inference Service in Helmet Detection Scenario](/examples/joint_inference/helmet_detection_inference/README.md). diff --git a/docs/setup/all-in-one.md b/docs/setup/all-in-one.md new file mode 100644 index 000000000..339d9e6cd --- /dev/null +++ b/docs/setup/all-in-one.md @@ -0,0 +1,37 @@ +### Deploy All In One Sedna +The [all-in-one script](/scripts/installation/all-in-one.sh) is used to install Sedna along with a mini Kubernetes environment locally, including: + - A Kubernetes v1.21 cluster with multi worker nodes, default zero worker node. + - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. + - Sedna, latest release version. + +It requires you: + - 2 CPUs or more + - 2GB+ free memory, depends on node number setting + - 10GB+ free disk space + - Internet connection(docker hub, github etc.) + - Linux platform, such as ubuntu/centos + - Docker 17.06+ + +For example: + + ```bash + curl https://raw.githubusercontent.com/kubeedge/sedna/master/scripts/installation/all-in-one.sh | NUM_EDGE_WORKERS=2 bash - + ``` + +Above command installs a mini Sedna environment, including: + - A Kubernetes v1.21 cluster with multi worker nodes, default none worker node. + - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. + - Sedna, latest release version. + +You can play it online on [https://www.katacoda.com/sedna-ai/scenarios/all-in-one]. + +Advanced options: +| Env Variable | Description| Default Value| +| --- | --- | --- | +|NUM_CLOUD_WORKERS | The cloud workers| 0| +|NUM_EDGE_WORKERS | The KubeEdge workers| 1| +|KUBEEDGE_VERSION | The KubeEdge version to be installed. |The latest KubeEdge release version| +|CLUSTER_NAME | The all-in-one cluster name| sedna-mini| +|FORCE_INSTALL_SEDNA | If 'true', force to reinstall Sedna|false| +|NODE_IMAGE | Custom node image| kubeedge/sedna-allinone-node:v1.21.1| +|REUSE_EDGE_CONTAINER | Whether reuse edge node containers or not|true| diff --git a/docs/setup/install.md b/docs/setup/install.md index cf44a1414..6fe55c8a6 100644 --- a/docs/setup/install.md +++ b/docs/setup/install.md @@ -1,4 +1,8 @@ -This guide covers how to install Sedna on existing KubeEdge environment, i.e. its two main components: [GM(GlobalManager)](/README.md#globalmanager) and [LC(LocalController)](/README.md#localcontroller). +This guide covers how to install Sedna on an existing Kubernetes environment, i.e. its two main components: [GM(GlobalManager)](/README.md#globalmanager) and [LC(LocalController)](/README.md#localcontroller). + +If you don't have an existing Kubernetes, you can: +1) Install Kubernetes by following the [Kubernetes website](https://kubernetes.io/docs/setup/). +2) Or follow [quick start](quick-start.md) for other options. ### Prerequisites - [Kubectl][kubectl] with right kubeconfig diff --git a/docs/setup/quick-start.md b/docs/setup/quick-start.md new file mode 100644 index 000000000..3d0d8fc5d --- /dev/null +++ b/docs/setup/quick-start.md @@ -0,0 +1,6 @@ +There are some ways to setup Sedna: +- If you have already an existing Kubernetes environment, you can follow [this instruction](./install.md) +- If you have none Kubernetes environment, you can follow [the all-in-one instruction](./all-in-one.md) +- If you want to develop Sedna, you can try [the local-up instruction](./local-up.md) + +After setup Sedna, you can try [examples](/examples). From 7a2d6758a9baf2d5b7e68d7dc3460d493de9ebdf Mon Sep 17 00:00:00 2001 From: llhuii Date: Fri, 17 Sep 2021 15:16:08 +0800 Subject: [PATCH 3/7] Update the setup quick start doc Signed-off-by: llhuii --- docs/setup/install.md | 4 +++- docs/setup/quick-start.md | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/setup/install.md b/docs/setup/install.md index 6fe55c8a6..b514e3c52 100644 --- a/docs/setup/install.md +++ b/docs/setup/install.md @@ -1,4 +1,6 @@ -This guide covers how to install Sedna on an existing Kubernetes environment, i.e. its two main components: [GM(GlobalManager)](/README.md#globalmanager) and [LC(LocalController)](/README.md#localcontroller). +This guide covers how to install Sedna on an existing Kubernetes environment. + +For interested readers, Sedna also has two important components that would be mentioned below, i.e., [GM(GlobalManager)](/README.md#globalmanager) and [LC(LocalController)](/README.md#localcontroller) for workerload generation and maintenance. If you don't have an existing Kubernetes, you can: 1) Install Kubernetes by following the [Kubernetes website](https://kubernetes.io/docs/setup/). diff --git a/docs/setup/quick-start.md b/docs/setup/quick-start.md index 3d0d8fc5d..d063fbb58 100644 --- a/docs/setup/quick-start.md +++ b/docs/setup/quick-start.md @@ -1,6 +1,6 @@ -There are some ways to setup Sedna: -- If you have already an existing Kubernetes environment, you can follow [this instruction](./install.md) -- If you have none Kubernetes environment, you can follow [the all-in-one instruction](./all-in-one.md) -- If you want to develop Sedna, you can try [the local-up instruction](./local-up.md) +There are some ways to set up Sedna, depends on your use case: +- If you have none Kubernetes environment and don't want to install Kubernetes manually, you can follow [the instruction](./all-in-one.md) to install all-in-one Sedna environment. +- Else you can follow [this instruction](./install.md) to install Sedna on existing Kubernetes cluster. +- Also [there is a local up script](./local-up.md) which is mainly used for developing Sedna, it boots a local Kubernetes cluster and installs Sedna based on local Sedna repository. -After setup Sedna, you can try [examples](/examples). +After set up Sedna, you can try out [examples](/examples). From 1a9c82c357b0598fc130ddcc81dada1b5f791858 Mon Sep 17 00:00:00 2001 From: llhuii Date: Fri, 17 Sep 2021 16:09:53 +0800 Subject: [PATCH 4/7] Update all-in-one.md Signed-off-by: llhuii --- docs/setup/all-in-one.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/all-in-one.md b/docs/setup/all-in-one.md index 339d9e6cd..10cfdf498 100644 --- a/docs/setup/all-in-one.md +++ b/docs/setup/all-in-one.md @@ -23,7 +23,7 @@ Above command installs a mini Sedna environment, including: - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. - Sedna, latest release version. -You can play it online on [https://www.katacoda.com/sedna-ai/scenarios/all-in-one]. +You can play it online on [katacoda](https://www.katacoda.com/kubeedge-sedna/scenarios/all-in-one). Advanced options: | Env Variable | Description| Default Value| From 54f1d1b313747d301fa107e4c28dd136389f3312 Mon Sep 17 00:00:00 2001 From: llhuii Date: Fri, 17 Sep 2021 17:19:27 +0800 Subject: [PATCH 5/7] Enhance all-in-one script Signed-off-by: llhuii --- scripts/installation/all-in-one.sh | 48 ++++++++++++++++++------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/scripts/installation/all-in-one.sh b/scripts/installation/all-in-one.sh index 2cf3b10b0..6002c2959 100755 --- a/scripts/installation/all-in-one.sh +++ b/scripts/installation/all-in-one.sh @@ -29,35 +29,43 @@ # # Advanced options, influential env vars: # -# NUM_CLOUD_WORKERS | optional | The cloud workers, default 0 -# NUM_EDGE_WORKERS | optional | The KubeEdge workers, default 1 -# KUBEEDGE_VERSION | optional | The KubeEdge version to be installed. -# if not specified, it will get latest release version. -# CLUSTER_NAME | optional | The all-in-one cluster name, default 'sedna-mini' -# FORCE_INSTALL_SEDNA | optional | If 'true', force reinstall Sedna, default false. -# NODE_IMAGE | optional | Custom node image -# REUSE_EDGE_CONTAINER | optional | Whether reuse edge node containers or not, default is true +# NUM_CLOUD_WORKERS | optional | The cloud workers, default 0 +# NUM_EDGE_WORKERS | optional | The KubeEdge workers, default 1 +# KUBEEDGE_VERSION | optional | The KubeEdge version to be installed. +# if not specified, it try to get latest version or v1.8.0 +# SEDNA_VERSION | optional | The Sedna version to be installed. +# if not specified, it will get latest release or v0.4.1 +# CLUSTER_NAME | optional | The all-in-one cluster name, default 'sedna-mini' +# FORCE_INSTALL_SEDNA | optional | If 'true', force reinstall Sedna, default false. +# NODE_IMAGE | optional | Custom node image +# REUSE_EDGE_CONTAINER | optional | Whether reuse edge node containers or not, default is true set -o errexit set -o nounset set -o pipefail +DEFAULT_SEDNA_VERSION=v0.4.1 +DEFAULT_KUBEEDGE_VERSION=v1.8.0 +DEFAULT_NODE_IMAGE_VERSION=v1.21.1 + + function prepare_env() { : ${CLUSTER_NAME:=sedna-mini} # here not use := because it ignore the error of get_latest_version command if [ -z "${KUBEEDGE_VERSION:-}" ]; then - KUBEEDGE_VERSION=$(get_latest_version kubeedge/kubeedge v1.8.0) + KUBEEDGE_VERSION=$(get_latest_version kubeedge/kubeedge $DEFAULT_KUBEEDGE_VERSION) + fi + + if [ -z "${SEDNA_VERSION:-}" ]; then + SEDNA_VERSION=$(get_latest_version kubeedge/sedna $DEFAULT_SEDNA_VERSION) fi : ${NUM_CLOUD_WORKERS:=0} : ${NUM_EDGE_WORKERS:=1} - # just reuse kind base image - # https://github.com/kubernetes-sigs/kind/blob/4910c3e221a858e68e29f9494170a38e1c4e8b80/pkg/build/nodeimage/defaults.go#L23 - - : ${ALLINONE_NODE_IMAGE:=kubeedge/sedna-allinone-node:v1.21.1} + : ${ALLINONE_NODE_IMAGE:=kubeedge/sedna-allinone-node:$DEFAULT_NODE_IMAGE_VERSION} readonly MAX_CLOUD_WORKERS=2 readonly MAX_EDGE_WORKERS=3 @@ -339,6 +347,7 @@ function create_and_setup_edgenodes() { } " + # fix cni config file gen_cni_config | docker exec -i $containername tee /etc/cni/net.d/10-edgecni.conflist >/dev/null done @@ -381,12 +390,12 @@ function install_sedna() { if run_in_control_plane kubectl get ns sedna; then if [ "$FORCE_INSTALL_SEDNA" != true ]; then log_info '"sedna" namespace already exists, no install Sedna control components.' - log_info 'If want to reinstall them, you can remove it `kubectl delete ns sedna` or set FORCE_INSTALL_SEDNA=true!' + log_info 'If want to reinstall them, you can remove it by `kubectl delete ns sedna` or set FORCE_INSTALL_SEDNA=true!' log_info return fi run_in_control_plane bash -ec " - curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=clean bash - + curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=clean SEDNA_VERSION=$SEDNA_VERSION bash - " fi @@ -399,7 +408,7 @@ function install_sedna() { function get_latest_version() { # get the latest version of specified gh repo - local repo=${1} default_version=${2} + local repo=${1} default_version=${2:-} # output of this latest page: # ... # "tag_name": "v1.0.0", @@ -410,8 +419,10 @@ function get_latest_version() { local url=https://api.github.com/repos/$repo/releases/latest if ! curl --fail -s $url | awk '/"tag_name":/&&$0=$2' | sed 's/[",]//g'; then log_error "Error to get latest version of $repo: $(curl -s $url | head)" - log_error "Fall back to default version: $default_version" - echo $default_version + [ -n "$default_version" ] && { + log_error "Fall back to default version: $default_version" + echo $default_version + } fi } @@ -468,7 +479,6 @@ function main() { case "$action" in create) - # llh setup_cloud setup_edge install_sedna From 479d407408dc143cac7a5a391622342f4418e2fd Mon Sep 17 00:00:00 2001 From: llhuii Date: Sat, 18 Sep 2021 11:41:07 +0800 Subject: [PATCH 6/7] all-in-one: make GM node schedulable When NUM_CLOUD_WORKERS >= 1, kind will taint master node with key='node-role.kubernetes.io/master'. By default, GM will be deployed at master node. So make master node schedulable, and be able to run workloads, i.e. GM here. Signed-off-by: llhuii --- scripts/installation/all-in-one.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/installation/all-in-one.sh b/scripts/installation/all-in-one.sh index 6002c2959..f0eb0e99d 100755 --- a/scripts/installation/all-in-one.sh +++ b/scripts/installation/all-in-one.sh @@ -72,6 +72,7 @@ function prepare_env() { # TODO: find a better way to figure this kind control plane readonly CONTROL_PLANE_NAME=${CLUSTER_NAME}-control-plane + readonly CLOUD_WORKER_NODE_NAME=${CLUSTER_NAME}-worker # cloudcore default websocket port : ${CLOUDCORE_WS_PORT:=10000} @@ -401,7 +402,9 @@ function install_sedna() { log_info "Installing Sedna Control Components..." + run_in_control_plane bash -ec " + kubectl taint $gm_node node-role.kubernetes.io/master- 2>/dev/null || true curl https://raw.githubusercontent.com/kubeedge/sedna/main/scripts/installation/install.sh | SEDNA_GM_NODE=$gm_node SEDNA_ACTION=create bash - " } From cc4952cb5af8306b734e68bd74c84cebc2a21eeb Mon Sep 17 00:00:00 2001 From: llhuii Date: Sat, 18 Sep 2021 10:11:01 +0800 Subject: [PATCH 7/7] Update docs according to review comment Signed-off-by: llhuii --- docs/setup/all-in-one.md | 19 ++++++++------- docs/setup/local-up.md | 38 +++++++++++++++++++++++------- docs/setup/quick-start.md | 8 +++++-- scripts/installation/all-in-one.sh | 24 +++++++++---------- 4 files changed, 58 insertions(+), 31 deletions(-) diff --git a/docs/setup/all-in-one.md b/docs/setup/all-in-one.md index 10cfdf498..a84193017 100644 --- a/docs/setup/all-in-one.md +++ b/docs/setup/all-in-one.md @@ -1,8 +1,8 @@ ### Deploy All In One Sedna The [all-in-one script](/scripts/installation/all-in-one.sh) is used to install Sedna along with a mini Kubernetes environment locally, including: - A Kubernetes v1.21 cluster with multi worker nodes, default zero worker node. - - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. - - Sedna, latest release version. + - KubeEdge with multi edge nodes, default is latest KubeEdge and one edge node. + - Sedna, default is the latest version. It requires you: - 2 CPUs or more @@ -15,22 +15,23 @@ It requires you: For example: ```bash - curl https://raw.githubusercontent.com/kubeedge/sedna/master/scripts/installation/all-in-one.sh | NUM_EDGE_WORKERS=2 bash - + curl https://raw.githubusercontent.com/kubeedge/sedna/master/scripts/installation/all-in-one.sh | NUM_EDGE_NODES=2 bash - ``` Above command installs a mini Sedna environment, including: - - A Kubernetes v1.21 cluster with multi worker nodes, default none worker node. - - KubeEdge with multi nodes, default is latest KubeEdge and one edge node. - - Sedna, latest release version. + - A Kubernetes v1.21 cluster with only one master node. + - KubeEdge with two edge nodes. + - The latest Sedna. You can play it online on [katacoda](https://www.katacoda.com/kubeedge-sedna/scenarios/all-in-one). Advanced options: | Env Variable | Description| Default Value| | --- | --- | --- | -|NUM_CLOUD_WORKERS | The cloud workers| 0| -|NUM_EDGE_WORKERS | The KubeEdge workers| 1| -|KUBEEDGE_VERSION | The KubeEdge version to be installed. |The latest KubeEdge release version| +|NUM_EDGE_NODES | Number of KubeEdge nodes| 1 | +|NUM_CLOUD_WORKER_NODES | Number of cloud **worker** nodes, not master node| 0| +|SEDNA_VERSION | The Sedna version to be installed. |The latest version| +|KUBEEDGE_VERSION | The KubeEdge version to be installed. |The latest version| |CLUSTER_NAME | The all-in-one cluster name| sedna-mini| |FORCE_INSTALL_SEDNA | If 'true', force to reinstall Sedna|false| |NODE_IMAGE | Custom node image| kubeedge/sedna-allinone-node:v1.21.1| diff --git a/docs/setup/local-up.md b/docs/setup/local-up.md index 2d370f43b..271e703c0 100644 --- a/docs/setup/local-up.md +++ b/docs/setup/local-up.md @@ -1,26 +1,48 @@ -### Deploy local Sedna clusters +### Deploy Local Sedna Cluster -Once you have docker running, you can create a local Sedna cluster with: +The [local-up script](/hack/local-up.sh) boots a local Kubernetes cluster, installs latest KubeEdge, and deploys Sedna based on the Sedna local repository. + +#### Use Case +When one is contributing new features for Sedna, codes like AI algorithms under testing can be frequently changed before final deployment. +When coding in that case, s/he would suffer from tortured re-installations and frequent failures of the whole complicated system. +To get rid of the torments, one can use the local-up installation, embraced the single-machine simulation for agiler development and testing. + +#### Setup + +It requires: + - 2 CPUs or more + - 1GB+ free memory + - 5GB+ free disk space + - Internet connection(docker hub, github etc.) + - Linux platform, such as ubuntu/centos + - Docker 17.06+ + - A local Sedna code repository + + +Then you can enter Sedna local code repository, and create a local Sedna cluster with: ``` bash hack/local-up.sh ``` -This script uses [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) to create a -local k8s cluster with one master node, and boots one edge node by running KubeEdge. -You can see them by using `kubectl get nodes -o wide`: + +In more details, this local-up script uses [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) to create a +local K8S cluster with one master node, and joins the K8S cluster by running KubeEdge. + +In another terminal, you can see them by using `kubectl get nodes -o wide`: ```shell NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME edge-node Ready agent,edge 3d21h v1.19.3-kubeedge-v1.6.1 192.168.0.233 Ubuntu 18.04.5 LTS 4.15.0-128-generic docker://20.10.2 sedna-control-plane Ready control-plane,master 3d21h v1.20.2 172.18.0.2 Ubuntu 20.10 4.15.0-128-generic containerd://1.5.0-beta.3-24-g95513021e ``` -You can access master node with: +You can login the master node with: ``` docker exec -it --detach-keys=ctrl-@ sedna-control-plane bash +# since the master node just uses containerd CRI runtime, you can alias the CRI cli 'crictl' as 'docker' alias docker=crictl ``` -Docker images can be loaded into the cluster nodes with: +After you have done developing, built worker image and want to run your worker into master node, your worker image should be loaded into the cluster nodes with: ``` -kind load docker-image my-custom-image --name sedna +kind load docker-image --name sedna ``` diff --git a/docs/setup/quick-start.md b/docs/setup/quick-start.md index d063fbb58..96c62a0ce 100644 --- a/docs/setup/quick-start.md +++ b/docs/setup/quick-start.md @@ -1,6 +1,10 @@ There are some ways to set up Sedna, depends on your use case: - If you have none Kubernetes environment and don't want to install Kubernetes manually, you can follow [the instruction](./all-in-one.md) to install all-in-one Sedna environment. -- Else you can follow [this instruction](./install.md) to install Sedna on existing Kubernetes cluster. -- Also [there is a local up script](./local-up.md) which is mainly used for developing Sedna, it boots a local Kubernetes cluster and installs Sedna based on local Sedna repository. + +- Otherwise, when Kubernetes is ready, you can follow [this instruction](./install.md) to install Sedna on the existing Kubernetes cluster. + +- One more thing: we also have a local-up install script available for easier Sedna development. The local-up version boots a local Kubernetes cluster and installs Sedna based on a local repository. The use case and setup procedures are available [here](./local-up.md). + After set up Sedna, you can try out [examples](/examples). + diff --git a/scripts/installation/all-in-one.sh b/scripts/installation/all-in-one.sh index f0eb0e99d..bd8557c58 100755 --- a/scripts/installation/all-in-one.sh +++ b/scripts/installation/all-in-one.sh @@ -29,8 +29,8 @@ # # Advanced options, influential env vars: # -# NUM_CLOUD_WORKERS | optional | The cloud workers, default 0 -# NUM_EDGE_WORKERS | optional | The KubeEdge workers, default 1 +# NUM_CLOUD_WORKER_NODES| optional | The number of cloud worker nodes, default 0 +# NUM_EDGE_NODES | optional | The number of KubeEdge nodes, default 1 # KUBEEDGE_VERSION | optional | The KubeEdge version to be installed. # if not specified, it try to get latest version or v1.8.0 # SEDNA_VERSION | optional | The Sedna version to be installed. @@ -62,13 +62,13 @@ function prepare_env() { SEDNA_VERSION=$(get_latest_version kubeedge/sedna $DEFAULT_SEDNA_VERSION) fi - : ${NUM_CLOUD_WORKERS:=0} - : ${NUM_EDGE_WORKERS:=1} + : ${NUM_CLOUD_WORKER_NODES:=0} + : ${NUM_EDGE_NODES:=1} : ${ALLINONE_NODE_IMAGE:=kubeedge/sedna-allinone-node:$DEFAULT_NODE_IMAGE_VERSION} - readonly MAX_CLOUD_WORKERS=2 - readonly MAX_EDGE_WORKERS=3 + readonly MAX_CLOUD_WORKER_NODES=2 + readonly MAX_EDGE_WORKER_NODES=3 # TODO: find a better way to figure this kind control plane readonly CONTROL_PLANE_NAME=${CLUSTER_NAME}-control-plane @@ -101,12 +101,12 @@ function prepare_env() { function validate_env() { - ((NUM_CLOUD_WORKERS<=MAX_CLOUD_WORKERS)) || { - log_fault "Only support NUM_CLOUD_WORKERS at most $MAX_CLOUD_WORKERS" + ((NUM_CLOUD_WORKER_NODES<=MAX_CLOUD_WORKER_NODES)) || { + log_fault "Only support NUM_CLOUD_WORKER_NODES at most $MAX_CLOUD_WORKER_NODES" } - ((NUM_EDGE_WORKERS<=MAX_EDGE_WORKERS)) || { - log_fault "Only support NUM_EDGE_WORKERS at most $MAX_EDGE_WORKERS" + ((NUM_EDGE_NODES<=MAX_EDGE_WORKER_NODES)) || { + log_fault "Only support NUM_EDGE_NODES at most $MAX_EDGE_WORKER_NODES" } } @@ -145,7 +145,7 @@ nodes: - containerPort: $CLOUDCORE_CERT_PORT EOF - for((i=0;i