From 556bcb85def25bca22d73c2a56e9dee384b9056b Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 4 Aug 2017 18:54:17 +0800 Subject: [PATCH 001/167] update image build script. --- image/build-image.sh | 9 +++++ image/install-pkg.sh | 2 ++ image/install-qingcloud-agent.sh | 13 +++++++ image/pre-install.sh | 8 +++++ image/pre_install.sh | 41 ---------------------- image/{update-bin.sh => update-k8s-bin.sh} | 6 ++-- image/update-sysconfig.sh | 21 +++++++++++ 7 files changed, 57 insertions(+), 43 deletions(-) create mode 100644 image/build-image.sh create mode 100644 image/install-qingcloud-agent.sh create mode 100755 image/pre-install.sh delete mode 100755 image/pre_install.sh rename image/{update-bin.sh => update-k8s-bin.sh} (72%) create mode 100644 image/update-sysconfig.sh diff --git a/image/build-image.sh b/image/build-image.sh new file mode 100644 index 0000000..47b0e32 --- /dev/null +++ b/image/build-image.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +K8S_HOME=$(dirname "${SCRIPTPATH}") + +${K8S_HOME}/script/install-pkg.sh +${K8S_HOME}/script/install-qingcloud-agent.sh +${K8S_HOME}/script/update-sysconfig.sh +${K8S_HOME}/script/update-k8s-bin.sh \ No newline at end of file diff --git a/image/install-pkg.sh b/image/install-pkg.sh index c50a112..50cae71 100755 --- a/image/install-pkg.sh +++ b/image/install-pkg.sh @@ -8,6 +8,8 @@ apt-cache policy docker-engine apt-get install -y docker-engine apt-get install -y ebtables socat +apt-get install -y jq apt-transport-https +apt-get remove network-manager apt-get upgrade -y diff --git a/image/install-qingcloud-agent.sh b/image/install-qingcloud-agent.sh new file mode 100644 index 0000000..d00d520 --- /dev/null +++ b/image/install-qingcloud-agent.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd /tmp +wget https://pek3a.qingstor.com/appcenter/developer/packages/app-agent-linux-amd64.tar.gz +tar -zxvf app-agent-linux-amd64.tar.gz +cd app-agent-linux-amd64/ +./install.sh + +chmod +x /etc/init.d/confd + +cd /tmp +rm -rf app-agent-linux-amd64/ +rm app-agent-linux-amd64.tar.gz \ No newline at end of file diff --git a/image/pre-install.sh b/image/pre-install.sh new file mode 100755 index 0000000..f180b7f --- /dev/null +++ b/image/pre-install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +apt-get install -y git wget +git clone https://github.com/QingCloudAppcenter/kubernetes.git /opt/kubernetes +cd /opt/kubernetes/image + +./build-image.sh + diff --git a/image/pre_install.sh b/image/pre_install.sh deleted file mode 100755 index bb7f63c..0000000 --- a/image/pre_install.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -apt-get install -y git wget jq apt-transport-https - -cd /tmp -wget https://pek3a.qingstor.com/appcenter/developer/packages/app-agent-linux-amd64.tar.gz -tar -zxvf app-agent-linux-amd64.tar.gz -cd app-agent-linux-amd64/ -./install.sh - -chmod +x /etc/init.d/confd - -cd /tmp -rm -rf app-agent-linux-amd64/ -rm app-agent-linux-amd64.tar.gz - -apt-get remove network-manager - - -cat << EOF > /etc/sysctl.conf -net.ipv4.ip_forward = 1 -vm.swappiness = 1 -net.ipv6.conf.all.disable_ipv6 = 1 -net.ipv6.conf.default.disable_ipv6 = 1 -net.ipv6.conf.lo.disable_ipv6 = 1 -net.ipv4.conf.all.rp_filter = 2 -vm.max_map_count=262144 -fs.file-max=200000 -fs.inotify.max_user_watches=1048576 -EOF - -sysctl -p - -#only allow dhcp to manager eth0, not auto hotplug other interface. -cat << EOF > /etc/network/interfaces -auto lo -iface lo inet loopback - -allow-hotplug eth0 -iface eth0 inet dhcp -EOF \ No newline at end of file diff --git a/image/update-bin.sh b/image/update-k8s-bin.sh similarity index 72% rename from image/update-bin.sh rename to image/update-k8s-bin.sh index 49a884c..ee91685 100755 --- a/image/update-bin.sh +++ b/image/update-k8s-bin.sh @@ -5,8 +5,8 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") echo "update bin" -k8s_bins=("kubelet" "kubectl" "kubeadm") -k8s_base_url="http://k8s-qingcloud.pek3a.qingstor.com/k8s/release/bin/linux/amd64" +k8s_bins=("kubelet" "kubectl") +k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/1.7.3/bin/" k8s_bin_path="/usr/bin" function download_k8s_bin() @@ -20,6 +20,8 @@ function download_k8s_bin() unlink "${k8s_bin_path}/${bin}" ln -s "${K8S_HOME}/bin/${bin}" "${k8s_bin_path}/${bin}" done + # use custom kubeadm + wget https://pek3a.qingstor.com/k8s-qingcloud/k8s/release/bin/linux/amd64/kubeadm chmod +x * popd } diff --git a/image/update-sysconfig.sh b/image/update-sysconfig.sh new file mode 100644 index 0000000..6c5fcee --- /dev/null +++ b/image/update-sysconfig.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +cat << EOF > /etc/sysctl.conf +net.ipv4.ip_forward = 1 +vm.swappiness = 1 +net.ipv4.conf.all.rp_filter = 2 +vm.max_map_count=262144 +fs.file-max=200000 +fs.inotify.max_user_watches=1048576 +EOF + +sysctl -p + +#only allow dhcp to manager eth0, not auto hotplug other interface. +cat << EOF > /etc/network/interfaces +auto lo +iface lo inet loopback + +allow-hotplug eth0 +iface eth0 inet dhcp +EOF \ No newline at end of file From faf98a29e0c03d7e0098bbc748bbb925a42b4b74 Mon Sep 17 00:00:00 2001 From: jolestar Date: Sat, 5 Aug 2017 11:43:07 +0800 Subject: [PATCH 002/167] update image build script. --- image/build-image.sh | 13 +++++++++---- image/{pull-images.sh => pull-docker-images.sh} | 0 image/update-cni.sh | 5 +---- image/update-k8s-bin.sh | 4 +--- image/update.sh | 7 ++----- 5 files changed, 13 insertions(+), 16 deletions(-) rename image/{pull-images.sh => pull-docker-images.sh} (100%) diff --git a/image/build-image.sh b/image/build-image.sh index 47b0e32..8bdc465 100644 --- a/image/build-image.sh +++ b/image/build-image.sh @@ -3,7 +3,12 @@ SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) K8S_HOME=$(dirname "${SCRIPTPATH}") -${K8S_HOME}/script/install-pkg.sh -${K8S_HOME}/script/install-qingcloud-agent.sh -${K8S_HOME}/script/update-sysconfig.sh -${K8S_HOME}/script/update-k8s-bin.sh \ No newline at end of file +${K8S_HOME}/image/install-pkg.sh +${K8S_HOME}/image/install-qingcloud-agent.sh +${K8S_HOME}/image/update-confd.sh +${K8S_HOME}/image/update-sysconfig.sh +${K8S_HOME}/image/update-k8s-bin.sh +${K8S_HOME}/image/update-cni.sh +${K8S_HOME}/image/update-logrotate.sh +${K8S_HOME}/image/install-systemd-conf.sh +${K8S_HOME}/image/pull-docker-images.sh \ No newline at end of file diff --git a/image/pull-images.sh b/image/pull-docker-images.sh similarity index 100% rename from image/pull-images.sh rename to image/pull-docker-images.sh diff --git a/image/update-cni.sh b/image/update-cni.sh index 1b60860..3e4afe7 100755 --- a/image/update-cni.sh +++ b/image/update-cni.sh @@ -18,7 +18,4 @@ wget -c https://pek3a.qingstor.com/k8s-qingcloud/k8s/tool/hostnic/${HOSTNIC_VERS tar -zxvf hostnic.tar.gz -C /opt/cni/bin rm hostnic.tar.gz -popd - -wget https://pek3a.qingstor.com/k8s-qingcloud/k8s/tool/niccleaner -O /usr/bin/niccleaner -chmod +x /usr/bin/niccleaner \ No newline at end of file +popd \ No newline at end of file diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index ee91685..8a29055 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -5,7 +5,7 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") echo "update bin" -k8s_bins=("kubelet" "kubectl") +k8s_bins=("kubelet" "kubectl" "kubeadmin") k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/1.7.3/bin/" k8s_bin_path="/usr/bin" @@ -20,8 +20,6 @@ function download_k8s_bin() unlink "${k8s_bin_path}/${bin}" ln -s "${K8S_HOME}/bin/${bin}" "${k8s_bin_path}/${bin}" done - # use custom kubeadm - wget https://pek3a.qingstor.com/k8s-qingcloud/k8s/release/bin/linux/amd64/kubeadm chmod +x * popd } diff --git a/image/update.sh b/image/update.sh index 5795d31..a6244ba 100755 --- a/image/update.sh +++ b/image/update.sh @@ -13,12 +13,9 @@ echo "stop service" systemctl stop kubelet ${K8S_HOME}/image/update-confd.sh - -${K8S_HOME}/image/update-bin.sh - +${K8S_HOME}/image/update-k8s-bin.sh ${K8S_HOME}/image/update-logrotate.sh - -${K8S_HOME}/image/pull-images.sh +${K8S_HOME}/image/pull-docker-images.sh if [ "${HOST_ROLE}" = "master" ]; then ${K8S_HOME}/script/update-manifests.sh From 66670e2600284dfc546f9141e0922263be1ee875 Mon Sep 17 00:00:00 2001 From: jolestar Date: Sat, 5 Aug 2017 11:46:01 +0800 Subject: [PATCH 003/167] exit on error. --- image/build-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/image/build-image.sh b/image/build-image.sh index 8bdc465..729c2fa 100644 --- a/image/build-image.sh +++ b/image/build-image.sh @@ -3,6 +3,10 @@ SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) K8S_HOME=$(dirname "${SCRIPTPATH}") +set -o errexit +set -o nounset +set -o pipefail + ${K8S_HOME}/image/install-pkg.sh ${K8S_HOME}/image/install-qingcloud-agent.sh ${K8S_HOME}/image/update-confd.sh From 425aa834fde6ac5ec74b0014031f65a10cd60bfe Mon Sep 17 00:00:00 2001 From: jolestar Date: Sat, 5 Aug 2017 11:46:20 +0800 Subject: [PATCH 004/167] update hyper version. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index a352ad7..c734f72 100644 --- a/version +++ b/version @@ -1 +1 @@ -HYPERKUBE_VERSION="qingcloud-98822b21b350f5cb120f542e671efd42429c2413" +HYPERKUBE_VERSION="v1.7.3" From 31eeb4048f5fd9c206c6d2787f402701fc2ac69e Mon Sep 17 00:00:00 2001 From: jolestar Date: Sat, 5 Aug 2017 14:45:55 +0800 Subject: [PATCH 005/167] update image build script. --- image/update-k8s-bin.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index 8a29055..bcdfdae 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -3,10 +3,12 @@ SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) K8S_HOME=$(dirname "${SCRIPTPATH}") +source ${K8S_HOME}/version + echo "update bin" k8s_bins=("kubelet" "kubectl" "kubeadmin") -k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/1.7.3/bin/" +k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/${HYPERKUBE_VERSION}/bin/" k8s_bin_path="/usr/bin" function download_k8s_bin() From 83a367313a8e7767fc73b39f5f685a58b2eabfb9 Mon Sep 17 00:00:00 2001 From: jolestar Date: Sat, 5 Aug 2017 14:48:53 +0800 Subject: [PATCH 006/167] update kubeadm usage. --- script/common.sh | 4 ++++ script/init-master.sh | 7 +++++-- script/init-node.sh | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/script/common.sh b/script/common.sh index 55f4408..eaee392 100755 --- a/script/common.sh +++ b/script/common.sh @@ -5,6 +5,10 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") source "/data/kubernetes/env.sh" source "${K8S_HOME}/version" +set -o errexit +set -o nounset +set -o pipefail + NODE_INIT_LOCK="/data/kubernetes/init.lock" function fail { diff --git a/script/init-master.sh b/script/init-master.sh index 2e9db92..4d90903 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -8,6 +8,9 @@ ensure_dir link_dynamic_dir init_token=$(get_or_gen_init_token) -retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf -kubeadm config --token ${init_token} --api-advertise-addresses ${HOST_IP} --skip-preflight-checks --api-external-dns-names ${ENV_API_EXTERNAL_DOMAIN} +#retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf +kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --api-external-dns-names ${ENV_API_EXTERNAL_DOMAIN} +kubeadm alpha phase kubeconfig client-certs --client-name kubelet --server http://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf +kubeadm alpha phase kubeconfig client-certs --client-name admin --server http://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf +kubeadm token create ${init_token} docker_login \ No newline at end of file diff --git a/script/init-node.sh b/script/init-node.sh index 6611370..f63cc43 100755 --- a/script/init-node.sh +++ b/script/init-node.sh @@ -6,5 +6,5 @@ source "${K8S_HOME}/script/common.sh" ensure_dir link_dynamic_dir -retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf +#retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf docker_login \ No newline at end of file From 2de6a21b7d3c7b14dc0c3660a414c8a96503d338 Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 09:17:32 +0800 Subject: [PATCH 007/167] chomd +x to script --- image/build-image.sh | 0 image/install-qingcloud-agent.sh | 0 image/update-sysconfig.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 image/build-image.sh mode change 100644 => 100755 image/install-qingcloud-agent.sh mode change 100644 => 100755 image/update-sysconfig.sh diff --git a/image/build-image.sh b/image/build-image.sh old mode 100644 new mode 100755 diff --git a/image/install-qingcloud-agent.sh b/image/install-qingcloud-agent.sh old mode 100644 new mode 100755 diff --git a/image/update-sysconfig.sh b/image/update-sysconfig.sh old mode 100644 new mode 100755 From 8cfbce9b7ffed0324d093589a3e7fcd081b50ccc Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 09:32:39 +0800 Subject: [PATCH 008/167] update k8s image script. --- image/install-qingcloud-agent.sh | 5 ++++- image/update-confd.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/image/install-qingcloud-agent.sh b/image/install-qingcloud-agent.sh index d00d520..e36d4cb 100755 --- a/image/install-qingcloud-agent.sh +++ b/image/install-qingcloud-agent.sh @@ -10,4 +10,7 @@ chmod +x /etc/init.d/confd cd /tmp rm -rf app-agent-linux-amd64/ -rm app-agent-linux-amd64.tar.gz \ No newline at end of file +rm app-agent-linux-amd64.tar.gz + +systemctl enable confd +systemctl disable confd \ No newline at end of file diff --git a/image/update-confd.sh b/image/update-confd.sh index fb71482..8cead09 100755 --- a/image/update-confd.sh +++ b/image/update-confd.sh @@ -9,4 +9,7 @@ rm -rf /etc/confd/templates/k8s mkdir -p /etc/confd cp -r ${K8S_HOME}/confd/* /etc/confd/ -systemctl restart confd +if systemctl is-active confd > /dev/null +then + systemctl restart confd +fi \ No newline at end of file From aea1f2dfb265ccdee0be6c185ae8cf8d4fa894a2 Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 09:38:05 +0800 Subject: [PATCH 009/167] add default dockerhub account. --- image/pull-docker-images.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/image/pull-docker-images.sh b/image/pull-docker-images.sh index 1dc7589..5c232f2 100755 --- a/image/pull-docker-images.sh +++ b/image/pull-docker-images.sh @@ -7,6 +7,8 @@ source ${K8S_HOME}/version echo "update images" +docker login -u guest -p guest dockerhub.qingcloud.com + docker pull dockerhub.qingcloud.com/google_containers/pause-amd64:3.0 docker pull dockerhub.qingcloud.com/google_containers/etcd-amd64:2.2.5 docker pull dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} From b3f91a224aaeb5aa953ee2e29ecf7228a8deb442 Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 09:42:50 +0800 Subject: [PATCH 010/167] fix typo. --- image/update-k8s-bin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index bcdfdae..cb263b9 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -7,7 +7,7 @@ source ${K8S_HOME}/version echo "update bin" -k8s_bins=("kubelet" "kubectl" "kubeadmin") +k8s_bins=("kubelet" "kubectl" "kubeadm") k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/${HYPERKUBE_VERSION}/bin/" k8s_bin_path="/usr/bin" From 0e56c4322003853c2487de6d78f18d5fc67fbcfd Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 09:52:43 +0800 Subject: [PATCH 011/167] fix k8s bin download url. --- image/update-k8s-bin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index cb263b9..c254e42 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -8,7 +8,7 @@ source ${K8S_HOME}/version echo "update bin" k8s_bins=("kubelet" "kubectl" "kubeadm") -k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/${HYPERKUBE_VERSION}/bin/" +k8s_base_url="https://pek3a.qingstor.com/k8s-qingcloud/k8s/${HYPERKUBE_VERSION}/bin" k8s_bin_path="/usr/bin" function download_k8s_bin() From ebaaec25b42062db08eee45f3066e77b1f20ae6c Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 11:26:47 +0800 Subject: [PATCH 012/167] update manifest to new version. --- image/pull-docker-images.sh | 5 +- k8s/manifests/etcd.json | 85 -------------- k8s/manifests/etcd.yaml | 50 ++++++++ k8s/manifests/kube-addon-manager.yaml | 25 +++- k8s/manifests/kube-apiserver.json | 119 ------------------- k8s/manifests/kube-apiserver.yaml | 70 +++++++++++ k8s/manifests/kube-controller-manager.json | 130 --------------------- k8s/manifests/kube-controller-manager.yaml | 57 +++++++++ k8s/manifests/kube-discovery.json | 53 --------- k8s/manifests/kube-scheduler.json | 47 -------- k8s/manifests/kube-scheduler.yaml | 42 +++++++ script/common.sh | 1 + script/init-master.sh | 18 ++- script/start-master.sh | 2 - 14 files changed, 256 insertions(+), 448 deletions(-) delete mode 100644 k8s/manifests/etcd.json create mode 100644 k8s/manifests/etcd.yaml delete mode 100644 k8s/manifests/kube-apiserver.json create mode 100644 k8s/manifests/kube-apiserver.yaml delete mode 100644 k8s/manifests/kube-controller-manager.json create mode 100644 k8s/manifests/kube-controller-manager.yaml delete mode 100644 k8s/manifests/kube-discovery.json delete mode 100644 k8s/manifests/kube-scheduler.json create mode 100644 k8s/manifests/kube-scheduler.yaml diff --git a/image/pull-docker-images.sh b/image/pull-docker-images.sh index 5c232f2..17d7bea 100755 --- a/image/pull-docker-images.sh +++ b/image/pull-docker-images.sh @@ -10,13 +10,12 @@ echo "update images" docker login -u guest -p guest dockerhub.qingcloud.com docker pull dockerhub.qingcloud.com/google_containers/pause-amd64:3.0 -docker pull dockerhub.qingcloud.com/google_containers/etcd-amd64:2.2.5 +docker pull dockerhub.qingcloud.com/google_containers/etcd-amd64:3.0.17 docker pull dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} docker pull dockerhub.qingcloud.com/google_containers/kubedns-amd64:1.7 docker pull dockerhub.qingcloud.com/google_containers/kube-dnsmasq-amd64:1.3 docker pull dockerhub.qingcloud.com/google_containers/exechealthz-amd64:1.1 -docker pull dockerhub.qingcloud.com/google_containers/kube-discovery-amd64:1.0 -docker pull dockerhub.qingcloud.com/google_containers/kube-addon-manager-amd64:v1.5.4.qingcloud +docker pull dockerhub.qingcloud.com/google_containers/kube-addon-manager:v6.4-beta.2 #docker pull dockerhub.qingcloud.com/coreos/flannel:v0.7.0-amd64 docker pull dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.5.0 docker pull dockerhub.qingcloud.com/fluent/fluent-bit-kubernetes-daemonset:0.11.13 diff --git a/k8s/manifests/etcd.json b/k8s/manifests/etcd.json deleted file mode 100644 index 184d9e4..0000000 --- a/k8s/manifests/etcd.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "etcd", - "namespace": "kube-system", - "creationTimestamp": null, - "labels": { - "component": "etcd", - "tier": "control-plane" - } - }, - "spec": { - "volumes": [ - { - "name": "certs", - "hostPath": { - "path": "/etc/ssl/certs" - } - }, - { - "name": "etcd", - "hostPath": { - "path": "/data/etcd" - } - }, - { - "name": "pki", - "hostPath": { - "path": "/etc/kubernetes" - } - } - ], - "containers": [ - { - "name": "etcd", - "image": "dockerhub.qingcloud.com/google_containers/etcd-amd64:2.2.5", - "imagePullPolicy":"IfNotPresent", - "command": [ - "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/etcd/data" - ], - "resources": { - "requests": { - "cpu": "200m" - } - }, - "volumeMounts": [ - { - "name": "certs", - "mountPath": "/etc/ssl/certs" - }, - { - "name": "etcd", - "mountPath": "/var/etcd" - }, - { - "name": "pki", - "readOnly": true, - "mountPath": "/etc/kubernetes/" - } - ], - "livenessProbe": { - "httpGet": { - "path": "/health", - "port": 2379, - "host": "127.0.0.1" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15, - "failureThreshold": 8 - }, - "securityContext": { - "seLinuxOptions": { - "type": "unconfined_t" - } - } - } - ], - "hostNetwork": true - }, - "status": {} -} diff --git a/k8s/manifests/etcd.yaml b/k8s/manifests/etcd.yaml new file mode 100644 index 0000000..7bb3aa4 --- /dev/null +++ b/k8s/manifests/etcd.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + component: etcd + tier: control-plane + name: etcd + namespace: kube-system +spec: + containers: + - command: + - etcd + - --listen-client-urls=http://127.0.0.1:2379 + - --advertise-client-urls=http://127.0.0.1:2379 + - --data-dir=/var/lib/etcd + image: dockerhub.qingcloud.com/google_containers/etcd-amd64:3.0.17 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /health + port: 2379 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: etcd + resources: {} + volumeMounts: + - mountPath: /etc/ssl/certs + name: certs + - mountPath: /var/lib/etcd + name: etcd + - mountPath: /etc/kubernetes + name: k8s + readOnly: true + hostNetwork: true + volumes: + - hostPath: + path: /etc/ssl/certs + name: certs + - hostPath: + path: /var/lib/etcd + name: etcd + - hostPath: + path: /etc/kubernetes + name: k8s \ No newline at end of file diff --git a/k8s/manifests/kube-addon-manager.yaml b/k8s/manifests/kube-addon-manager.yaml index 7968d78..b978238 100644 --- a/k8s/manifests/kube-addon-manager.yaml +++ b/k8s/manifests/kube-addon-manager.yaml @@ -3,24 +3,39 @@ kind: Pod metadata: name: kube-addon-manager namespace: kube-system - version: v1 + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + labels: + component: kube-addon-manager spec: hostNetwork: true containers: - name: kube-addon-manager - image: dockerhub.qingcloud.com/google_containers/kube-addon-manager-amd64:v1.5.4.qingcloud + # When updating version also bump it in: + # - cluster/images/hyperkube/static-pods/addon-manager-singlenode.json + # - cluster/images/hyperkube/static-pods/addon-manager-multinode.json + # - test/kubemark/resources/manifests/kube-addon-manager.yaml + image: dockerhub.qingcloud.com/google_containers/kube-addon-manager:v6.4-beta.2 imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + - /opt/kube-addons.sh 1>>/var/log/kube-addon-manager.log 2>&1 resources: requests: cpu: 5m - # TODO(yifan): Figure out what's the memory usage should be here. - # See https://github.com/kubernetes/kubernetes/issues/23641. - memory: 100Mi + memory: 50Mi volumeMounts: - mountPath: /etc/kubernetes/ name: addons readOnly: true + - mountPath: /var/log + name: varlog + readOnly: false volumes: - hostPath: path: /etc/kubernetes/ name: addons + - hostPath: + path: /var/log + name: varlog \ No newline at end of file diff --git a/k8s/manifests/kube-apiserver.json b/k8s/manifests/kube-apiserver.json deleted file mode 100644 index a52d988..0000000 --- a/k8s/manifests/kube-apiserver.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "kube-apiserver", - "namespace": "kube-system", - "creationTimestamp": null, - "labels": { - "component": "kube-apiserver", - "tier": "control-plane" - } - }, - "spec": { - "volumes": [ - { - "name": "certs", - "hostPath": { - "path": "/etc/ssl/certs" - } - }, - { - "name": "pki", - "hostPath": { - "path": "/etc/kubernetes" - } - } - ], - "containers": [ - { - "name": "kube-apiserver", - "image": "dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION}", - "imagePullPolicy":"IfNotPresent", - "command": [ - "/hyperkube", - "apiserver", - "--v=${KUBE_LOG_LEVEL}", - "--insecure-bind-address=127.0.0.1", - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", - "--service-cluster-ip-range=10.96.0.0/16", - "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--client-ca-file=/etc/kubernetes/pki/ca.pem", - "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", - "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--token-auth-file=/etc/kubernetes/pki/tokens.csv", - "--secure-port=6443", - "--runtime-config", - "batch/v2alpha1=true", - "--allow-privileged", - "--etcd-servers=http://127.0.0.1:2379", - "--cloud-provider=qingcloud", - "--cloud-config=/etc/kubernetes/qingcloud.conf" - ], - "resources": { - "requests": { - "cpu": "250m" - } - }, - "volumeMounts": [ - { - "name": "certs", - "mountPath": "/etc/ssl/certs" - }, - { - "name": "pki", - "readOnly": true, - "mountPath": "/etc/kubernetes/" - } - ], - "livenessProbe": { - "httpGet": { - "path": "/healthz", - "port": 8080, - "host": "127.0.0.1" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15, - "failureThreshold": 8 - } - }, - { - "name": "hosts-sync", - "image": "dockerhub.qingcloud.com/qingcloud/file-sync:0.2", - "imagePullPolicy":"IfNotPresent", - "env":[ - { - "name":"FROM_FILE", - "value":"/etc/kubernetes/hosts" - }, - { - "name":"TO_FILE", - "value":"/etc/hosts" - }, - { - "name":"CHECK_INTERVAL", - "value":"2" - } - ], - "resources": { - "limits": { - "cpu": "10m", - "memory": "50Mi" - }, - "requests": { - "cpu": "10m", - "memory": "50Mi" - } - }, - "volumeMounts": [ - { - "name": "pki", - "readOnly": true, - "mountPath": "/etc/kubernetes/" - } - ] - } - ], - "hostNetwork": true - } -} diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml new file mode 100644 index 0000000..50fa041 --- /dev/null +++ b/k8s/manifests/kube-apiserver.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + component: kube-apiserver + tier: control-plane + name: kube-apiserver + namespace: kube-system +spec: + containers: + - command: + - kube-apiserver + - --v=${KUBE_LOG_LEVEL} + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-allowed-names=front-proxy-client + - --service-cluster-ip-range=10.96.0.0/12 + - --secure-port=6443 + - --insecure-port=0 + - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt + - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key + - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt + - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt + - --allow-privileged=true + - --experimental-bootstrap-token-auth=true + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --requestheader-username-headers=X-Remote-User + - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt + - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota + - --requestheader-group-headers=X-Remote-Group + - --service-account-key-file=/etc/kubernetes/pki/sa.pub + - --client-ca-file=/etc/kubernetes/pki/ca.crt + - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key + - --authorization-mode=Node,RBAC + - --advertise-address=${HOST_IP--feature-gates} + - --etcd-servers=http://127.0.0.1:2379 + - --runtime-config batch/v2alpha1=true + - --feature-gates AllAlpha=true + image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: kube-apiserver + resources: + requests: + cpu: 250m + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s + readOnly: true + - mountPath: /etc/ssl/certs + name: certs + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes + name: k8s + - hostPath: + path: /etc/ssl/certs + name: certs \ No newline at end of file diff --git a/k8s/manifests/kube-controller-manager.json b/k8s/manifests/kube-controller-manager.json deleted file mode 100644 index 6395b28..0000000 --- a/k8s/manifests/kube-controller-manager.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "kube-controller-manager", - "namespace": "kube-system", - "creationTimestamp": null, - "labels": { - "component": "kube-controller-manager", - "tier": "control-plane" - } - }, - "spec": { - "volumes": [ - { - "name": "certs", - "hostPath": { - "path": "/etc/ssl/certs" - } - }, - { - "name": "pki", - "hostPath": { - "path": "/etc/kubernetes" - } - }, - { - "name": "qingcloud", - "hostPath": { - "path": "/etc/qingcloud" - } - } - ], - "containers": [ - { - "name": "kube-controller-manager", - "image": "dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION}", - "imagePullPolicy":"IfNotPresent", - "command": [ - "/hyperkube", - "controller-manager", - "--v=${KUBE_LOG_LEVEL}", - "--address=127.0.0.1", - "--leader-elect", - "--master=127.0.0.1:8080", - "--cluster-name=kubernetes", - "--root-ca-file=/etc/kubernetes/pki/ca.pem", - "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", - "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", - "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", - "--allocate-node-cidrs=false", - "--service-cluster-ip-range=10.96.0.0/16", - "--feature-gates", - "AllAlpha=true", - "--cloud-provider=qingcloud", - "--cloud-config=/etc/kubernetes/qingcloud.conf" - ], - "resources": { - "requests": { - "cpu": "200m" - } - }, - "volumeMounts": [ - { - "name": "certs", - "mountPath": "/etc/ssl/certs" - }, - { - "name": "pki", - "readOnly": true, - "mountPath": "/etc/kubernetes/" - }, - { - "name": "qingcloud", - "readOnly": true, - "mountPath": "/etc/qingcloud/" - } - ], - "livenessProbe": { - "httpGet": { - "path": "/healthz", - "port": 10252, - "host": "127.0.0.1" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15, - "failureThreshold": 8 - } - }, - { - "name": "hosts-sync", - "image": "dockerhub.qingcloud.com/qingcloud/file-sync:0.2", - "imagePullPolicy":"IfNotPresent", - "env":[ - { - "name":"FROM_FILE", - "value":"/etc/kubernetes/hosts" - }, - { - "name":"TO_FILE", - "value":"/etc/hosts" - }, - { - "name":"CHECK_INTERVAL", - "value":"2" - } - ], - "resources": { - "limits": { - "cpu": "10m", - "memory": "50Mi" - }, - "requests": { - "cpu": "10m", - "memory": "50Mi" - } - }, - "volumeMounts": [ - { - "name": "pki", - "readOnly": true, - "mountPath": "/etc/kubernetes/" - } - ] - } - ], - "hostNetwork": true - } -} diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml new file mode 100644 index 0000000..140e9dc --- /dev/null +++ b/k8s/manifests/kube-controller-manager.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + component: kube-controller-manager + tier: control-plane + name: kube-controller-manager + namespace: kube-system +spec: + containers: + - command: + - kube-controller-manager + - --v=${KUBE_LOG_LEVEL} + - --kubeconfig=/etc/kubernetes/controller-manager.conf + - --service-account-private-key-file=/etc/kubernetes/pki/sa.key + - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt + - --address=127.0.0.1 + - --leader-elect=true + - --use-service-account-credentials=true + - --controllers=*,bootstrapsigner,tokencleaner + - --root-ca-file=/etc/kubernetes/pki/ca.crt + - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key + - --feature-gates AllAlpha=true + - --allocate-node-cidrs=false + - --service-cluster-ip-range=10.96.0.0/16 + image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: kube-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s + readOnly: true + - mountPath: /etc/ssl/certs + name: certs + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes + name: k8s + - hostPath: + path: /etc/ssl/certs + name: certs \ No newline at end of file diff --git a/k8s/manifests/kube-discovery.json b/k8s/manifests/kube-discovery.json deleted file mode 100644 index 2915853..0000000 --- a/k8s/manifests/kube-discovery.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "kube-discovery", - "namespace": "kube-system", - "labels": { - "component": "kube-discovery", - "tier": "control-plane" - } - }, - "spec": { - "volumes": [ - { - "name": "clusterinfo", - "hostPath": { - "path": "/etc/kubernetes/pki" - } - } - ], - "containers": [ - { - "name": "kube-discovery", - "image": "dockerhub.qingcloud.com/google_containers/kube-discovery-amd64:1.0", - "imagePullPolicy":"IfNotPresent", - "command": [ - "/usr/local/bin/kube-discovery" - ], - "ports": [ - { - "name": "http", - "hostPort": 9898, - "containerPort": 9898 - } - ], - "resources": {}, - "volumeMounts": [ - { - "name": "clusterinfo", - "readOnly": true, - "mountPath": "/tmp/secret" - } - ], - "securityContext": { - "seLinuxOptions": { - "type": "unconfined_t" - } - } - } - ], - "hostNetwork": true - } -} diff --git a/k8s/manifests/kube-scheduler.json b/k8s/manifests/kube-scheduler.json deleted file mode 100644 index c236173..0000000 --- a/k8s/manifests/kube-scheduler.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "kube-scheduler", - "namespace": "kube-system", - "creationTimestamp": null, - "labels": { - "component": "kube-scheduler", - "tier": "control-plane" - } - }, - "spec": { - "containers": [ - { - "name": "kube-scheduler", - "image": "dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION}", - "imagePullPolicy":"IfNotPresent", - "command": [ - "/hyperkube", - "scheduler", - "--v=${KUBE_LOG_LEVEL}", - "--address=127.0.0.1", - "--leader-elect", - "--master=127.0.0.1:8080" - ], - "resources": { - "requests": { - "cpu": "100m" - } - }, - "livenessProbe": { - "httpGet": { - "path": "/healthz", - "port": 10251, - "host": "127.0.0.1" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15, - "failureThreshold": 8 - } - } - ], - "hostNetwork": true - }, - "status": {} -} diff --git a/k8s/manifests/kube-scheduler.yaml b/k8s/manifests/kube-scheduler.yaml new file mode 100644 index 0000000..50efe7c --- /dev/null +++ b/k8s/manifests/kube-scheduler.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + component: kube-scheduler + tier: control-plane + name: kube-scheduler + namespace: kube-system +spec: + containers: + - command: + - kube-scheduler + - --address=127.0.0.1 + - --leader-elect=true + - --kubeconfig=/etc/kubernetes/scheduler.conf + image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10251 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: kube-scheduler + resources: + requests: + cpu: 100m + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s + readOnly: true + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes + name: k8s \ No newline at end of file diff --git a/script/common.sh b/script/common.sh index eaee392..1077982 100755 --- a/script/common.sh +++ b/script/common.sh @@ -73,6 +73,7 @@ function replace_vars(){ tmpfile="/tmp/${prefix}-${name}" sed 's/${HYPERKUBE_VERSION}/'"${HYPERKUBE_VERSION}"'/g' ${from} > ${tmpfile} sed -i 's/${KUBE_LOG_LEVEL}/'"${ENV_KUBE_LOG_LEVEL}"'/g' ${tmpfile} + sed -i 's/${HOST_IP}/'"${HOST_IP}"'/g' ${tmpfile} if [ "${to}" == "/data/kubernetes/addons/monitor/es-controller.yaml" ] then diff --git a/script/init-master.sh b/script/init-master.sh index 4d90903..1c2f2df 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -10,7 +10,17 @@ link_dynamic_dir init_token=$(get_or_gen_init_token) #retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --api-external-dns-names ${ENV_API_EXTERNAL_DOMAIN} -kubeadm alpha phase kubeconfig client-certs --client-name kubelet --server http://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf -kubeadm alpha phase kubeconfig client-certs --client-name admin --server http://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf -kubeadm token create ${init_token} -docker_login \ No newline at end of file +kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:node:${HOST_INSTANCE_ID} --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf +kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf +kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-scheduler --server https://${MASTER_IP}:6443 > /etc/kubernetes/scheduler.conf +kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --server https://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf +docker_login + +process_manifests + +systemctl start docker +systemctl start kubelet +wait_kubelet +wait_apiserver +train_master +retry kubeadm token create ${init_token} --token-ttl 0 diff --git a/script/start-master.sh b/script/start-master.sh index 591edf8..9da7366 100755 --- a/script/start-master.sh +++ b/script/start-master.sh @@ -7,8 +7,6 @@ source "${K8S_HOME}/script/common.sh" ensure_dir -process_manifests - systemctl start docker systemctl start kubelet wait_kubelet From 21148b988edfd1c32d3636359c3536b11d4b6d4c Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 11:35:16 +0800 Subject: [PATCH 013/167] update manifest process. --- script/common.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/script/common.sh b/script/common.sh index 1077982..90e80a3 100755 --- a/script/common.sh +++ b/script/common.sh @@ -93,8 +93,8 @@ function replace_vars(){ function update_k8s_manifests(){ echo "echo update k8s manifests" - mkdir /data/kubernetes/manifests/ || rm -rf /data/kubernetes/manifests/* - mkdir /data/kubernetes/addons/ || rm -rf /data/kubernetes/addons/* + #mkdir /data/kubernetes/manifests/ || rm -rf /data/kubernetes/manifests/* + #mkdir /data/kubernetes/addons/ || rm -rf /data/kubernetes/addons/* process_manifests } @@ -106,14 +106,14 @@ function process_manifests(){ replace_vars ${f} /data/kubernetes/manifests/${name} done - for addon in ${K8S_HOME}/k8s/addons/*; do - addon_name=$(basename $addon) - mkdir -p /data/kubernetes/addons/${addon_name} - for f in ${addon}/*; do - name=$(basename ${f}) - replace_vars ${f} /data/kubernetes/addons/${addon_name}/${name} - done - done +# for addon in ${K8S_HOME}/k8s/addons/*; do +# addon_name=$(basename $addon) +# mkdir -p /data/kubernetes/addons/${addon_name} +# for f in ${addon}/*; do +# name=$(basename ${f}) +# replace_vars ${f} /data/kubernetes/addons/${addon_name}/${name} +# done +# done } function scale_es(){ From 79d5ab8a0ff9ddc5b12845e37f9cace660d8acb5 Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 11:51:17 +0800 Subject: [PATCH 014/167] fix manifest. --- k8s/manifests/kube-apiserver.yaml | 6 ++++-- k8s/manifests/kube-controller-manager.yaml | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index 50fa041..9136784 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -37,8 +37,10 @@ spec: - --authorization-mode=Node,RBAC - --advertise-address=${HOST_IP--feature-gates} - --etcd-servers=http://127.0.0.1:2379 - - --runtime-config batch/v2alpha1=true - - --feature-gates AllAlpha=true + - --runtime-config + - batch/v2alpha1=true + - --feature-gates + - AllAlpha=true image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} imagePullPolicy: IfNotPresent livenessProbe: diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 140e9dc..fbc703c 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -23,7 +23,8 @@ spec: - --controllers=*,bootstrapsigner,tokencleaner - --root-ca-file=/etc/kubernetes/pki/ca.crt - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - - --feature-gates AllAlpha=true + - --feature-gates + - AllAlpha=true - --allocate-node-cidrs=false - --service-cluster-ip-range=10.96.0.0/16 image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} From dfbc66bf3b8c5af59948cc9111278c772c90e44b Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 11:54:46 +0800 Subject: [PATCH 015/167] fix manifest. --- k8s/manifests/kube-apiserver.yaml | 2 +- k8s/manifests/kube-controller-manager.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index 9136784..dbd4ff5 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -35,7 +35,7 @@ spec: - --client-ca-file=/etc/kubernetes/pki/ca.crt - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - --authorization-mode=Node,RBAC - - --advertise-address=${HOST_IP--feature-gates} + - --advertise-address=${HOST_IP} - --etcd-servers=http://127.0.0.1:2379 - --runtime-config - batch/v2alpha1=true diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index fbc703c..a897d20 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -23,10 +23,10 @@ spec: - --controllers=*,bootstrapsigner,tokencleaner - --root-ca-file=/etc/kubernetes/pki/ca.crt - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - - --feature-gates - - AllAlpha=true - --allocate-node-cidrs=false - --service-cluster-ip-range=10.96.0.0/16 + - --feature-gates + - AllAlpha=true image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} imagePullPolicy: IfNotPresent livenessProbe: From a883db94d34b00f5002d8c57e79d28456293d51b Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 18:37:14 +0800 Subject: [PATCH 016/167] update addons manifest --- image/pull-docker-images.sh | 10 +- .../dashboard/dashboard-deployment.yaml | 16 +- k8s/addons/kube-dns/kube-dns-deployment.json | 151 ----------------- k8s/addons/kube-dns/kube-dns-deployment.yaml | 158 ++++++++++++++++++ k8s/addons/kube-dns/kube-dns-service.json | 38 ----- k8s/addons/kube-dns/kube-dns-service.yaml | 25 +++ k8s/addons/kube-proxy/kube-proxy.json | 82 --------- k8s/addons/kube-proxy/kube-proxy.yaml | 57 +++++++ k8s/addons/monitor/es-controller.yaml | 11 +- .../fluent-bit-daemonset-elasticsearch.yaml | 12 +- k8s/addons/monitor/heapster-controller.yaml | 10 +- k8s/addons/monitor/kibana-controller.yaml | 8 +- k8s/addons/qingcloud/clean-log-cronjob.yaml | 5 +- .../qingcloud-storage-class-capacity.yaml | 11 -- .../qingcloud/qingcloud-storage-class.yaml | 9 - k8s/manifests/kube-apiserver.yaml | 2 +- 16 files changed, 290 insertions(+), 315 deletions(-) delete mode 100644 k8s/addons/kube-dns/kube-dns-deployment.json create mode 100644 k8s/addons/kube-dns/kube-dns-deployment.yaml delete mode 100644 k8s/addons/kube-dns/kube-dns-service.json create mode 100644 k8s/addons/kube-dns/kube-dns-service.yaml delete mode 100644 k8s/addons/kube-proxy/kube-proxy.json create mode 100644 k8s/addons/kube-proxy/kube-proxy.yaml delete mode 100644 k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml delete mode 100644 k8s/addons/qingcloud/qingcloud-storage-class.yaml diff --git a/image/pull-docker-images.sh b/image/pull-docker-images.sh index 17d7bea..81ce121 100755 --- a/image/pull-docker-images.sh +++ b/image/pull-docker-images.sh @@ -12,16 +12,16 @@ docker login -u guest -p guest dockerhub.qingcloud.com docker pull dockerhub.qingcloud.com/google_containers/pause-amd64:3.0 docker pull dockerhub.qingcloud.com/google_containers/etcd-amd64:3.0.17 docker pull dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} -docker pull dockerhub.qingcloud.com/google_containers/kubedns-amd64:1.7 -docker pull dockerhub.qingcloud.com/google_containers/kube-dnsmasq-amd64:1.3 -docker pull dockerhub.qingcloud.com/google_containers/exechealthz-amd64:1.1 +docker pull dockerhub.qingcloud.com/google_containers/k8s-dns-kube-dns-amd64:1.14.4 +docker pull dockerhub.qingcloud.com/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4 +docker pull dockerhub.qingcloud.com/google_containers/k8s-dns-sidecar-amd64:1.14.4 docker pull dockerhub.qingcloud.com/google_containers/kube-addon-manager:v6.4-beta.2 #docker pull dockerhub.qingcloud.com/coreos/flannel:v0.7.0-amd64 -docker pull dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.5.0 +docker pull dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.6.1 docker pull dockerhub.qingcloud.com/fluent/fluent-bit-kubernetes-daemonset:0.11.13 docker pull dockerhub.qingcloud.com/google_containers/heapster-amd64:v1.4.0 docker pull dockerhub.qingcloud.com/google_containers/addon-resizer:1.7 docker pull dockerhub.qingcloud.com/google_containers/kibana:v5.4.0 docker pull dockerhub.qingcloud.com/google_containers/elasticsearch:v5.4.0-1 docker pull dockerhub.qingcloud.com/qingcloud/elasticsearch-curator:5.1.1 -docker pull dockerhub.qingcloud.com/qingcloud/file-sync:0.2 \ No newline at end of file +docker pull dockerhub.qingcloud.com/qingcloud/file-sync:0.7 \ No newline at end of file diff --git a/k8s/addons/dashboard/dashboard-deployment.yaml b/k8s/addons/dashboard/dashboard-deployment.yaml index 4a38dd9..05c57a0 100644 --- a/k8s/addons/dashboard/dashboard-deployment.yaml +++ b/k8s/addons/dashboard/dashboard-deployment.yaml @@ -12,23 +12,23 @@ spec: k8s-app: kubernetes-dashboard template: metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" labels: k8s-app: kubernetes-dashboard - annotations: - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated", "value":"master", "effect":"NoSchedule"}]' spec: containers: - name: kubernetes-dashboard - image: dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.5.0 + image: dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.6.1 imagePullPolicy: IfNotPresent resources: # keep request = limit to keep this container in guaranteed class limits: cpu: 100m - memory: 50Mi + memory: 300Mi requests: cpu: 100m - memory: 50Mi + memory: 100Mi ports: - containerPort: 9090 livenessProbe: @@ -37,5 +37,11 @@ spec: port: 9090 initialDelaySeconds: 30 timeoutSeconds: 30 + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + value: "master" + effect: NoSchedule nodeSelector: role: master \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-deployment.json b/k8s/addons/kube-dns/kube-dns-deployment.json deleted file mode 100644 index 4f9a43e..0000000 --- a/k8s/addons/kube-dns/kube-dns-deployment.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "kind": "Deployment", - "apiVersion": "extensions/v1beta1", - "metadata": { - "name": "kube-dns", - "namespace": "kube-system", - "labels":{ - "addonmanager.kubernetes.io/mode": "EnsureExists" - } - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "component": "kube-dns", - "k8s-app": "kube-dns", - "name": "kube-dns", - "tier": "node" - } - }, - "template": { - "metadata": { - "labels": { - "component": "kube-dns", - "k8s-app": "kube-dns", - "name": "kube-dns", - "tier": "node" - }, - "annotations": { - "scheduler.alpha.kubernetes.io/affinity": "{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"beta.kubernetes.io/arch\",\"operator\":\"In\",\"values\":[\"amd64\"]}]}]}}}", - "scheduler.alpha.kubernetes.io/tolerations": "[{\"key\":\"dedicated\",\"value\":\"master\",\"effect\":\"NoSchedule\"}]" - } - }, - "spec": { - "containers": [ - { - "name": "kube-dns", - "image": "dockerhub.qingcloud.com/google_containers/kubedns-amd64:1.7", - "imagePullPolicy":"IfNotPresent", - "args": [ - "--domain=cluster.local", - "--dns-port=10053" - ], - "ports": [ - { - "name": "dns-local", - "containerPort": 10053, - "protocol": "UDP" - }, - { - "name": "dns-tcp-local", - "containerPort": 10053, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "100m", - "memory": "170Mi" - }, - "requests": { - "cpu": "100m", - "memory": "170Mi" - } - }, - "livenessProbe": { - "httpGet": { - "path": "/healthz", - "port": 8080, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "timeoutSeconds": 5, - "successThreshold": 1, - "failureThreshold": 1 - }, - "readinessProbe": { - "httpGet": { - "path": "/readiness", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "timeoutSeconds": 5 - } - }, - { - "name": "dnsmasq", - "image": "dockerhub.qingcloud.com/google_containers/kube-dnsmasq-amd64:1.3", - "imagePullPolicy":"IfNotPresent", - "args": [ - "--cache-size=1000", - "--no-resolv", - "--server=127.0.0.1#10053" - ], - "ports": [ - { - "name": "dns", - "containerPort": 53, - "protocol": "UDP" - }, - { - "name": "dns-tcp", - "containerPort": 53, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "100m", - "memory": "170Mi" - }, - "requests": { - "cpu": "100m", - "memory": "170Mi" - } - } - }, - { - "name": "healthz", - "image": "dockerhub.qingcloud.com/google_containers/exechealthz-amd64:1.1", - "imagePullPolicy":"IfNotPresent", - "args": [ - "-cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1:53 \u003e/dev/null \u0026\u0026 nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 \u003e/dev/null", - "-port=8080", - "-quiet" - ], - "ports": [ - { - "containerPort": 8080, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "10m", - "memory": "50Mi" - }, - "requests": { - "cpu": "10m", - "memory": "50Mi" - } - } - } - ], - "dnsPolicy": "Default", - "nodeSelector": {"role": "master"} - } - } - } -} \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-deployment.yaml b/k8s/addons/kube-dns/kube-dns-deployment.yaml new file mode 100644 index 0000000..79c91dd --- /dev/null +++ b/k8s/addons/kube-dns/kube-dns-deployment.yaml @@ -0,0 +1,158 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns +spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + volumes: + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + containers: + - name: kubedns + image: dockerhub.qingcloud.com/google_containers/k8s-dns-kube-dns-amd64:1.14.4 + imagePullPolicy: IfNotPresent + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain={{ .DNSDomain }}. + - --dns-port=10053 + - --config-dir=/kube-dns-config + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + volumeMounts: + - name: kube-dns-config + mountPath: /kube-dns-config + - name: dnsmasq + image: dockerhub.qingcloud.com/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k + - --cache-size=1000 + - --log-facility=- + - --server=/{{ .DNSDomain }}/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + # see: https://github.com/kubernetes/kubernetes/issues/29055 for details + resources: + requests: + cpu: 150m + memory: 20Mi + volumeMounts: + - name: kube-dns-config + mountPath: /etc/k8s/dns/dnsmasq-nanny + - name: sidecar + image: dockerhub.qingcloud.com/google_containers/k8s-dns-sidecar-amd64:1.14.4 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ .DNSDomain }},5,A + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ .DNSDomain }},5,A + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + memory: 20Mi + cpu: 10m + dnsPolicy: Default # Don't use cluster DNS. + serviceAccountName: kube-dns + tolerations: + - key: "CriticalAddonsOnly" + operator: Exists + - key: "dedicated" + value: "master" + effect: NoSchedule + nodeSelector: + role: master \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-service.json b/k8s/addons/kube-dns/kube-dns-service.json deleted file mode 100644 index f20ebdd..0000000 --- a/k8s/addons/kube-dns/kube-dns-service.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "kube-dns", - "namespace": "kube-system", - "labels": { - "component": "kube-dns", - "k8s-app": "kube-dns", - "name": "kube-dns", - "tier": "node", - "kubernetes.io/cluster-service": "true", - "addonmanager.kubernetes.io/mode": "EnsureExists" - } - }, - "spec": { - "ports": [ - { - "name": "dns", - "protocol": "UDP", - "port": 53, - "targetPort": 0, - "nodePort": 0 - }, - { - "name": "dns-tcp", - "protocol": "TCP", - "port": 53, - "targetPort": 0, - "nodePort": 0 - } - ], - "selector": { - "name": "kube-dns" - }, - "clusterIP": "10.96.0.10" - } -} \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-service.yaml b/k8s/addons/kube-dns/kube-dns-service.yaml new file mode 100644 index 0000000..858d439 --- /dev/null +++ b/k8s/addons/kube-dns/kube-dns-service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: KubeDNS + addonmanager.kubernetes.io/mode: EnsureExists + name: kube-dns + namespace: kube-system +spec: + clusterIP: 10.96.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns + sessionAffinity: None + type: ClusterIP \ No newline at end of file diff --git a/k8s/addons/kube-proxy/kube-proxy.json b/k8s/addons/kube-proxy/kube-proxy.json deleted file mode 100644 index c609fe0..0000000 --- a/k8s/addons/kube-proxy/kube-proxy.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "kind": "DaemonSet", - "apiVersion": "extensions/v1beta1", - "metadata": { - "name": "kube-proxy", - "namespace": "kube-system", - "labels":{ - "addonmanager.kubernetes.io/mode": "EnsureExists" - } - }, - "spec": { - "selector": { - "matchLabels": { - "component": "kube-proxy", - "k8s-app": "kube-proxy", - "name": "kube-proxy", - "tier": "node" - } - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "component": "kube-proxy", - "k8s-app": "kube-proxy", - "name": "kube-proxy", - "tier": "node" - }, - "annotations": { - "scheduler.alpha.kubernetes.io/tolerations": "[{\"key\":\"dedicated\", \"operator\": \"Exists\"}]" - } - }, - "spec": { - "volumes": [ - { - "name": "kubeconfig", - "hostPath": { - "path": "/etc/kubernetes/kubelet.conf" - } - }, - { - "name": "dbus", - "hostPath": { - "path": "/var/run/dbus" - } - } - ], - "containers": [ - { - "name": "kube-proxy", - "image": "dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION}", - "imagePullPolicy":"IfNotPresent", - "command": [ - "/hyperkube", - "proxy", - "--v=5", - "--masquerade-all", - "--feature-gates", - "AllAlpha=true", - "--kubeconfig=/run/kubeconfig" - ], - "resources": {}, - "volumeMounts": [ - { - "name": "dbus", - "mountPath": "/var/run/dbus" - }, - { - "name": "kubeconfig", - "mountPath": "/run/kubeconfig" - } - ], - "securityContext": { - "privileged": true - } - } - ], - "hostNetwork": true - } - } - } -} \ No newline at end of file diff --git a/k8s/addons/kube-proxy/kube-proxy.yaml b/k8s/addons/kube-proxy/kube-proxy.yaml new file mode 100644 index 0000000..600764c --- /dev/null +++ b/k8s/addons/kube-proxy/kube-proxy.yaml @@ -0,0 +1,57 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: kube-proxy + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + containers: + - name: kube-proxy + image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:v1.7.3 + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/kube-proxy + - --v=${KUBE_LOG_LEVEL} + - --masquerade-all + - --feature-gates + - AllAlpha=true + - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + # TODO: Make this a file hostpath mount + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + hostNetwork: true + serviceAccountName: kube-proxy + volumes: + - name: kube-proxy + configMap: + name: kube-proxy + - name: xtables-lock + hostPath: + path: /run/xtables.lock + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + operator: "Exists" + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" \ No newline at end of file diff --git a/k8s/addons/monitor/es-controller.yaml b/k8s/addons/monitor/es-controller.yaml index 183d833..3582dcf 100644 --- a/k8s/addons/monitor/es-controller.yaml +++ b/k8s/addons/monitor/es-controller.yaml @@ -5,7 +5,6 @@ metadata: namespace: kube-system labels: k8s-app: elasticsearch-logging - version: v1 addonmanager.kubernetes.io/mode: EnsureExists spec: serviceName: "elasticsearch-logging" @@ -13,14 +12,12 @@ spec: selector: matchLabels: k8s-app: elasticsearch-logging - version: v1 template: metadata: labels: k8s-app: elasticsearch-logging - version: v1 annotations: - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated", "value":"log", "effect":"NoSchedule"}]' + scheduler.alpha.kubernetes.io/critical-pod: '' spec: containers: - image: dockerhub.qingcloud.com/google_containers/elasticsearch:v5.4.0-1 @@ -53,5 +50,11 @@ spec: - name: es-persistent-storage hostPath: path: /data/es + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + value: "log" + effect: NoSchedule nodeSelector: role: log diff --git a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml index 7bdc950..02fb9d1 100644 --- a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml +++ b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml @@ -5,16 +5,14 @@ metadata: namespace: kube-system labels: k8s-app: fluent-bit-logging - version: v1 addonmanager.kubernetes.io/mode: EnsureExists spec: template: metadata: labels: k8s-app: fluent-bit-logging - version: v1 annotations: - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator": "Exists"}]' + scheduler.alpha.kubernetes.io/critical-pod: '' spec: containers: - name: fluent-bit @@ -49,3 +47,11 @@ spec: - name: varlibdockercontainers hostPath: path: /data/var/lib/docker/containers + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + operator: "Exists" + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" diff --git a/k8s/addons/monitor/heapster-controller.yaml b/k8s/addons/monitor/heapster-controller.yaml index 2d9342c..397bdd4 100644 --- a/k8s/addons/monitor/heapster-controller.yaml +++ b/k8s/addons/monitor/heapster-controller.yaml @@ -19,7 +19,7 @@ spec: k8s-app: heapster version: v1.4.0 annotations: - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated", "value":"log", "effect":"NoSchedule"}]' + scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccount: "heapster" containers: @@ -125,5 +125,11 @@ spec: - name: usr-ca-certs hostPath: path: "/usr/share/ca-certificates" + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + value: "log" + effect: NoSchedule nodeSelector: - role: log + role: log diff --git a/k8s/addons/monitor/kibana-controller.yaml b/k8s/addons/monitor/kibana-controller.yaml index abb5f3d..fc97e4f 100644 --- a/k8s/addons/monitor/kibana-controller.yaml +++ b/k8s/addons/monitor/kibana-controller.yaml @@ -16,7 +16,7 @@ spec: labels: k8s-app: kibana-logging annotations: - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated", "value":"log", "effect":"NoSchedule"}]' + scheduler.alpha.kubernetes.io/critical-pod: "" spec: containers: - name: kibana-logging @@ -36,5 +36,11 @@ spec: - containerPort: 5601 name: ui protocol: TCP + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + value: "log" + effect: NoSchedule nodeSelector: role: log diff --git a/k8s/addons/qingcloud/clean-log-cronjob.yaml b/k8s/addons/qingcloud/clean-log-cronjob.yaml index 87dea00..70085eb 100644 --- a/k8s/addons/qingcloud/clean-log-cronjob.yaml +++ b/k8s/addons/qingcloud/clean-log-cronjob.yaml @@ -7,9 +7,8 @@ metadata: addonmanager.kubernetes.io/mode: EnsureExists spec: schedule: "0 1 * * *" -# The following two option are supported from version 1.6 -# successfulJobsHistoryLimit: 1 -# failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 jobTemplate: metadata: labels: diff --git a/k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml b/k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml deleted file mode 100644 index 0025256..0000000 --- a/k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1beta1 -metadata: - name: qingcloud-storageclass-capacity - labels: - addonmanager.kubernetes.io/mode: EnsureExists - annotations: - storageclass.beta.kubernetes.io/is-default-class: "false" -provisioner: kubernetes.io/qingcloud-volume -parameters: - type: "2" diff --git a/k8s/addons/qingcloud/qingcloud-storage-class.yaml b/k8s/addons/qingcloud/qingcloud-storage-class.yaml deleted file mode 100644 index c693848..0000000 --- a/k8s/addons/qingcloud/qingcloud-storage-class.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1beta1 -metadata: - name: qingcloud-storageclass - labels: - addonmanager.kubernetes.io/mode: EnsureExists - annotations: - storageclass.beta.kubernetes.io/is-default-class: "true" -provisioner: kubernetes.io/qingcloud-volume \ No newline at end of file diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index dbd4ff5..89689b4 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -26,7 +26,7 @@ spec: - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - --allow-privileged=true - --experimental-bootstrap-token-auth=true - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-preferred-address-types=InternalIP,Hostname - --requestheader-username-headers=X-Remote-User - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota From 06a253231f9635aeea19b76c3ce4375bfecb7ed7 Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 7 Aug 2017 19:26:25 +0800 Subject: [PATCH 017/167] update install script. --- image/install-pkg.sh | 2 +- image/update-k8s-bin.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/image/install-pkg.sh b/image/install-pkg.sh index 50cae71..ea8c200 100755 --- a/image/install-pkg.sh +++ b/image/install-pkg.sh @@ -8,7 +8,7 @@ apt-cache policy docker-engine apt-get install -y docker-engine apt-get install -y ebtables socat -apt-get install -y jq apt-transport-https +apt-get install -y jq apt-transport-https bash-completion apt-get remove network-manager apt-get upgrade -y diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index c254e42..99b9340 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -29,3 +29,5 @@ function download_k8s_bin() rm -rf ${K8S_HOME}/bin/* download_k8s_bin +kubectl completion bash >/etc/profile.d/kubectl.sh +source /etc/profile \ No newline at end of file From 4d6aa52ad6a615521e57e21226cdcd1f8996424a Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 10:54:18 +0800 Subject: [PATCH 018/167] update addons config. --- .../dashboard/dashboard-deployment.yaml | 1 + k8s/addons/kube-dns/kube-dns-cm.yaml | 7 +++ k8s/addons/kube-dns/kube-dns-deployment.yaml | 2 + k8s/addons/kube-dns/kube-dns-sa.yaml | 8 +++ k8s/addons/monitor/es-clusterrole.yaml | 17 ++++++ k8s/addons/monitor/es-clusterrolebinding.yaml | 18 ++++++ k8s/addons/monitor/es-serviceaccount.yaml | 10 ++++ ...es-controller.yaml => es-statefulset.yaml} | 12 +++- .../fluent-bit-daemonset-elasticsearch.yaml | 2 + .../monitor/fluentd-es-clusterrole.yaml | 18 ++++++ .../fluentd-es-clusterrolebinding.yaml | 17 ++++++ .../monitor/fluentd-es-serviceaccount.yaml | 9 +++ ...ntroller.yaml => heapster-deployment.yaml} | 4 +- k8s/addons/monitor/heapster-rbac.yaml | 58 +++++++++++++++++++ k8s/addons/monitor/heapster-sa.yaml | 8 +++ k8s/addons/monitor/heapster-service.yaml | 8 --- ...controller.yaml => kibana-deployment.yaml} | 1 + 17 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 k8s/addons/kube-dns/kube-dns-cm.yaml create mode 100644 k8s/addons/kube-dns/kube-dns-sa.yaml create mode 100644 k8s/addons/monitor/es-clusterrole.yaml create mode 100644 k8s/addons/monitor/es-clusterrolebinding.yaml create mode 100644 k8s/addons/monitor/es-serviceaccount.yaml rename k8s/addons/monitor/{es-controller.yaml => es-statefulset.yaml} (80%) create mode 100644 k8s/addons/monitor/fluentd-es-clusterrole.yaml create mode 100644 k8s/addons/monitor/fluentd-es-clusterrolebinding.yaml create mode 100644 k8s/addons/monitor/fluentd-es-serviceaccount.yaml rename k8s/addons/monitor/{heapster-controller.yaml => heapster-deployment.yaml} (98%) create mode 100644 k8s/addons/monitor/heapster-rbac.yaml create mode 100644 k8s/addons/monitor/heapster-sa.yaml rename k8s/addons/monitor/{kibana-controller.yaml => kibana-deployment.yaml} (96%) diff --git a/k8s/addons/dashboard/dashboard-deployment.yaml b/k8s/addons/dashboard/dashboard-deployment.yaml index 05c57a0..cc307a0 100644 --- a/k8s/addons/dashboard/dashboard-deployment.yaml +++ b/k8s/addons/dashboard/dashboard-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: kube-system labels: k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: EnsureExists spec: selector: diff --git a/k8s/addons/kube-dns/kube-dns-cm.yaml b/k8s/addons/kube-dns/kube-dns-cm.yaml new file mode 100644 index 0000000..279e3f8 --- /dev/null +++ b/k8s/addons/kube-dns/kube-dns-cm.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-deployment.yaml b/k8s/addons/kube-dns/kube-dns-deployment.yaml index 79c91dd..ce61654 100644 --- a/k8s/addons/kube-dns/kube-dns-deployment.yaml +++ b/k8s/addons/kube-dns/kube-dns-deployment.yaml @@ -5,6 +5,8 @@ metadata: namespace: kube-system labels: k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: EnsureExists spec: # replicas: not specified here: # 1. In order to make Addon Manager do not reconcile this replicas parameter. diff --git a/k8s/addons/kube-dns/kube-dns-sa.yaml b/k8s/addons/kube-dns/kube-dns-sa.yaml new file mode 100644 index 0000000..4a0640b --- /dev/null +++ b/k8s/addons/kube-dns/kube-dns-sa.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: EnsureExists \ No newline at end of file diff --git a/k8s/addons/monitor/es-clusterrole.yaml b/k8s/addons/monitor/es-clusterrole.yaml new file mode 100644 index 0000000..e77f51c --- /dev/null +++ b/k8s/addons/monitor/es-clusterrole.yaml @@ -0,0 +1,17 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - "services" + - "namespaces" + - "endpoints" + verbs: + - "get" diff --git a/k8s/addons/monitor/es-clusterrolebinding.yaml b/k8s/addons/monitor/es-clusterrolebinding.yaml new file mode 100644 index 0000000..ee3847b --- /dev/null +++ b/k8s/addons/monitor/es-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + namespace: kube-system + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: +- kind: ServiceAccount + name: elasticsearch-logging + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: elasticsearch-logging + apiGroup: "" diff --git a/k8s/addons/monitor/es-serviceaccount.yaml b/k8s/addons/monitor/es-serviceaccount.yaml new file mode 100644 index 0000000..6f4ede4 --- /dev/null +++ b/k8s/addons/monitor/es-serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + version: v1 + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/k8s/addons/monitor/es-controller.yaml b/k8s/addons/monitor/es-statefulset.yaml similarity index 80% rename from k8s/addons/monitor/es-controller.yaml rename to k8s/addons/monitor/es-statefulset.yaml index 3582dcf..40cbf71 100644 --- a/k8s/addons/monitor/es-controller.yaml +++ b/k8s/addons/monitor/es-statefulset.yaml @@ -1,10 +1,11 @@ apiVersion: apps/v1beta1 kind: StatefulSet metadata: - name: elasticsearch-logging-v1 + name: elasticsearch-logging namespace: kube-system labels: k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: EnsureExists spec: serviceName: "elasticsearch-logging" @@ -19,8 +20,15 @@ spec: annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: + serviceAccountName: elasticsearch-logging + initContainers: + - image: dockerhub.qingcloud.com/alpine:3.6 + command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] + name: elasticsearch-logging-init + securityContext: + privileged: true containers: - - image: dockerhub.qingcloud.com/google_containers/elasticsearch:v5.4.0-1 + - image: dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 imagePullPolicy: IfNotPresent name: elasticsearch-logging resources: diff --git a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml index 02fb9d1..ad12083 100644 --- a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml +++ b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml @@ -5,6 +5,7 @@ metadata: namespace: kube-system labels: k8s-app: fluent-bit-logging + kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: EnsureExists spec: template: @@ -14,6 +15,7 @@ spec: annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: + serviceAccountName: fluentd-es containers: - name: fluent-bit image: dockerhub.qingcloud.com/fluent/fluent-bit-kubernetes-daemonset:0.11.13 diff --git a/k8s/addons/monitor/fluentd-es-clusterrole.yaml b/k8s/addons/monitor/fluentd-es-clusterrole.yaml new file mode 100644 index 0000000..3549564 --- /dev/null +++ b/k8s/addons/monitor/fluentd-es-clusterrole.yaml @@ -0,0 +1,18 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: fluentd-es + labels: + k8s-app: fluentd-es + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - "namespaces" + - "pods" + verbs: + - "get" + - "watch" + - "list" diff --git a/k8s/addons/monitor/fluentd-es-clusterrolebinding.yaml b/k8s/addons/monitor/fluentd-es-clusterrolebinding.yaml new file mode 100644 index 0000000..24ff206 --- /dev/null +++ b/k8s/addons/monitor/fluentd-es-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: fluentd-es + labels: + k8s-app: fluentd-es + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: +- kind: ServiceAccount + name: fluentd-es + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: fluentd-es + apiGroup: "" diff --git a/k8s/addons/monitor/fluentd-es-serviceaccount.yaml b/k8s/addons/monitor/fluentd-es-serviceaccount.yaml new file mode 100644 index 0000000..3a26be2 --- /dev/null +++ b/k8s/addons/monitor/fluentd-es-serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd-es + namespace: kube-system + labels: + k8s-app: fluentd-es + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/k8s/addons/monitor/heapster-controller.yaml b/k8s/addons/monitor/heapster-deployment.yaml similarity index 98% rename from k8s/addons/monitor/heapster-controller.yaml rename to k8s/addons/monitor/heapster-deployment.yaml index 397bdd4..173bff3 100644 --- a/k8s/addons/monitor/heapster-controller.yaml +++ b/k8s/addons/monitor/heapster-deployment.yaml @@ -6,18 +6,16 @@ metadata: labels: k8s-app: heapster addonmanager.kubernetes.io/mode: EnsureExists - version: v1.4.0 + kubernetes.io/cluster-service: "true" spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.4.0 template: metadata: labels: k8s-app: heapster - version: v1.4.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: diff --git a/k8s/addons/monitor/heapster-rbac.yaml b/k8s/addons/monitor/heapster-rbac.yaml new file mode 100644 index 0000000..58fa1b9 --- /dev/null +++ b/k8s/addons/monitor/heapster-rbac.yaml @@ -0,0 +1,58 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: heapster-binding + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:heapster +subjects: +- kind: ServiceAccount + name: heapster + namespace: kube-system +--- +# Heapster's pod_nanny monitors the heapster deployment & its pod(s), and scales +# the resources of the deployment if necessary. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: system:pod-nanny + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "extensions" + resources: + - deployments + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: heapster-binding + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system:pod-nanny +subjects: +- kind: ServiceAccount + name: heapster + namespace: kube-system +--- diff --git a/k8s/addons/monitor/heapster-sa.yaml b/k8s/addons/monitor/heapster-sa.yaml new file mode 100644 index 0000000..7a4e972 --- /dev/null +++ b/k8s/addons/monitor/heapster-sa.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: heapster + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile \ No newline at end of file diff --git a/k8s/addons/monitor/heapster-service.yaml b/k8s/addons/monitor/heapster-service.yaml index 267f5f0..b85e733 100644 --- a/k8s/addons/monitor/heapster-service.yaml +++ b/k8s/addons/monitor/heapster-service.yaml @@ -1,11 +1,3 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: kube-system - name: heapster - labels: - addonmanager.kubernetes.io/mode: EnsureExists ---- kind: Service apiVersion: v1 metadata: diff --git a/k8s/addons/monitor/kibana-controller.yaml b/k8s/addons/monitor/kibana-deployment.yaml similarity index 96% rename from k8s/addons/monitor/kibana-controller.yaml rename to k8s/addons/monitor/kibana-deployment.yaml index fc97e4f..8099a80 100644 --- a/k8s/addons/monitor/kibana-controller.yaml +++ b/k8s/addons/monitor/kibana-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: kube-system labels: k8s-app: kibana-logging + kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: EnsureExists spec: replicas: 1 From 14daddd04bb7c44b22905b407a1cdb52a50f3f5f Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 10:54:36 +0800 Subject: [PATCH 019/167] update docker images. --- docker/elasticsearch/build.sh | 6 +- docker/grafana/Dockerfile | 8 - docker/grafana/Makefile | 96 - docker/grafana/dashboards/cluster.json | 2465 ------------------------ docker/grafana/dashboards/pods.json | 1060 ---------- docker/grafana/run.sh | 16 - docker/grafana/setup_grafana.go | 159 -- docker/influxdb/Dockerfile | 30 - docker/influxdb/Makefile | 50 - docker/influxdb/config.toml | 126 -- docker/influxdb/run.sh | 141 -- docker/kibana/Dockerfile | 39 - docker/kibana/Makefile | 24 - docker/kibana/build.sh | 7 +- docker/kibana/run.sh | 26 - 15 files changed, 7 insertions(+), 4246 deletions(-) delete mode 100644 docker/grafana/Dockerfile delete mode 100644 docker/grafana/Makefile delete mode 100644 docker/grafana/dashboards/cluster.json delete mode 100644 docker/grafana/dashboards/pods.json delete mode 100644 docker/grafana/run.sh delete mode 100644 docker/grafana/setup_grafana.go delete mode 100644 docker/influxdb/Dockerfile delete mode 100644 docker/influxdb/Makefile delete mode 100644 docker/influxdb/config.toml delete mode 100644 docker/influxdb/run.sh delete mode 100644 docker/kibana/Dockerfile delete mode 100644 docker/kibana/Makefile delete mode 100644 docker/kibana/run.sh diff --git a/docker/elasticsearch/build.sh b/docker/elasticsearch/build.sh index c2c0d6c..d0d98d2 100644 --- a/docker/elasticsearch/build.sh +++ b/docker/elasticsearch/build.sh @@ -1,5 +1,5 @@ #!/bin/sh - docker pull gcr.io/google_containers/elasticsearch:v5.4.0-1 - docker tag gcr.io/google_containers/elasticsearch:v5.4.0-1 dockerhub.qingcloud.com/google_containers/elasticsearch:v5.4.0-1 - docker push dockerhub.qingcloud.com/google_containers/elasticsearch:v5.4.0-1 +docker pull gcr.io/google_containers/elasticsearch:v5.5.1 +docker tag gcr.io/google_containers/elasticsearch:v5.5.1 dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 +docker push dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 diff --git a/docker/grafana/Dockerfile b/docker/grafana/Dockerfile deleted file mode 100644 index 9a55537..0000000 --- a/docker/grafana/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM BASEIMAGE - -ADD grafana.tar / -COPY dashboards /dashboards -COPY run.sh / -COPY setup_grafana /usr/bin/ - -ENTRYPOINT ["/run.sh"] diff --git a/docker/grafana/Makefile b/docker/grafana/Makefile deleted file mode 100644 index 829a5c9..0000000 --- a/docker/grafana/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build the influxdb image for amd64, arm, arm64, ppc64le and s390x -# -# Usage: -# [PREFIX=gcr.io/google_containers] [ARCH=amd64] make (build|push) - -all: build - -VERSION?=v4.2.0 -PREFIX?=dockerhub.qingcloud.com/google_containers -ARCH?=amd64 -TEMP_DIR:=$(shell mktemp -d) -LDFLAGS=-w -X main.version=$(VERSION) -X main.commit=unknown-dev -X main.timestamp=0 -extldflags '-static' -DEB_BUILD=4.2.0 -KUBE_CROSS_IMAGE=gcr.io/google_containers/kube-cross:v1.8.3-1 - -# s390x -ALL_ARCHITECTURES=amd64 arm arm64 ppc64le -ML_PLATFORMS=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/s390x - -# Set default base image dynamically for each arch -ifeq ($(ARCH),amd64) - BASEIMAGE?=busybox - CC=gcc -endif -ifeq ($(ARCH),armhfrm) - BASEIMAGE?=armhf/busybox - CC=arm-linux-gnueabi-gcc -endif -ifeq ($(ARCH),arm64) - BASEIMAGE?=aarch64/busybox - CC=aarch64-linux-gnu-gcc -endif -ifeq ($(ARCH),ppc64le) - BASEIMAGE?=ppc64le/busybox - CC=powerpc64le-linux-gnu-gcc -endif -ifeq ($(ARCH),s390x) - BASEIMAGE?=s390x/busybox - CC=s390x-linux-gnu-gcc -endif - -build: - # Copy the whole directory to a temporary dir and set the base image - cp -r ./* $(TEMP_DIR) - - cd $(TEMP_DIR) && sed -i -e "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile - - # This script downloads the official grafana deb package, compiles grafana for the right architecture which replaces the built-in, dynamically linked binaries - # Then the rootfs will be compressed into a tarball again, in order to be ADDed in the Dockerfile. - # Lastly, it compiles the go helper - docker run --rm -it -v $(TEMP_DIR):/build -w /go/src/github.com/grafana/grafana $(KUBE_CROSS_IMAGE) /bin/bash -c "\ - curl -sSL https://github.com/grafana/grafana/archive/$(VERSION).tar.gz | tar -xz --strip-components=1 \ - && curl -sSL https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_$(DEB_BUILD)_amd64.deb > /tmp/grafana.deb \ - && mkdir /tmp/grafanarootfs && dpkg -x /tmp/grafana.deb /tmp/grafanarootfs \ - && CGO_ENABLED=1 GOARCH=$(ARCH) CC=$(CC) go build --ldflags=\"$(LDFLAGS)\" -o /tmp/grafanarootfs/usr/sbin/grafana-server ./pkg/cmd/grafana-server \ - && CGO_ENABLED=1 GOARCH=$(ARCH) CC=$(CC) go build --ldflags=\"$(LDFLAGS)\" -o /tmp/grafanarootfs/usr/sbin/grafana-cli ./pkg/cmd/grafana-cli \ - && cd /tmp/grafanarootfs && tar -cf /build/grafana.tar . \ - && cd /build && CGO_ENABLED=0 GOARCH=$(ARCH) go build -o setup_grafana setup_grafana.go" - - docker build --pull -t $(PREFIX)/heapster-grafana-$(ARCH):$(VERSION) $(TEMP_DIR) - - rm -rf $(TEMP_DIR) - -# Should depend on target: ./manifest-tool -push: gcr-login $(addprefix sub-push-,$(ALL_ARCHITECTURES)) -# ./manifest-tool push from-args --platforms $(ML_PLATFORMS) --template $(PREFIX)/heapster-grafana-ARCH:$(VERSION) --target $(PREFIX)/heapster-grafana:$(VERSION) - -sub-push-%: - $(MAKE) ARCH=$* PREFIX=$(PREFIX) VERSION=$(VERSION) build - docker push $(PREFIX)/heapster-grafana-$*:$(VERSION) - -# TODO(luxas): As soon as it's working to push fat manifests to gcr.io, reenable this code -#./manifest-tool: -# curl -sSL https://github.com/luxas/manifest-tool/releases/download/v0.3.0/manifest-tool > manifest-tool -# chmod +x manifest-tool - -gcr-login: -ifeq ($(findstring gcr.io,$(PREFIX)),gcr.io) - @echo "If you are pushing to a gcr.io registry, you have to be logged in via 'docker login'; 'gcloud docker push' can't push manifest lists yet." - @echo "This script is automatically logging you in now." - gcloud docker -a -endif diff --git a/docker/grafana/dashboards/cluster.json b/docker/grafana/dashboards/cluster.json deleted file mode 100644 index 69f2c2f..0000000 --- a/docker/grafana/dashboards/cluster.json +++ /dev/null @@ -1,2465 +0,0 @@ -{ - "dashboard": { - "id": null, - "title": "Cluster", - "originalTitle": "Cluster", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": true, - "leftYAxisLabel": "Millicores", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/usage_rate", - "query": "SELECT sum(\"value\") FROM \"cpu/usage_rate\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Limit", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/limit", - "query": "SELECT sum(\"value\") FROM \"cpu/limit\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Request", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/request", - "query": "SELECT sum(\"value\") FROM \"cpu/request\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Overall Cluster CPU Usage", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "short", - "short" - ] - } - ], - "title": "Row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": true, - "leftYAxisLabel": "Millicores", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/usage_rate", - "query": "SELECT sum(\"value\") FROM \"cpu/usage_rate\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Limit $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/limit", - "query": "SELECT sum(\"value\") FROM \"cpu/limit\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Request $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/request", - "query": "SELECT sum(\"value\") FROM \"cpu/request\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage by Node", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "short", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 6, - "isNew": true, - "leftYAxisLabel": "Millicores", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/usage_rate", - "query": "SELECT sum(\"value\") FROM \"cpu/usage_rate\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Limit $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/limit", - "query": "SELECT sum(\"value\") FROM \"cpu/limit\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Request $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/request", - "query": "SELECT sum(\"value\") FROM \"cpu/request\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual CPU Usage: $nodename", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "short", - "short" - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "isNew": true, - "leftYAxisLabel": "bytes", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/usage", - "query": "SELECT sum(\"value\") FROM \"memory/usage\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Working Set", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/working_set", - "query": "SELECT sum(\"value\") FROM \"memory/working_set\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Limit", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "hide": false, - "measurement": "memory/limit", - "query": "SELECT sum(\"value\") FROM \"memory/limit\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Request", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "hide": false, - "measurement": "memory/request", - "query": "SELECT sum(\"value\") FROM \"memory/request\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Overall Cluster Memory Usage", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "leftYAxisLabel": "Bytes", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/usage", - "query": "SELECT sum(\"value\") FROM \"memory/usage\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Working Set $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/working_set", - "query": "SELECT sum(\"value\") FROM \"memory/working_set\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Limit $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/limit", - "query": "SELECT sum(\"value\") FROM \"memory/limit\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Request $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/request", - "query": "SELECT sum(\"value\") FROM \"memory/request\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage by Node", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 5, - "isNew": true, - "leftYAxisLabel": "Bytes", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/usage", - "query": "SELECT sum(\"value\") FROM \"memory/usage\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Working Set $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/working_set", - "query": "SELECT sum(\"value\") FROM \"memory/working_set\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Limit $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/limit", - "query": "SELECT sum(\"value\") FROM \"memory/limit\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Request $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/request", - "query": "SELECT sum(\"value\") FROM \"memory/request\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual Node Memory Usage: $nodename", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 7, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Tx", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/tx_rate", - "query": "SELECT sum(\"value\") FROM \"network/tx_rate\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Rx", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/rx_rate", - "query": "SELECT sum(\"value\") FROM \"network/rx_rate\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Overall Cluster Network Usage", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "Bps", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 8, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Tx $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/tx_rate", - "query": "SELECT sum(\"value\") FROM \"network/tx_rate\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Rx $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/rx_rate", - "query": "SELECT sum(\"value\") FROM \"network/rx_rate\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Usage by Node", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "Bps", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 9, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Tx $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/tx_rate", - "query": "SELECT sum(\"value\") FROM \"network/tx_rate\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Rx $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/rx_rate", - "query": "SELECT sum(\"value\") FROM \"network/rx_rate\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual Node Network Usage: $nodename", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "Bps", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 10, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/usage", - "query": "SELECT sum(\"value\") FROM \"filesystem/usage\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Limit", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/limit", - "query": "SELECT sum(\"value\") FROM \"filesystem/limit\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval) fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Overall Cluster Filesystem Usage", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 11, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/usage", - "query": "SELECT sum(\"value\") FROM \"filesystem/usage\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - }, - { - "alias": "Limit $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/limit", - "query": "SELECT sum(\"value\") FROM \"filesystem/limit\" WHERE \"type\" = 'node' AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Filesystem Usage by Node", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 12, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/usage", - "query": "SELECT sum(\"value\") FROM \"filesystem/usage\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - }, - { - "alias": "Limit $tag_nodename", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "nodename" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/limit", - "query": "SELECT sum(\"value\") FROM \"filesystem/limit\" WHERE \"type\" = 'node' AND \"nodename\" =~ /$nodename$/ AND $timeFilter GROUP BY time($interval), \"nodename\" fill(null)", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "node" - }, - { - "condition": "AND", - "key": "nodename", - "operator": "=~", - "value": "/$nodename$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual Filesystem Usage: $nodename", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [ - { - "allFormat": "glob", - "current": { - "text": "kubernetes-minion-hly9", - "value": "kubernetes-minion-hly9" - }, - "datasource": null, - "includeAll": false, - "multi": false, - "multiFormat": "glob", - "name": "nodename", - "options": [ - { - "text": "kubernetes-minion-a7kc", - "value": "kubernetes-minion-a7kc", - "selected": false - }, - { - "text": "kubernetes-minion-hly9", - "value": "kubernetes-minion-hly9", - "selected": true - }, - { - "text": "kubernetes-minion-sfos", - "value": "kubernetes-minion-sfos", - "selected": false - } - ], - "query": "SHOW TAG VALUES FROM \"uptime\" WITH KEY = \"nodename\"", - "refresh": true, - "regex": "", - "type": "query" - } - ] - }, - "annotations": { - "list": [] - }, - "refresh": false, - "schemaVersion": 8, - "version": 10, - "links": [] - }, - "overwrite": false -} diff --git a/docker/grafana/dashboards/pods.json b/docker/grafana/dashboards/pods.json deleted file mode 100644 index f8f8440..0000000 --- a/docker/grafana/dashboards/pods.json +++ /dev/null @@ -1,1060 +0,0 @@ -{ - "dashboard": { - "id": null, - "title": "Pods", - "originalTitle": "Pods", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 6, - "isNew": true, - "leftYAxisLabel": "Millicores", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "type": "tag", - "params": [ - "container_name" - ] - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/usage_rate", - "query": "SELECT sum(\"value\") FROM \"cpu/usage_rate\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Limit $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "type": "tag", - "params": [ - "container_name" - ] - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/limit", - "query": "SELECT sum(\"value\") FROM \"cpu/limit\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Request $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "type": "tag", - "params": [ - "container_name" - ] - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu/request", - "query": "SELECT sum(\"value\") FROM \"cpu/request\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual CPU Usage: $namespace $podname", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "short", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 7, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "container_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/usage", - "query": "SELECT sum(\"value\") FROM \"memory/usage\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Limit $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "container_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/limit", - "query": "SELECT sum(\"value\") FROM \"memory/limit\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Request $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "container_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/request", - "query": "SELECT sum(\"value\") FROM \"memory/request\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Working Set $namespace $podname $tag_container_name", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "container_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "memory/working_set", - "query": "SELECT sum(\"value\") FROM \"memory/working_set\" WHERE \"type\" = 'pod_container' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval), \"container_name\" fill(null)", - "rawQuery": false, - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod_container" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual Memory Usage: $namespace $podname", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 8, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Tx $namespace $podname", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/tx_rate", - "query": "SELECT sum(\"value\") FROM \"network/tx_rate\" WHERE \"type\" = 'pod' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Rx $namespace $podname", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "network/rx_rate", - "query": "SELECT sum(\"value\") FROM \"network/rx_rate\" WHERE \"type\" = 'pod' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Individual Network Usage: $namespace $podname", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "Bps", - "short" - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "leftLogBase": 1, - "leftMax": null, - "leftMin": null, - "rightLogBase": 1, - "rightMax": null, - "rightMin": null, - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 9, - "isNew": true, - "leftYAxisLabel": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage $namespace $podname", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/usage", - "query": "SELECT sum(\"value\") FROM \"filesystem/usage\" WHERE \"type\" = 'pod' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - }, - { - "alias": "Limit $namespace $podname", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "filesystem/limit", - "query": "SELECT sum(\"value\") FROM \"filesystem/limit\" WHERE \"type\" = 'pod' AND \"namespace_name\" =~ /$namespace$/ AND \"pod_name\" =~ /$podname$/ AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "type", - "operator": "=", - "value": "pod" - }, - { - "condition": "AND", - "key": "namespace_name", - "operator": "=~", - "value": "/$namespace$/" - }, - { - "condition": "AND", - "key": "pod_name", - "operator": "=~", - "value": "/$podname$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Filesystem Usage: $namespace $podname", - "tooltip": { - "shared": true, - "value_type": "cumulative" - }, - "type": "graph", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bytes", - "short" - ] - } - ], - "title": "Row" - } - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [ - { - "allFormat": "glob", - "current": { - "tags": [], - "text": "kube-system", - "value": "kube-system" - }, - "datasource": null, - "includeAll": false, - "multi": false, - "multiFormat": "glob", - "name": "namespace", - "options": [ - { - "selected": false, - "text": "default", - "value": "default" - }, - { - "selected": true, - "text": "kube-system", - "value": "kube-system" - } - ], - "query": "SHOW TAG VALUES FROM \"uptime\" WITH KEY = \"namespace_name\"", - "refresh": false, - "type": "query" - }, - { - "allFormat": "glob", - "current": { - "text": "heapster-v19-33x3r", - "value": "heapster-v19-33x3r", - "tags": [] - }, - "datasource": null, - "includeAll": false, - "multi": false, - "multiFormat": "glob", - "name": "podname", - "options": [ - { - "text": "fluentd-cloud-logging-kubernetes-minion-a7kc", - "value": "fluentd-cloud-logging-kubernetes-minion-a7kc", - "selected": false - }, - { - "text": "fluentd-cloud-logging-kubernetes-minion-hly9", - "value": "fluentd-cloud-logging-kubernetes-minion-hly9", - "selected": false - }, - { - "text": "fluentd-cloud-logging-kubernetes-minion-sfos", - "value": "fluentd-cloud-logging-kubernetes-minion-sfos", - "selected": false - }, - { - "text": "heapster-v19-33x3r", - "value": "heapster-v19-33x3r", - "selected": true - }, - { - "text": "kube-dns-v10-v2yty", - "value": "kube-dns-v10-v2yty", - "selected": false - }, - { - "text": "kubernetes-dashboard-gqcgk", - "value": "kubernetes-dashboard-gqcgk", - "selected": false - }, - { - "text": "kubernetes-dashboard-v0.1.0-jz8z8", - "value": "kubernetes-dashboard-v0.1.0-jz8z8", - "selected": false - }, - { - "text": "l7-lb-controller-eojgv", - "value": "l7-lb-controller-eojgv", - "selected": false - }, - { - "text": "monitoring-influxdb-grafana-v6-qxucl", - "value": "monitoring-influxdb-grafana-v6-qxucl", - "selected": false - }, - { - "text": "php-apache-m19ce", - "value": "php-apache-m19ce", - "selected": false - } - ], - "query": "SHOW TAG VALUES FROM \"uptime\" WITH KEY = \"pod_name\" WHERE \"namespace_name\" =~ /$namespace$/", - "refresh": true, - "regex": "", - "type": "query" - } - ] - }, - "annotations": { - "list": [] - }, - "refresh": false, - "schemaVersion": 8, - "version": 14, - "links": [] - }, - "overwrite": false -} \ No newline at end of file diff --git a/docker/grafana/run.sh b/docker/grafana/run.sh deleted file mode 100644 index 399a4d7..0000000 --- a/docker/grafana/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -# Allow access to dashboards without having to log in -# Export these variables so grafana picks them up -export GF_AUTH_ANONYMOUS_ENABLED=${GF_AUTH_ANONYMOUS_ENABLED:-true} -export GF_SERVER_PROTOCOL=${GF_SERVER_PROTOCOL:-http} - -echo "Starting a utility program that will configure Grafana" -setup_grafana >/dev/stdout 2>/dev/stderr & - -echo "Starting Grafana in foreground mode" -exec /usr/sbin/grafana-server \ - --homepath=/usr/share/grafana \ - --config=/etc/grafana/grafana.ini \ - cfg:default.paths.data=/var/lib/grafana \ - cfg:default.paths.logs=/var/log/grafana diff --git a/docker/grafana/setup_grafana.go b/docker/grafana/setup_grafana.go deleted file mode 100644 index 0b70fe3..0000000 --- a/docker/grafana/setup_grafana.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2014 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - "time" -) - -// How many seconds the program should wait before trying to connect to the dashboard again -const RetryTimeout = 5 - -type grafanaConfig struct { - Name string `json:"name"` - Type string `json:"type"` - Access string `json:"access"` - IsDefault bool `json:"isDefault"` - URL string `json:"url"` - Password string `json:"password"` - User string `json:"user"` - Database string `json:"database"` -} - -func main() { - - envParams := map[string]string{ - "grafana_user": "admin", - "grafana_passwd": "admin", - "grafana_port": "3000", - "influxdb_host": "monitoring-influxdb", - "influxdb_port": "8086", - "influxdb_database": "k8s", - "influxdb_user": "root", - "influxdb_password": "root", - "influxdb_service_url": "", - "dashboard_location": "/dashboards", - "gf_auth_anonymous_enabled": "true", - "gf_security_admin_user": "", - "gf_security_admin_password": "", - "gf_server_http_port": "", - "gf_server_protocol": "http", - "backend_access_mode": "proxy", - } - - for k := range envParams { - if v := os.Getenv(strings.ToUpper(k)); v != "" { - envParams[k] = v - } - } - - if envParams["influxdb_service_url"] == "" { - envParams["influxdb_service_url"] = fmt.Sprintf("http://%s:%s", envParams["influxdb_host"], envParams["influxdb_port"]) - } - - cfg := grafanaConfig{ - Name: "influxdb-datasource", - Type: "influxdb", - Access: envParams["backend_access_mode"], - IsDefault: true, - URL: envParams["influxdb_service_url"], - User: envParams["influxdb_user"], - Password: envParams["influxdb_password"], - Database: envParams["influxdb_database"], - } - // Override setup env vars with Grafana configuration env vars if present - adminUser := envParams["grafana_user"] - if user, ok := envParams["gf_security_admin_user"]; ok && len(user) != 0 { - adminUser = user - } - adminPassword := envParams["grafana_passwd"] - if password, ok := envParams["gf_security_admin_password"]; ok && len(password) != 0 { - adminPassword = password - } - httpPort := envParams["grafana_port"] - if port, ok := envParams["gf_server_http_port"]; ok && len(port) != 0 { - httpPort = port - } - - grafanaURL := fmt.Sprintf("%s://%s:%s@localhost:%s", envParams["gf_server_protocol"], adminUser, adminPassword, httpPort) - - for { - res, err := http.Get(grafanaURL + "/api/org") - if err != nil { - fmt.Printf("Can't access the Grafana dashboard. Error: %v. Retrying after %d seconds...\n", err, RetryTimeout) - time.Sleep(RetryTimeout * time.Second) - continue - } - - _, err = ioutil.ReadAll(res.Body) - res.Body.Close() - if err != nil { - fmt.Printf("Can't access the Grafana dashboard. Error: %v. Retrying after %d seconds...\n", err, RetryTimeout) - time.Sleep(RetryTimeout * time.Second) - continue - } - - fmt.Println("Connected to the Grafana dashboard.") - break - } - - b := new(bytes.Buffer) - json.NewEncoder(b).Encode(cfg) - - for { - _, err := http.Post(grafanaURL+"/api/datasources", "application/json; charset=utf-8", b) - if err != nil { - fmt.Printf("Failed to configure the Grafana dashboard. Error: %v. Retrying after %d seconds...\n", err, RetryTimeout) - time.Sleep(RetryTimeout * time.Second) - continue - } - - fmt.Println("The datasource for the Grafana dashboard is now set.") - break - } - - dashboardDir := envParams["dashboard_location"] - files, err := ioutil.ReadDir(dashboardDir) - if err != nil { - fmt.Printf("Failed to read the the directory the json files should be in. Exiting... Error: %v\n", err) - os.Exit(1) - } - for _, file := range files { - if file.IsDir() { - continue - } - - filePath := filepath.Join(dashboardDir, file.Name()) - jsonbytes, err := ioutil.ReadFile(filePath) - if err != nil { - fmt.Printf("Failed to read the json file: %s. Proceeding with the next one. Error: %v\n", filePath, err) - continue - } - - _, err = http.Post(grafanaURL+"/api/dashboards/db", "application/json; charset=utf-8", bytes.NewReader(jsonbytes)) - if err != nil { - fmt.Printf("Failed to post the json file: %s. Proceeding with the next one. Error: %v\n", filePath, err) - continue - } - } -} diff --git a/docker/influxdb/Dockerfile b/docker/influxdb/Dockerfile deleted file mode 100644 index 55955e9..0000000 --- a/docker/influxdb/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM influxdb:1.3.0-alpine - -# Install InfluxDB -ENV INFLUXDB_VERSION 1.3.0 - -ADD config.toml /config/config.toml -ADD run.sh /bin/run.sh -RUN chmod +x /bin/*.sh ;\ - apk update; \ - apk add sed bash curl; - -ENV PRE_CREATE_DB **None** -ENV SSL_SUPPORT **False** -ENV SSL_CERT **None** - -# Admin server WebUI -EXPOSE 8083 - -# HTTP API -EXPOSE 8086 - -# Raft port (for clustering, don't expose publicly!) -#EXPOSE 8090 - -# Protobuf port (for clustering, don't expose publicly!) -#EXPOSE 8099 - -VOLUME ["/data"] - -CMD ["/bin/run.sh"] diff --git a/docker/influxdb/Makefile b/docker/influxdb/Makefile deleted file mode 100644 index ef5baaf..0000000 --- a/docker/influxdb/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build the influxdb image for amd64, arm, arm64, ppc64le and s390x -# -# Usage: -# [PREFIX=gcr.io/google_containers] [ARCH=amd64] make (build|push) - -all: build - -VERSION?=v1.3.0 -PREFIX?=dockerhub.qingcloud.com/google_containers -ARCH?=amd64 - - -build: - # Copy the whole directory to a temporary dir and set the base image - - docker build --pull -t $(PREFIX)/heapster-influxdb-$(ARCH):$(VERSION) . - -# Should depend on target: ./manifest-tool -push: gcr-login $(addprefix sub-push-,$(ALL_ARCHITECTURES)) -# ./manifest-tool push from-args --platforms $(ML_PLATFORMS) --template $(PREFIX)/heapster-influxdb-ARCH:$(VERSION) --target $(PREFIX)/heapster-influxdb:$(VERSION) - -sub-push-%: - $(MAKE) ARCH=$* PREFIX=$(PREFIX) VERSION=$(VERSION) build - docker push $(PREFIX)/heapster-influxdb-$(ARCH):$(VERSION) - -gcr-login: -ifeq ($(findstring gcr.io,$(PREFIX)),gcr.io) - @echo "If you are pushing to a gcr.io registry, you have to be logged in via 'docker login'; 'gcloud docker push' can't push manifest lists yet." - @echo "This script is automatically logging you in now." - gcloud docker -a -endif - -# TODO(luxas): As soon as it's working to push fat manifests to gcr.io, reenable this code -#./manifest-tool: -# curl -sSL https://github.com/luxas/manifest-tool/releases/download/v0.3.0/manifest-tool > manifest-tool -# chmod +x manifest-tool diff --git a/docker/influxdb/config.toml b/docker/influxdb/config.toml deleted file mode 100644 index 3743cd0..0000000 --- a/docker/influxdb/config.toml +++ /dev/null @@ -1,126 +0,0 @@ -reporting-disabled = true -bind-address = "0.0.0.0:8088" - -[meta] - dir = "/data/meta" - retention-autocreate = true - logging-enabled = true - -[data] - dir = "/data/data" - wal-dir = "/data/wal" - query-log-enabled = true - cache-max-memory-size = 1073741824 - cache-snapshot-memory-size = 26214400 - cache-snapshot-write-cold-duration = "10m0s" - compact-full-write-cold-duration = "4h0m0s" - max-series-per-database = 1000000 - max-values-per-tag = 100000 - trace-logging-enabled = false - -[coordinator] - write-timeout = "10s" - max-concurrent-queries = 0 - query-timeout = "0s" - log-queries-after = "0s" - max-select-point = 0 - max-select-series = 0 - max-select-buckets = 0 - -[retention] - enabled = true - check-interval = "30m0s" - -[admin] - enabled = false - bind-address = ":8083" - https-enabled = false - https-certificate = "/etc/ssl/influxdb.pem" - -[shard-precreation] - enabled = true - check-interval = "10m0s" - advance-period = "30m0s" - -[monitor] - store-enabled = true - store-database = "_internal" - store-interval = "10s" - -[subscriber] - enabled = true - http-timeout = "30s" - insecure-skip-verify = false - ca-certs = "" - write-concurrency = 40 - write-buffer-size = 1000 - -[http] - enabled = true - bind-address = ":8086" - auth-enabled = false - log-enabled = true - write-tracing = false - pprof-enabled = false - https-enabled = false - https-certificate = "/etc/ssl/influxdb.pem" - https-private-key = "" - max-row-limit = 10000 - max-connection-limit = 0 - shared-secret = "" - realm = "InfluxDB" - unix-socket-enabled = false - bind-socket = "/var/run/influxdb.sock" - -[[graphite]] - enabled = false - bind-address = ":2003" - database = "graphite" - retention-policy = "" - protocol = "tcp" - batch-size = 5000 - batch-pending = 10 - batch-timeout = "1s" - consistency-level = "one" - separator = "." - udp-read-buffer = 0 - -[[collectd]] - enabled = false - bind-address = ":25826" - database = "collectd" - retention-policy = "" - batch-size = 5000 - batch-pending = 10 - batch-timeout = "10s" - read-buffer = 0 - typesdb = "/usr/share/collectd/types.db" - -[[opentsdb]] - enabled = false - bind-address = ":4242" - database = "opentsdb" - retention-policy = "" - consistency-level = "one" - tls-enabled = false - certificate = "/etc/ssl/influxdb.pem" - batch-size = 1000 - batch-pending = 5 - batch-timeout = "1s" - log-point-errors = true - -[[udp]] - enabled = false - bind-address = ":8089" - database = "udp" - retention-policy = "" - batch-size = 5000 - batch-pending = 10 - read-buffer = 0 - batch-timeout = "1s" - precision = "" - -[continuous_queries] - log-enabled = true - enabled = true - run-interval = "1s" diff --git a/docker/influxdb/run.sh b/docker/influxdb/run.sh deleted file mode 100644 index 93e62e8..0000000 --- a/docker/influxdb/run.sh +++ /dev/null @@ -1,141 +0,0 @@ - #!/bin/bash -CONFIG_FILE="/config/config.toml" -INFLUX_HOST="localhost" -INFLUX_API_PORT="8086" -API_URL="http://${INFLUX_HOST}:${INFLUX_API_PORT}" - -# Dynamically change the value of 'max-open-shards' to what 'ulimit -n' returns -sed -i "s/^max-open-shards.*/max-open-shards = $(ulimit -n)/" ${CONFIG_FILE} - -# Configure InfluxDB Cluster -if [ -n "${FORCE_HOSTNAME}" ]; then - if [ "${FORCE_HOSTNAME}" == "auto" ]; then - #set hostname with IPv4 eth0 - HOSTIPNAME=$(ip a show dev eth0 | grep inet | grep eth0 | sed -e 's/^.*inet.//g' -e 's/\/.*$//g') - /usr/bin/perl -p -i -e "s/hostname = \"localhost\"/hostname = \"${HOSTIPNAME}\"/g" ${CONFIG_FILE} - else - /usr/bin/perl -p -i -e "s/hostname = \"localhost\"/hostname = \"${FORCE_HOSTNAME}\"/g" ${CONFIG_FILE} - fi -fi - -# NOTE: 'seed-servers.' is nowhere to be found in config.toml, this cannot work anymore! NEED FOR REVIEW! -# if [ -n "${SEEDS}" ]; then -# SEEDS=$(eval SEEDS=$SEEDS ; echo $SEEDS | grep '^\".*\"$' || echo "\""$SEEDS"\"" | sed -e 's/, */", "/g') -# /usr/bin/perl -p -i -e "s/^# seed-servers.*$/seed-servers = [${SEEDS}]/g" ${CONFIG_FILE} -# fi - -if [ -n "${REPLI_FACTOR}" ]; then - /usr/bin/perl -p -i -e "s/replication-factor = 1/replication-factor = ${REPLI_FACTOR}/g" ${CONFIG_FILE} -fi - -if [ "${PRE_CREATE_DB}" == "**None**" ]; then - unset PRE_CREATE_DB -fi - -# NOTE: It seems this is not used anymore... -# -# if [ "${SSL_CERT}" == "**None**" ]; then -# unset SSL_CERT -# fi -# -# if [ "${SSL_SUPPORT}" == "**False**" ]; then -# unset SSL_SUPPORT -# fi - -# Add Graphite support -if [ -n "${GRAPHITE_DB}" ]; then - echo "GRAPHITE_DB: ${GRAPHITE_DB}" - sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/false/true/; s/\"graphitedb\"/\"${GRAPHITE_DB}\"/g; }" ${CONFIG_FILE} -fi - -if [ -n "${GRAPHITE_BINDING}" ]; then - echo "GRAPHITE_BINDING: ${GRAPHITE_BINDING}" - sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/\:2003/${GRAPHITE_BINDING}/; }" ${CONFIG_FILE} -fi - -if [ -n "${GRAPHITE_PROTOCOL}" ]; then - echo "GRAPHITE_PROTOCOL: ${GRAPHITE_PROTOCOL}" - sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/tcp/${GRAPHITE_PROTOCOL}/; }" ${CONFIG_FILE} -fi - -if [ -n "${GRAPHITE_TEMPLATE}" ]; then - echo "GRAPHITE_TEMPLATE: ${GRAPHITE_TEMPLATE}" - sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/instance\.profile\.measurement\*/${GRAPHITE_TEMPLATE}/; }" ${CONFIG_FILE} -fi - -# Add Collectd support -if [ -n "${COLLECTD_DB}" ]; then - echo "COLLECTD_DB: ${COLLECTD_DB}" - sed -i -r -e "/^\[\[collectd\]\]/, /^$/ { s/false/true/; s/( *)# *(.*)\"collectd\"/\1\2\"${COLLECTD_DB}\"/g;}" ${CONFIG_FILE} -fi -if [ -n "${COLLECTD_BINDING}" ]; then - echo "COLLECTD_BINDING: ${COLLECTD_BINDING}" - sed -i -r -e "/^\[\[collectd\]\]/, /^$/ { s/( *)# *(.*)\":25826\"/\1\2\"${COLLECTD_BINDING}\"/g;}" ${CONFIG_FILE} -fi -if [ -n "${COLLECTD_RETENTION_POLICY}" ]; then - echo "COLLECTD_RETENTION_POLICY: ${COLLECTD_RETENTION_POLICY}" - sed -i -r -e "/^\[\[collectd\]\]/, /^$/ { s/( *)# *(retention-policy.*)\"\"/\1\2\"${COLLECTD_RETENTION_POLICY}\"/g;}" ${CONFIG_FILE} -fi - -# Add UDP support -if [ -n "${UDP_DB}" ]; then - sed -i -r -e "/^\[\[udp\]\]/, /^$/ { s/false/true/; s/#//g; s/\"udpdb\"/\"${UDP_DB}\"/g; }" ${CONFIG_FILE} -fi -if [ -n "${UDP_PORT}" ]; then - sed -i -r -e "/^\[\[udp\]\]/, /^$/ { s/4444/${UDP_PORT}/; }" ${CONFIG_FILE} -fi - - -echo "influxdb configuration: " -cat ${CONFIG_FILE} -echo "=> Starting InfluxDB ..." -if [ -n "${JOIN}" ]; then - exec influxd -config=${CONFIG_FILE} -join ${JOIN} & -else - exec influxd -config=${CONFIG_FILE} & -fi -pid=$! -# Pre create database on the initiation of the container -if [ -n "${PRE_CREATE_DB}" ]; then - echo "=> About to create the following database: ${PRE_CREATE_DB}" - if [ -f "/data/.pre_db_created" ]; then - echo "=> Database had been created before, skipping ..." - else - arr=$(echo ${PRE_CREATE_DB} | tr ";" "\n") - - #wait for the startup of influxdb - RET=1 - while [[ $RET -ne 0 ]]; do - echo "=> Waiting for confirmation of InfluxDB service startup ..." - sleep 3 - curl -k ${API_URL}/ping 2> /dev/null - RET=$? - done - echo "" - - PASS=${INFLUXDB_INIT_PWD:-root} - if [ -n "${ADMIN_USER}" ]; then - echo "=> Creating admin user" - influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -execute="CREATE USER ${ADMIN_USER} WITH PASSWORD '${PASS}' WITH ALL PRIVILEGES" - for x in $arr - do - echo "=> Creating database: ${x}" - influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -username=${ADMIN_USER} -password="${PASS}" -execute="create database ${x}" - influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -username=${ADMIN_USER} -password="${PASS}" -execute="grant all PRIVILEGES on ${x} to ${ADMIN_USER}" - done - echo "" - else - for x in $arr - do - echo "=> Creating database: ${x}" - influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -execute="create database \"${x}\"" - done - fi - - touch "/data/.pre_db_created" - fi -else - echo "=> No database need to be pre-created" -fi - -wait $pid diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile deleted file mode 100644 index 2ee6794..0000000 --- a/docker/kibana/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A Dockerfile for creating a Kibana container that is designed -# to work with Kubernetes logging. - -FROM gcr.io/google_containers/ubuntu-slim:0.6 - - -ENV DEBIAN_FRONTEND noninteractive -ENV KIBANA_VERSION 5.5.0 - -RUN apt-get update \ - && apt-get install -y curl \ - && apt-get clean - -RUN set -x \ - && cd / \ - && mkdir /kibana \ - && curl -O https://artifacts.elastic.co/downloads/kibana/kibana-$KIBANA_VERSION-linux-x86_64.tar.gz \ - && tar xf kibana-$KIBANA_VERSION-linux-x86_64.tar.gz -C /kibana --strip-components=1 \ - && rm kibana-$KIBANA_VERSION-linux-x86_64.tar.gz - -COPY run.sh /run.sh - -EXPOSE 5601 - -CMD ["/run.sh"] diff --git a/docker/kibana/Makefile b/docker/kibana/Makefile deleted file mode 100644 index a20f580..0000000 --- a/docker/kibana/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.PHONY: build push - -TAG = v5.5.0 -PREFIX = dockerhub.qingcloud.com/google_containers - -build: - docker build --pull -t $(PREFIX)/kibana:$(TAG) . - -push: - docker push $(PREFIX)/kibana:$(TAG) diff --git a/docker/kibana/build.sh b/docker/kibana/build.sh index c2927c0..9b5c419 100644 --- a/docker/kibana/build.sh +++ b/docker/kibana/build.sh @@ -1,5 +1,6 @@ #!/bin/bash -docker pull gcr.io/google_containers/kibana:v5.4.0 -docker tag gcr.io/google_containers/kibana:v5.4.0 dockerhub.qingcloud.com/google_containers/kibana:v5.4.0 -docker push dockerhub.qingcloud.com/google_containers/kibana:v5.4.0 +docker pull docker.elastic.co/kibana/kibana:5.5.1 +docker tag docker.elastic.co/kibana/kibana:5.5.1 dockerhub.qingcloud.com/elastic/kibana:5.5.1 +docker push dockerhub.qingcloud.com/elastic/kibana:5.5.1 + diff --git a/docker/kibana/run.sh b/docker/kibana/run.sh deleted file mode 100644 index 2662ca9..0000000 --- a/docker/kibana/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export ELASTICSEARCH_URL=${ELASTICSEARCH_URL:-"http://localhost:9200"} -echo ELASTICSEARCH_URL=${ELASTICSEARCH_URL} - -export KIBANA_BASE_URL=${KIBANA_BASE_URL:-"''"} -echo "server.basePath: ${KIBANA_BASE_URL}" -echo "server.basePath: ${KIBANA_BASE_URL}" >> /kibana/config/kibana.yml -echo "server.host: ${KIBANA_HOST}" -echo "server.host: ${KIBANA_HOST}" >> /kibana/config/kibana.yml - -/kibana/bin/kibana -e ${ELASTICSEARCH_URL} From 98800af008beba2725226eb2b627e403f380001c Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 11:45:42 +0800 Subject: [PATCH 020/167] update docker images. --- image/pull-docker-images.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/image/pull-docker-images.sh b/image/pull-docker-images.sh index 81ce121..1029741 100755 --- a/image/pull-docker-images.sh +++ b/image/pull-docker-images.sh @@ -7,6 +7,14 @@ source ${K8S_HOME}/version echo "update images" +cat << EOF > /etc/docker/daemon.json +{ + "storage-driver": "overlay2" +} +EOF + +systemcl restart docker + docker login -u guest -p guest dockerhub.qingcloud.com docker pull dockerhub.qingcloud.com/google_containers/pause-amd64:3.0 @@ -21,7 +29,7 @@ docker pull dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64 docker pull dockerhub.qingcloud.com/fluent/fluent-bit-kubernetes-daemonset:0.11.13 docker pull dockerhub.qingcloud.com/google_containers/heapster-amd64:v1.4.0 docker pull dockerhub.qingcloud.com/google_containers/addon-resizer:1.7 -docker pull dockerhub.qingcloud.com/google_containers/kibana:v5.4.0 -docker pull dockerhub.qingcloud.com/google_containers/elasticsearch:v5.4.0-1 +docker pull dockerhub.qingcloud.com/elastic/kibana:5.5.1 +docker pull dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 docker pull dockerhub.qingcloud.com/qingcloud/elasticsearch-curator:5.1.1 docker pull dockerhub.qingcloud.com/qingcloud/file-sync:0.7 \ No newline at end of file From 971e98531704b416a71736c85639d75b82310247 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 11:46:02 +0800 Subject: [PATCH 021/167] update docker daemon remove ipv6 false. --- confd/templates/k8s/daemon.json.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/confd/templates/k8s/daemon.json.tmpl b/confd/templates/k8s/daemon.json.tmpl index c468b08..9077a5c 100644 --- a/confd/templates/k8s/daemon.json.tmpl +++ b/confd/templates/k8s/daemon.json.tmpl @@ -1,10 +1,9 @@ { "live-restore": true, "graph": "/data/var/lib/docker", - "storage-driver": "overlay2", {{$registries := getv "/env/insecure-registries"}} {{if ne $registries ""}} "insecure-registries": {{split $registries " "|toJson}}, {{end}} - "ipv6": false + "storage-driver": "overlay2" } \ No newline at end of file From ced17f8f293fe5a042fe0399e5ad26c848b5da2c Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 16:41:55 +0800 Subject: [PATCH 022/167] client use same image with node. --- confd/conf.d/k8s/interfaces.toml | 7 +++++++ confd/templates/k8s/interfaces.tmpl | 14 ++++++++++++++ image/update-k8s-bin.sh | 1 + script/common.sh | 5 ++++- script/init-client.sh | 13 +++++++++++++ script/reload-interface.sh | 2 ++ 6 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 confd/conf.d/k8s/interfaces.toml create mode 100644 confd/templates/k8s/interfaces.tmpl create mode 100644 script/init-client.sh create mode 100644 script/reload-interface.sh diff --git a/confd/conf.d/k8s/interfaces.toml b/confd/conf.d/k8s/interfaces.toml new file mode 100644 index 0000000..af71948 --- /dev/null +++ b/confd/conf.d/k8s/interfaces.toml @@ -0,0 +1,7 @@ +[template] +src = "k8s/interfaces.tmpl" +dest = "/etc/network/interfaces" +keys = [ + "/", +] +reload_cmd = "/opt/kubernetes/script/reload-interface.sh" \ No newline at end of file diff --git a/confd/templates/k8s/interfaces.tmpl b/confd/templates/k8s/interfaces.tmpl new file mode 100644 index 0000000..3de12b0 --- /dev/null +++ b/confd/templates/k8s/interfaces.tmpl @@ -0,0 +1,14 @@ +{{$role := getv "host/role"}}{{if eq $role "client"}} +auto lo +iface lo inet loopback +{{$masters := getvs "/hosts/master/*/ip"}} +allow-hotplug eth0 +iface eth0 inet dhcp + dns-search default.svc.cluster.local svc.cluster.local cluster.local + dns-nameservers 10.96.0.10 + up route add -net 10.96.0.0 netmask 255.255.0.0 gw {{index $masters 0}}{{else}} +auto lo +iface lo inet loopback + +allow-hotplug eth0 +iface eth0 inet dhcp{{end}} \ No newline at end of file diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index 99b9340..b369d5b 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -30,4 +30,5 @@ rm -rf ${K8S_HOME}/bin/* download_k8s_bin kubectl completion bash >/etc/profile.d/kubectl.sh +kubeadm completion bash >/etc/profile.d/kubeadm.sh source /etc/profile \ No newline at end of file diff --git a/script/common.sh b/script/common.sh index 90e80a3..64d1594 100755 --- a/script/common.sh +++ b/script/common.sh @@ -42,6 +42,9 @@ function mykubectl(){ } function ensure_dir(){ + if [ ! -d /root/.kube ]; then + mkdir /root/.kube + fi if [ ! -d /data/kubernetes ]; then mkdir -p /data/kubernetes fi @@ -137,7 +140,7 @@ function join_node(){ echo "master ip: ${MASTER_IP} init_token: ${init_token}" - retry kubeadm join ${MASTER_IP} --token ${init_token} --skip-preflight-checks + retry kubeadm join ${MASTER_IP}:6443 --token ${init_token} --skip-preflight-checks touch ${NODE_INIT_LOCK} } diff --git a/script/init-client.sh b/script/init-client.sh new file mode 100644 index 0000000..cb2d4d6 --- /dev/null +++ b/script/init-client.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +K8S_HOME=$(dirname "${SCRIPTPATH}") + +source "${K8S_HOME}/script/common.sh" + +sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config +systemctl restart ssh + +ensure_dir +join_node + +ln -fs /etc/kubernetes/kubelet.conf /root/.kube/config \ No newline at end of file diff --git a/script/reload-interface.sh b/script/reload-interface.sh new file mode 100644 index 0000000..9a49d9a --- /dev/null +++ b/script/reload-interface.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +ifdown eth0 && ifup eth0 \ No newline at end of file From 83ba60ec18c84cf8c04933c9c388ec7ad8dbc542 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 16:46:49 +0800 Subject: [PATCH 023/167] fix typo. --- image/pull-docker-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/pull-docker-images.sh b/image/pull-docker-images.sh index 1029741..55eb34d 100755 --- a/image/pull-docker-images.sh +++ b/image/pull-docker-images.sh @@ -13,7 +13,7 @@ cat << EOF > /etc/docker/daemon.json } EOF -systemcl restart docker +systemctl restart docker docker login -u guest -p guest dockerhub.qingcloud.com From 1626300e6c2c171e584a1865aeef1212b5368460 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:14:06 +0800 Subject: [PATCH 024/167] update cloud-controller-manager config. --- confd/templates/k8s/kubelet.tmpl | 2 +- k8s/manifests/kube-apiserver.yaml | 1 - .../kube-cloud-controller-manager.yaml | 49 +++++++++++++++++++ k8s/manifests/kube-controller-manager.yaml | 1 + script/init-master.sh | 1 + 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 k8s/manifests/kube-cloud-controller-manager.yaml diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index 19bc136..35e5f1a 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -27,6 +27,6 @@ KUBELET_KUBECONFIG_ARGS="--kubeconfig=/etc/kubernetes/kubelet.conf --require-kub KUBELET_DNS_ARGS="--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" -KUBE_CLOUD_PROVIDER="--cloud-provider=qingcloud --cloud-config=/etc/kubernetes/qingcloud.conf" +KUBE_CLOUD_PROVIDER="--cloud-provider=external" KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates AllAlpha=true --root-dir /data/var/lib/kubelet" \ No newline at end of file diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index 89689b4..719b802 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -3,7 +3,6 @@ kind: Pod metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" - creationTimestamp: null labels: component: kube-apiserver tier: control-plane diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml new file mode 100644 index 0000000..8cf0e98 --- /dev/null +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + component: kube-cloud-controller-manager + tier: control-plane + name: kube-cloud-controller-manager + namespace: kube-system +spec: + containers: + - args: + - --v=${KUBE_LOG_LEVEL} + - --cloud-provider qingcloud + - --cloud-config /etc/kubernetes/qingcloud.conf + - --kubeconfig=/etc/kubernetes/cloud-controller-manager.conf + - -use-service-account-credentials=true + - --feature-gates + - AllAlpha=true + image: dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10253 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: qingcloud-cloud-controller-manager + resources: + requests: + cpu: 250m + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s + readOnly: true + - mountPath: /etc/ssl/certs + name: certs + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes + name: k8s + - hostPath: + path: /etc/ssl/certs + name: certs \ No newline at end of file diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index a897d20..57dc68d 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -25,6 +25,7 @@ spec: - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --allocate-node-cidrs=false - --service-cluster-ip-range=10.96.0.0/16 + - --cloud-provider=external - --feature-gates - AllAlpha=true image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} diff --git a/script/init-master.sh b/script/init-master.sh index 1c2f2df..13db364 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -12,6 +12,7 @@ init_token=$(get_or_gen_init_token) kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --api-external-dns-names ${ENV_API_EXTERNAL_DOMAIN} kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:node:${HOST_INSTANCE_ID} --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf +kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:cloud-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/cloud-controller-manager.conf kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-scheduler --server https://${MASTER_IP}:6443 > /etc/kubernetes/scheduler.conf kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --server https://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf docker_login From a4dc4b429eddf87d09ff4aa15311d38b85869139 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:37:27 +0800 Subject: [PATCH 025/167] update script. --- script/common.sh | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/script/common.sh b/script/common.sh index 64d1594..21a42a2 100755 --- a/script/common.sh +++ b/script/common.sh @@ -57,6 +57,7 @@ function ensure_dir(){ } function get_or_gen_init_token(){ + local init_token="" if [ -f "/data/kubernetes/init_token" ]; then init_token=$(cat /data/kubernetes/init_token) fi @@ -68,12 +69,12 @@ function get_or_gen_init_token(){ } function replace_vars(){ - from=$1 - to=$2 + local from=$1 + local to=$2 echo "process ${from} to ${to}" - prefix=$(timestamp) - name=$(basename ${from}) - tmpfile="/tmp/${prefix}-${name}" + local prefix=$(timestamp) + local name=$(basename ${from}) + local tmpfile="/tmp/${prefix}-${name}" sed 's/${HYPERKUBE_VERSION}/'"${HYPERKUBE_VERSION}"'/g' ${from} > ${tmpfile} sed -i 's/${KUBE_LOG_LEVEL}/'"${ENV_KUBE_LOG_LEVEL}"'/g' ${tmpfile} sed -i 's/${HOST_IP}/'"${HOST_IP}"'/g' ${tmpfile} @@ -82,8 +83,10 @@ function replace_vars(){ then sed -i 's/replicas:\s./replicas: '"${LOG_COUNT}"'/g' ${tmpfile} fi - - diff ${tmpfile} ${to} >> /dev/null + if [ -f ${to} ] + then + diff ${tmpfile} ${to} >> /dev/null + fi if [ "$?" -ne 0 ] then cp ${tmpfile} ${to} @@ -96,8 +99,8 @@ function replace_vars(){ function update_k8s_manifests(){ echo "echo update k8s manifests" - #mkdir /data/kubernetes/manifests/ || rm -rf /data/kubernetes/manifests/* - #mkdir /data/kubernetes/addons/ || rm -rf /data/kubernetes/addons/* + mkdir /data/kubernetes/manifests/ || rm -rf /data/kubernetes/manifests/* + mkdir /data/kubernetes/addons/ || rm -rf /data/kubernetes/addons/* process_manifests } @@ -109,14 +112,14 @@ function process_manifests(){ replace_vars ${f} /data/kubernetes/manifests/${name} done -# for addon in ${K8S_HOME}/k8s/addons/*; do -# addon_name=$(basename $addon) -# mkdir -p /data/kubernetes/addons/${addon_name} -# for f in ${addon}/*; do -# name=$(basename ${f}) -# replace_vars ${f} /data/kubernetes/addons/${addon_name}/${name} -# done -# done + for addon in ${K8S_HOME}/k8s/addons/*; do + addon_name=$(basename $addon) + mkdir -p /data/kubernetes/addons/${addon_name} + for f in ${addon}/*; do + name=$(basename ${f}) + replace_vars ${f} /data/kubernetes/addons/${addon_name}/${name} + done + done } function scale_es(){ From 1b28aea7a825275d409ca4f949f42909c00c4c99 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:38:18 +0800 Subject: [PATCH 026/167] update script add execute. --- script/init-client.sh | 0 script/reload-interface.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 script/init-client.sh mode change 100644 => 100755 script/reload-interface.sh diff --git a/script/init-client.sh b/script/init-client.sh old mode 100644 new mode 100755 diff --git a/script/reload-interface.sh b/script/reload-interface.sh old mode 100644 new mode 100755 From 3f5c0c4381b08842928449d1b65f0356c7bcefd6 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:44:08 +0800 Subject: [PATCH 027/167] update script --- script/common.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/script/common.sh b/script/common.sh index 21a42a2..fa3145d 100755 --- a/script/common.sh +++ b/script/common.sh @@ -252,11 +252,20 @@ function drain_node(){ } function link_dynamic_dir(){ - mkdir -p /data/var && mkdir /data/var/lib && mkdir /data/var/log - mv /var/lib/docker /data/var/lib/ - ln -s /data/var/lib/docker /var/lib/docker - mkdir /data/var/lib/kubelet && ln -s /data/var/lib/kubelet /var/lib/kubelet - ln -s /root/.docker /data/var/lib/kubelet/.docker + if [ ! -d "/data/var" ] + then + mkdir -p /data/var && mkdir /data/var/lib && mkdir /data/var/log + fi + if [ -d /var/lib/docker ] + then + mv /var/lib/docker /data/var/lib/ + ln -s /data/var/lib/docker /var/lib/docker + fi + if [ ! -d "/data/var/lib/kubelet" ] + then + mkdir /data/var/lib/kubelet && ln -s /data/var/lib/kubelet /var/lib/kubelet + fi + ln -fs /root/.docker /data/var/lib/kubelet/.docker } function docker_stop_rm_all () { From 33fa1ecef56b8d2c42e2740b10847dc42f1c2388 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:46:14 +0800 Subject: [PATCH 028/167] update script --- script/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/common.sh b/script/common.sh index fa3145d..190c8ff 100755 --- a/script/common.sh +++ b/script/common.sh @@ -256,7 +256,7 @@ function link_dynamic_dir(){ then mkdir -p /data/var && mkdir /data/var/lib && mkdir /data/var/log fi - if [ -d /var/lib/docker ] + if [ -d /var/lib/docker ] && [ ! -L /var/lib/docker ] then mv /var/lib/docker /data/var/lib/ ln -s /data/var/lib/docker /var/lib/docker From 35aae5ce54e59d4197cef82e9cb5596347b81a3e Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:49:19 +0800 Subject: [PATCH 029/167] fix init-master --- script/init-master.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/init-master.sh b/script/init-master.sh index 13db364..6cdd2b4 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -9,7 +9,7 @@ link_dynamic_dir init_token=$(get_or_gen_init_token) #retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf -kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --api-external-dns-names ${ENV_API_EXTERNAL_DOMAIN} +kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --cert-altnames ${ENV_API_EXTERNAL_DOMAIN} kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:node:${HOST_INSTANCE_ID} --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:cloud-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/cloud-controller-manager.conf From eda1aeda6f77226829b57ba2934b8a51d0f58acd Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:52:31 +0800 Subject: [PATCH 030/167] fix docker login --- script/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/common.sh b/script/common.sh index 190c8ff..db1196f 100755 --- a/script/common.sh +++ b/script/common.sh @@ -292,9 +292,9 @@ function wait_qingcloudvolume_detach(){ } function docker_login(){ - if [ ! -z "${DOCKERHUB_USERNAME}" ] && [ ! -z "${DOCKERHUB_PASSWORD}" ] + if [ ! -z "${ENV_DOCKERHUB_USERNAME}" ] && [ ! -z "${ENV_DOCKERHUB_PASSWORD}" ] then - retry docker login dockerhub.qingcloud.com -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} + retry docker login dockerhub.qingcloud.com -u ${ENV_DOCKERHUB_USERNAME} -p ${ENV_DOCKERHUB_PASSWORD} fi } From 92575f7af70874cc3270a3f74620de4308d57c8f Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:57:25 +0800 Subject: [PATCH 031/167] fix replace var --- script/common.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/script/common.sh b/script/common.sh index db1196f..6a2bc42 100755 --- a/script/common.sh +++ b/script/common.sh @@ -86,14 +86,18 @@ function replace_vars(){ if [ -f ${to} ] then diff ${tmpfile} ${to} >> /dev/null - fi - if [ "$?" -ne 0 ] - then - cp ${tmpfile} ${to} - echo "${to} update" + if [ "$?" -ne 0 ] + then + cp ${tmpfile} ${to} + echo "${to} update" + else + echo "${to} in sync" + fi else - echo "${to} in sync" + cp ${tmpfile} ${to} + echo "${to} create" fi + rm ${tmpfile} } From 33554d8b6552f8d378d5ba39a4a5024819bf9ada Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 17:57:57 +0800 Subject: [PATCH 032/167] fix replace var --- script/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/common.sh b/script/common.sh index 6a2bc42..f858b6b 100755 --- a/script/common.sh +++ b/script/common.sh @@ -97,7 +97,7 @@ function replace_vars(){ cp ${tmpfile} ${to} echo "${to} create" fi - + rm ${tmpfile} } From 5fcc6be4a1e8596bb7d7b16194e763e6fee9fea6 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 18:00:33 +0800 Subject: [PATCH 033/167] fix kubeconfig --- script/init-master.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/init-master.sh b/script/init-master.sh index 6cdd2b4..98c30c4 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -10,10 +10,10 @@ link_dynamic_dir init_token=$(get_or_gen_init_token) #retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --cert-altnames ${ENV_API_EXTERNAL_DOMAIN} -kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:node:${HOST_INSTANCE_ID} --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf -kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf -kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:cloud-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/cloud-controller-manager.conf -kubeadm alpha phase kubeconfig client-certs --client-name --client-name system:kube-scheduler --server https://${MASTER_IP}:6443 > /etc/kubernetes/scheduler.conf +kubeadm alpha phase kubeconfig client-certs --client-name system:node:${HOST_INSTANCE_ID} --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf +kubeadm alpha phase kubeconfig client-certs --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf +kubeadm alpha phase kubeconfig client-certs --client-name system:cloud-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/cloud-controller-manager.conf +kubeadm alpha phase kubeconfig client-certs --client-name system:kube-scheduler --server https://${MASTER_IP}:6443 > /etc/kubernetes/scheduler.conf kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --server https://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf docker_login From 64df8237b37bbefe246261d3bfebc307cb45d299 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 18:09:47 +0800 Subject: [PATCH 034/167] enable kube-apiserver insecure access. --- k8s/manifests/kube-apiserver.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index 719b802..4ff9697 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -17,7 +17,8 @@ spec: - --requestheader-allowed-names=front-proxy-client - --service-cluster-ip-range=10.96.0.0/12 - --secure-port=6443 - - --insecure-port=0 + - --insecure-port=8080 + - --insecure-bind-address=127.0.0.1 - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key From 2bb0e9dfa0ccf576b69cc5bacd510ab40c794921 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 18:11:46 +0800 Subject: [PATCH 035/167] disable errexit --- script/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/common.sh b/script/common.sh index f858b6b..6a84809 100755 --- a/script/common.sh +++ b/script/common.sh @@ -5,7 +5,8 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") source "/data/kubernetes/env.sh" source "${K8S_HOME}/version" -set -o errexit +set -e +#set -o errexit set -o nounset set -o pipefail From 37cf8d70c88f65e3cfa7a37c804c8c83b588d811 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 19:08:12 +0800 Subject: [PATCH 036/167] update kube-cloud-controller-manager --- k8s/manifests/kube-cloud-controller-manager.yaml | 3 ++- script/common.sh | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 8cf0e98..9ee1c8c 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -10,7 +10,8 @@ metadata: namespace: kube-system spec: containers: - - args: + - command: + - qingcloud-cloud-controller-manager - --v=${KUBE_LOG_LEVEL} - --cloud-provider qingcloud - --cloud-config /etc/kubernetes/qingcloud.conf diff --git a/script/common.sh b/script/common.sh index 6a84809..f4d15ad 100755 --- a/script/common.sh +++ b/script/common.sh @@ -5,7 +5,6 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") source "/data/kubernetes/env.sh" source "${K8S_HOME}/version" -set -e #set -o errexit set -o nounset set -o pipefail From 51903334b1a248e7bebac1e8c57a76910fbdfe11 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 19:23:32 +0800 Subject: [PATCH 037/167] update cloud-controller-manager manifests --- k8s/manifests/kube-cloud-controller-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 9ee1c8c..905e917 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -11,7 +11,7 @@ metadata: spec: containers: - command: - - qingcloud-cloud-controller-manager + - /qingcloud-cloud-controller-manager - --v=${KUBE_LOG_LEVEL} - --cloud-provider qingcloud - --cloud-config /etc/kubernetes/qingcloud.conf From 7195f300a07540c2283fed0274dd8995e1890bf2 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 19:43:04 +0800 Subject: [PATCH 038/167] update cloud-controller-manager config --- k8s/manifests/kube-cloud-controller-manager.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 905e917..cc2a140 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -13,10 +13,10 @@ spec: - command: - /qingcloud-cloud-controller-manager - --v=${KUBE_LOG_LEVEL} - - --cloud-provider qingcloud - - --cloud-config /etc/kubernetes/qingcloud.conf + - --cloud-provider=qingcloud + - --cloud-config=/etc/kubernetes/qingcloud.conf - --kubeconfig=/etc/kubernetes/cloud-controller-manager.conf - - -use-service-account-credentials=true + - --use-service-account-credentials=true - --feature-gates - AllAlpha=true image: dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager @@ -40,11 +40,17 @@ spec: readOnly: true - mountPath: /etc/ssl/certs name: certs + - mountPath: /etc/qingcloud + name: qingcloud + readOnly: true hostNetwork: true volumes: - hostPath: path: /etc/kubernetes - name: k8s + name: k8s - hostPath: path: /etc/ssl/certs - name: certs \ No newline at end of file + name: certs + - hostPath: + path: /etc/qingcloud + name: qingcloud \ No newline at end of file From b8bd75071cee1a1b96410ba6c841a181ad35a016 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 9 Aug 2017 19:48:09 +0800 Subject: [PATCH 039/167] fix cloud-controller-manager --- k8s/manifests/kube-cloud-controller-manager.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index cc2a140..7a855e9 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -47,10 +47,10 @@ spec: volumes: - hostPath: path: /etc/kubernetes - name: k8s + name: k8s - hostPath: path: /etc/ssl/certs - name: certs + name: certs - hostPath: path: /etc/qingcloud - name: qingcloud \ No newline at end of file + name: qingcloud \ No newline at end of file From e3e52c3da1827a5fd5faf1212c11f847ece3aeb8 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 09:43:11 +0800 Subject: [PATCH 040/167] update cloud-controller-manager --- k8s/manifests/kube-cloud-controller-manager.yaml | 6 +++--- k8s/manifests/kube-controller-manager.yaml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 7a855e9..5c0507d 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -4,9 +4,9 @@ metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" labels: - component: kube-cloud-controller-manager + component: cloud-controller-manager tier: control-plane - name: kube-cloud-controller-manager + name: cloud-controller-manager namespace: kube-system spec: containers: @@ -15,7 +15,7 @@ spec: - --v=${KUBE_LOG_LEVEL} - --cloud-provider=qingcloud - --cloud-config=/etc/kubernetes/qingcloud.conf - - --kubeconfig=/etc/kubernetes/cloud-controller-manager.conf + - --kubeconfig=/etc/kubernetes/controller-manager.conf - --use-service-account-credentials=true - --feature-gates - AllAlpha=true diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 57dc68d..ee35fc6 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -3,7 +3,6 @@ kind: Pod metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" - creationTimestamp: null labels: component: kube-controller-manager tier: control-plane From 1e071d47458442b92d8656a20363d7f6bab6776f Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 10:23:46 +0800 Subject: [PATCH 041/167] cloud-controller-manager - --service-account-private-key-file config --- k8s/manifests/kube-cloud-controller-manager.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 5c0507d..95bddfb 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -17,8 +17,8 @@ spec: - --cloud-config=/etc/kubernetes/qingcloud.conf - --kubeconfig=/etc/kubernetes/controller-manager.conf - --use-service-account-credentials=true - - --feature-gates - - AllAlpha=true + - --service-account-private-key-file=/etc/kubernetes/pki/sa.key + - --feature-gates=AllAlpha=true image: dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager imagePullPolicy: IfNotPresent livenessProbe: From 9cd354ed7158a77e1269cdc06a84e8f38a53859a Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 10:37:36 +0800 Subject: [PATCH 042/167] update kubelet config --- confd/templates/k8s/kubelet.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index 35e5f1a..4f4974b 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -29,4 +29,4 @@ KUBELET_DNS_ARGS="--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" KUBE_CLOUD_PROVIDER="--cloud-provider=external" -KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates AllAlpha=true --root-dir /data/var/lib/kubelet" \ No newline at end of file +KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false --root-dir /data/var/lib/kubelet" \ No newline at end of file From be463dec2cf6a33a8ba7ce8ff0dc5e4c44202568 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 11:09:12 +0800 Subject: [PATCH 043/167] update kubelet config --- confd/templates/k8s/kubelet.tmpl | 6 +++++- systemd/kubelet.service | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index 4f4974b..7dcd435 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -23,10 +23,14 @@ KUBELET_NETWORK_ARGS="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-b KUBELET_SYSTEM_PODS_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --pod-infra-container-image=dockerhub.qingcloud.com/google_containers/pause-amd64:3.0" -KUBELET_KUBECONFIG_ARGS="--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" +KUBELET_KUBECONFIG_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" KUBELET_DNS_ARGS="--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" KUBE_CLOUD_PROVIDER="--cloud-provider=external" +KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt" + +KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" + KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false --root-dir /data/var/lib/kubelet" \ No newline at end of file diff --git a/systemd/kubelet.service b/systemd/kubelet.service index 5046dea..1fa1b57 100644 --- a/systemd/kubelet.service +++ b/systemd/kubelet.service @@ -19,6 +19,8 @@ ExecStart=/usr/bin/kubelet \ $KUBELET_KUBECONFIG_ARGS \ $KUBELET_DNS_ARGS \ $KUBE_CLOUD_PROVIDER \ + $KUBELET_AUTHZ_ARGS \ + $KUBELET_CGROUP_ARGS \ $KUBELET_EXTRA_ARGS \ Restart=on-failure KillMode=process From d929010feb2e15877a3b617b23578d021f8f404d Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 11:47:15 +0800 Subject: [PATCH 044/167] update etcd dir. --- k8s/manifests/etcd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/manifests/etcd.yaml b/k8s/manifests/etcd.yaml index 7bb3aa4..93ffe8d 100644 --- a/k8s/manifests/etcd.yaml +++ b/k8s/manifests/etcd.yaml @@ -43,7 +43,7 @@ spec: path: /etc/ssl/certs name: certs - hostPath: - path: /var/lib/etcd + path: /data/etcd name: etcd - hostPath: path: /etc/kubernetes From 113e482985e0edfa2fd24c6382fef93c6b6ee2e0 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 11:52:33 +0800 Subject: [PATCH 045/167] update kubelet config. --- confd/templates/k8s/kubelet.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index 7dcd435..8e14a78 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -33,4 +33,4 @@ KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernete KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" -KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false --root-dir /data/var/lib/kubelet" \ No newline at end of file +KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir /data/var/lib/kubelet" \ No newline at end of file From 1cfe6845701018532c2e81ac1fc8aa38ff1f1cfd Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 12:04:27 +0800 Subject: [PATCH 046/167] update cloud-controller-manager --- k8s/manifests/kube-cloud-controller-manager.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 95bddfb..25f9fd6 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -16,9 +16,11 @@ spec: - --cloud-provider=qingcloud - --cloud-config=/etc/kubernetes/qingcloud.conf - --kubeconfig=/etc/kubernetes/controller-manager.conf + - --configure-cloud-routes=false + - --allocate-node-cidrs=false - --use-service-account-credentials=true - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --feature-gates=AllAlpha=true +# - --feature-gates=AllAlpha=true image: dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager imagePullPolicy: IfNotPresent livenessProbe: From 68d52c7a1ab4a28d818385afb7968e51ac0681e0 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 13:11:55 +0800 Subject: [PATCH 047/167] update cloud-controller-manager --- k8s/manifests/kube-cloud-controller-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 25f9fd6..34d9c79 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -15,7 +15,7 @@ spec: - --v=${KUBE_LOG_LEVEL} - --cloud-provider=qingcloud - --cloud-config=/etc/kubernetes/qingcloud.conf - - --kubeconfig=/etc/kubernetes/controller-manager.conf + - --kubeconfig=/etc/kubernetes/admin.conf - --configure-cloud-routes=false - --allocate-node-cidrs=false - --use-service-account-credentials=true From 2e3b9cd04dc8b7133c77909f0f0525c5050379cf Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 13:19:26 +0800 Subject: [PATCH 048/167] update dns deployment --- k8s/addons/kube-dns/kube-dns-deployment.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/addons/kube-dns/kube-dns-deployment.yaml b/k8s/addons/kube-dns/kube-dns-deployment.yaml index ce61654..1a6ad75 100644 --- a/k8s/addons/kube-dns/kube-dns-deployment.yaml +++ b/k8s/addons/kube-dns/kube-dns-deployment.yaml @@ -64,7 +64,7 @@ spec: initialDelaySeconds: 3 timeoutSeconds: 5 args: - - --domain={{ .DNSDomain }}. + - --domain=cluster.local. - --dns-port=10053 - --config-dir=/kube-dns-config - --v=2 @@ -105,7 +105,7 @@ spec: - -k - --cache-size=1000 - --log-facility=- - - --server=/{{ .DNSDomain }}/127.0.0.1#10053 + - --server=/cluster.local/127.0.0.1#10053 - --server=/in-addr.arpa/127.0.0.1#10053 - --server=/ip6.arpa/127.0.0.1#10053 ports: @@ -138,8 +138,8 @@ spec: args: - --v=2 - --logtostderr - - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ .DNSDomain }},5,A - - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ .DNSDomain }},5,A + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A ports: - containerPort: 10054 name: metrics From 26d3fca14279a11b901e100f88b7b6a6b0d0aafd Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 13:22:14 +0800 Subject: [PATCH 049/167] update feature-gates config --- k8s/manifests/kube-apiserver.yaml | 3 +-- k8s/manifests/kube-cloud-controller-manager.yaml | 2 +- k8s/manifests/kube-controller-manager.yaml | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index 4ff9697..e5d89b4 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -39,8 +39,7 @@ spec: - --etcd-servers=http://127.0.0.1:2379 - --runtime-config - batch/v2alpha1=true - - --feature-gates - - AllAlpha=true + - --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} imagePullPolicy: IfNotPresent livenessProbe: diff --git a/k8s/manifests/kube-cloud-controller-manager.yaml b/k8s/manifests/kube-cloud-controller-manager.yaml index 34d9c79..53263b1 100644 --- a/k8s/manifests/kube-cloud-controller-manager.yaml +++ b/k8s/manifests/kube-cloud-controller-manager.yaml @@ -20,7 +20,7 @@ spec: - --allocate-node-cidrs=false - --use-service-account-credentials=true - --service-account-private-key-file=/etc/kubernetes/pki/sa.key -# - --feature-gates=AllAlpha=true + - --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false image: dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager imagePullPolicy: IfNotPresent livenessProbe: diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index ee35fc6..fc2be2a 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -25,8 +25,7 @@ spec: - --allocate-node-cidrs=false - --service-cluster-ip-range=10.96.0.0/16 - --cloud-provider=external - - --feature-gates - - AllAlpha=true + - --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} imagePullPolicy: IfNotPresent livenessProbe: From 8e7438413022db40d0245adf06c46dc30205e4d3 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 13:30:18 +0800 Subject: [PATCH 050/167] update to addonmanager.kubernetes.io/mode: Reconcile --- k8s/addons/dashboard/dashboard-deployment.yaml | 2 +- k8s/addons/dashboard/dashboard-service.yaml | 2 +- k8s/addons/kube-dns/kube-dns-cm.yaml | 2 +- k8s/addons/kube-dns/kube-dns-deployment.yaml | 2 +- k8s/addons/kube-dns/kube-dns-sa.yaml | 2 +- k8s/addons/kube-dns/kube-dns-service.yaml | 2 +- k8s/addons/monitor/es-service.yaml | 2 +- k8s/addons/monitor/es-statefulset.yaml | 2 +- k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml | 2 +- k8s/addons/monitor/fluentbit-service.yaml | 2 +- k8s/addons/monitor/heapster-deployment.yaml | 2 +- k8s/addons/monitor/heapster-service.yaml | 2 +- k8s/addons/monitor/kibana-deployment.yaml | 2 +- k8s/addons/monitor/kibana-service.yaml | 2 +- k8s/addons/qingcloud/clean-log-cronjob.yaml | 2 +- k8s/bak/flannel/kube-flannel.yml | 4 ++-- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/k8s/addons/dashboard/dashboard-deployment.yaml b/k8s/addons/dashboard/dashboard-deployment.yaml index cc307a0..d6b6781 100644 --- a/k8s/addons/dashboard/dashboard-deployment.yaml +++ b/k8s/addons/dashboard/dashboard-deployment.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: kubernetes-dashboard kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: diff --git a/k8s/addons/dashboard/dashboard-service.yaml b/k8s/addons/dashboard/dashboard-service.yaml index c01e28a..fbd7139 100644 --- a/k8s/addons/dashboard/dashboard-service.yaml +++ b/k8s/addons/dashboard/dashboard-service.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: kubernetes-dashboard kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: selector: k8s-app: kubernetes-dashboard diff --git a/k8s/addons/kube-dns/kube-dns-cm.yaml b/k8s/addons/kube-dns/kube-dns-cm.yaml index 279e3f8..bae4d6c 100644 --- a/k8s/addons/kube-dns/kube-dns-cm.yaml +++ b/k8s/addons/kube-dns/kube-dns-cm.yaml @@ -4,4 +4,4 @@ metadata: name: kube-dns namespace: kube-system labels: - addonmanager.kubernetes.io/mode: EnsureExists \ No newline at end of file + addonmanager.kubernetes.io/mode: Reconcile \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-deployment.yaml b/k8s/addons/kube-dns/kube-dns-deployment.yaml index 1a6ad75..26c7030 100644 --- a/k8s/addons/kube-dns/kube-dns-deployment.yaml +++ b/k8s/addons/kube-dns/kube-dns-deployment.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: # replicas: not specified here: # 1. In order to make Addon Manager do not reconcile this replicas parameter. diff --git a/k8s/addons/kube-dns/kube-dns-sa.yaml b/k8s/addons/kube-dns/kube-dns-sa.yaml index 4a0640b..48564f2 100644 --- a/k8s/addons/kube-dns/kube-dns-sa.yaml +++ b/k8s/addons/kube-dns/kube-dns-sa.yaml @@ -5,4 +5,4 @@ metadata: namespace: kube-system labels: kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists \ No newline at end of file + addonmanager.kubernetes.io/mode: Reconcile \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-service.yaml b/k8s/addons/kube-dns/kube-dns-service.yaml index 858d439..88d5593 100644 --- a/k8s/addons/kube-dns/kube-dns-service.yaml +++ b/k8s/addons/kube-dns/kube-dns-service.yaml @@ -5,7 +5,7 @@ metadata: k8s-app: kube-dns kubernetes.io/cluster-service: "true" kubernetes.io/name: KubeDNS - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile name: kube-dns namespace: kube-system spec: diff --git a/k8s/addons/monitor/es-service.yaml b/k8s/addons/monitor/es-service.yaml index 669030d..4544651 100644 --- a/k8s/addons/monitor/es-service.yaml +++ b/k8s/addons/monitor/es-service.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: elasticsearch-logging kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "Elasticsearch" spec: ports: diff --git a/k8s/addons/monitor/es-statefulset.yaml b/k8s/addons/monitor/es-statefulset.yaml index 40cbf71..7467b2a 100644 --- a/k8s/addons/monitor/es-statefulset.yaml +++ b/k8s/addons/monitor/es-statefulset.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: elasticsearch-logging kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: serviceName: "elasticsearch-logging" replicas: 2 diff --git a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml index ad12083..a436b35 100644 --- a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml +++ b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: fluent-bit-logging kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: template: metadata: diff --git a/k8s/addons/monitor/fluentbit-service.yaml b/k8s/addons/monitor/fluentbit-service.yaml index 12c4620..e4e1fac 100644 --- a/k8s/addons/monitor/fluentbit-service.yaml +++ b/k8s/addons/monitor/fluentbit-service.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: fluent-bit-logging kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "Fluentbit" spec: ports: diff --git a/k8s/addons/monitor/heapster-deployment.yaml b/k8s/addons/monitor/heapster-deployment.yaml index 173bff3..9f5335d 100644 --- a/k8s/addons/monitor/heapster-deployment.yaml +++ b/k8s/addons/monitor/heapster-deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: k8s-app: heapster - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/cluster-service: "true" spec: replicas: 1 diff --git a/k8s/addons/monitor/heapster-service.yaml b/k8s/addons/monitor/heapster-service.yaml index b85e733..6d6bb77 100644 --- a/k8s/addons/monitor/heapster-service.yaml +++ b/k8s/addons/monitor/heapster-service.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: heapster kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "Heapster" spec: ports: diff --git a/k8s/addons/monitor/kibana-deployment.yaml b/k8s/addons/monitor/kibana-deployment.yaml index 8099a80..dac4f35 100644 --- a/k8s/addons/monitor/kibana-deployment.yaml +++ b/k8s/addons/monitor/kibana-deployment.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: kibana-logging kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: replicas: 1 selector: diff --git a/k8s/addons/monitor/kibana-service.yaml b/k8s/addons/monitor/kibana-service.yaml index 47ea39f..d241ef7 100644 --- a/k8s/addons/monitor/kibana-service.yaml +++ b/k8s/addons/monitor/kibana-service.yaml @@ -6,7 +6,7 @@ metadata: labels: k8s-app: kibana-logging kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "Kibana" spec: ports: diff --git a/k8s/addons/qingcloud/clean-log-cronjob.yaml b/k8s/addons/qingcloud/clean-log-cronjob.yaml index 70085eb..7d5970d 100644 --- a/k8s/addons/qingcloud/clean-log-cronjob.yaml +++ b/k8s/addons/qingcloud/clean-log-cronjob.yaml @@ -4,7 +4,7 @@ metadata: name: clean-log namespace: kube-system labels: - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile spec: schedule: "0 1 * * *" successfulJobsHistoryLimit: 1 diff --git a/k8s/bak/flannel/kube-flannel.yml b/k8s/bak/flannel/kube-flannel.yml index 7576d9a..33dcbcc 100644 --- a/k8s/bak/flannel/kube-flannel.yml +++ b/k8s/bak/flannel/kube-flannel.yml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: flannel labels: - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile --- kind: ConfigMap apiVersion: v1 @@ -13,7 +13,7 @@ metadata: labels: tier: node app: flannel - addonmanager.kubernetes.io/mode: EnsureExists + addonmanager.kubernetes.io/mode: Reconcile data: cni-conf.json: | { From 31a39968b89b82b7a13586f414c969db435e23d4 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 13:50:43 +0800 Subject: [PATCH 051/167] update kubeproxy config. --- k8s/addons/kube-proxy/kube-proxy.yaml | 18 +++++++++--------- k8s/manifests/kube-controller-manager.yaml | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/k8s/addons/kube-proxy/kube-proxy.yaml b/k8s/addons/kube-proxy/kube-proxy.yaml index 600764c..713119a 100644 --- a/k8s/addons/kube-proxy/kube-proxy.yaml +++ b/k8s/addons/kube-proxy/kube-proxy.yaml @@ -3,6 +3,7 @@ kind: DaemonSet metadata: labels: k8s-app: kube-proxy + addonmanager.kubernetes.io/mode: Reconcile name: kube-proxy namespace: kube-system spec: @@ -20,20 +21,19 @@ spec: spec: containers: - name: kube-proxy - image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:v1.7.3 + image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} imagePullPolicy: IfNotPresent command: - - /usr/local/bin/kube-proxy + - kube-proxy - --v=${KUBE_LOG_LEVEL} - --masquerade-all - - --feature-gates - - AllAlpha=true + - --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf securityContext: privileged: true volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy + - mountPath: /var/lib/kube-proxy/kubeconfig.conf + name: kube-proxy-config # TODO: Make this a file hostpath mount - mountPath: /run/xtables.lock name: xtables-lock @@ -41,9 +41,9 @@ spec: hostNetwork: true serviceAccountName: kube-proxy volumes: - - name: kube-proxy - configMap: - name: kube-proxy + - name: kube-proxy-config + hostPath: + path: /etc/kubernetes/kubelet.conf - name: xtables-lock hostPath: path: /run/xtables.lock diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index fc2be2a..4568bb8 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -49,9 +49,9 @@ spec: name: certs hostNetwork: true volumes: - - hostPath: + - name: k8s + hostPath: path: /etc/kubernetes - name: k8s - - hostPath: - path: /etc/ssl/certs - name: certs \ No newline at end of file + - name: certs + hostPath: + path: /etc/ssl/certs \ No newline at end of file From 8fb46820ba600033c9fd042073a1535248724be7 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 14:41:16 +0800 Subject: [PATCH 052/167] update kubeproxy --- k8s/addons/kube-proxy/kube-proxy-cm.yaml | 27 ++++++++++++++++++++++++ k8s/addons/kube-proxy/kube-proxy-sa.yaml | 8 +++++++ k8s/addons/kube-proxy/kube-proxy.yaml | 10 ++++----- 3 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 k8s/addons/kube-proxy/kube-proxy-cm.yaml create mode 100644 k8s/addons/kube-proxy/kube-proxy-sa.yaml diff --git a/k8s/addons/kube-proxy/kube-proxy-cm.yaml b/k8s/addons/kube-proxy/kube-proxy-cm.yaml new file mode 100644 index 0000000..a8d32f9 --- /dev/null +++ b/k8s/addons/kube-proxy/kube-proxy-cm.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-proxy + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${MASTER_IP}:6443 + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token \ No newline at end of file diff --git a/k8s/addons/kube-proxy/kube-proxy-sa.yaml b/k8s/addons/kube-proxy/kube-proxy-sa.yaml new file mode 100644 index 0000000..1b866c9 --- /dev/null +++ b/k8s/addons/kube-proxy/kube-proxy-sa.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-proxy + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile \ No newline at end of file diff --git a/k8s/addons/kube-proxy/kube-proxy.yaml b/k8s/addons/kube-proxy/kube-proxy.yaml index 713119a..fa117fc 100644 --- a/k8s/addons/kube-proxy/kube-proxy.yaml +++ b/k8s/addons/kube-proxy/kube-proxy.yaml @@ -32,8 +32,8 @@ spec: securityContext: privileged: true volumeMounts: - - mountPath: /var/lib/kube-proxy/kubeconfig.conf - name: kube-proxy-config + - mountPath: /var/lib/kube-proxy + name: kube-proxy # TODO: Make this a file hostpath mount - mountPath: /run/xtables.lock name: xtables-lock @@ -41,9 +41,9 @@ spec: hostNetwork: true serviceAccountName: kube-proxy volumes: - - name: kube-proxy-config - hostPath: - path: /etc/kubernetes/kubelet.conf + - name: kube-proxy + configMap: + name: kube-proxy - name: xtables-lock hostPath: path: /run/xtables.lock From eb8b97364bf76c16b553bd1adbec98ccae1ebe9a Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 14:45:08 +0800 Subject: [PATCH 053/167] update script. --- script/common.sh | 1 + script/init-master.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/common.sh b/script/common.sh index f4d15ad..a1024c7 100755 --- a/script/common.sh +++ b/script/common.sh @@ -78,6 +78,7 @@ function replace_vars(){ sed 's/${HYPERKUBE_VERSION}/'"${HYPERKUBE_VERSION}"'/g' ${from} > ${tmpfile} sed -i 's/${KUBE_LOG_LEVEL}/'"${ENV_KUBE_LOG_LEVEL}"'/g' ${tmpfile} sed -i 's/${HOST_IP}/'"${HOST_IP}"'/g' ${tmpfile} + sed -i 's/${MASTER_IP}/'"${MASTER_IP}"'/g' ${tmpfile} if [ "${to}" == "/data/kubernetes/addons/monitor/es-controller.yaml" ] then diff --git a/script/init-master.sh b/script/init-master.sh index 98c30c4..2a49c4c 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -10,11 +10,11 @@ link_dynamic_dir init_token=$(get_or_gen_init_token) #retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --cert-altnames ${ENV_API_EXTERNAL_DOMAIN} -kubeadm alpha phase kubeconfig client-certs --client-name system:node:${HOST_INSTANCE_ID} --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf +kubeadm alpha phase kubeconfig client-certs --client-name system:node:${HOST_INSTANCE_ID} --organization system:nodes --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf kubeadm alpha phase kubeconfig client-certs --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf kubeadm alpha phase kubeconfig client-certs --client-name system:cloud-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/cloud-controller-manager.conf kubeadm alpha phase kubeconfig client-certs --client-name system:kube-scheduler --server https://${MASTER_IP}:6443 > /etc/kubernetes/scheduler.conf -kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --server https://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf +kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --organization system:masters --server https://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf docker_login process_manifests From 03d522ee97c538e9470fd0cbf1e49341c2fd4038 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 15:05:55 +0800 Subject: [PATCH 054/167] update kubeproxy --- .../kube-proxy/kube-proxy-clusterrolebinding.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 k8s/addons/kube-proxy/kube-proxy-clusterrolebinding.yaml diff --git a/k8s/addons/kube-proxy/kube-proxy-clusterrolebinding.yaml b/k8s/addons/kube-proxy/kube-proxy-clusterrolebinding.yaml new file mode 100644 index 0000000..2707a78 --- /dev/null +++ b/k8s/addons/kube-proxy/kube-proxy-clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: system:kube-proxy + labels: + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier +subjects: +- kind: ServiceAccount + name: kube-proxy + namespace: kube-system \ No newline at end of file From 39ba9efda0b2b077186a10ee00851370b930e97d Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 16:48:20 +0800 Subject: [PATCH 055/167] add update sshd config --- image/build-image.sh | 1 + image/update-sshd-config.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 image/update-sshd-config.sh diff --git a/image/build-image.sh b/image/build-image.sh index 729c2fa..4cfc24b 100755 --- a/image/build-image.sh +++ b/image/build-image.sh @@ -11,6 +11,7 @@ ${K8S_HOME}/image/install-pkg.sh ${K8S_HOME}/image/install-qingcloud-agent.sh ${K8S_HOME}/image/update-confd.sh ${K8S_HOME}/image/update-sysconfig.sh +${K8S_HOME}/image/update-sshd-config.sh ${K8S_HOME}/image/update-k8s-bin.sh ${K8S_HOME}/image/update-cni.sh ${K8S_HOME}/image/update-logrotate.sh diff --git a/image/update-sshd-config.sh b/image/update-sshd-config.sh new file mode 100644 index 0000000..df18887 --- /dev/null +++ b/image/update-sshd-config.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +sed -i 's/TCPKeepAlive yes/TCPKeepAlive no/g' /etc/ssh/sshd_config +if ! grep "ClientAliveInterval" /etc/ssh/sshd_config >/dev/null +then + echo "ClientAliveInterval 120" >> /etc/ssh/sshd_config +fi +if ! grep "ClientAliveCountMax" /etc/ssh/sshd_config >/dev/null +then + echo "ClientAliveCountMax 720" >> /etc/ssh/sshd_config +fi + +systemctl restart ssh \ No newline at end of file From 81cc4184a0424a9a6a5e207f80694fd37f7de5e4 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 16:48:38 +0800 Subject: [PATCH 056/167] update init client. --- script/init-client.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/init-client.sh b/script/init-client.sh index cb2d4d6..d6bdd58 100755 --- a/script/init-client.sh +++ b/script/init-client.sh @@ -4,6 +4,7 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") source "${K8S_HOME}/script/common.sh" +echo "root:k8s" |chpasswd sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config systemctl restart ssh From 757da8e10dba1903dfb60468fb1c26524576da90 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 16:51:04 +0800 Subject: [PATCH 057/167] chmod +x to update-sshd-config --- image/update-sshd-config.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 image/update-sshd-config.sh diff --git a/image/update-sshd-config.sh b/image/update-sshd-config.sh old mode 100644 new mode 100755 From 62d5290a7bcdde3492985ae348fb3d225da426d0 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 17:45:48 +0800 Subject: [PATCH 058/167] update script. --- script/common.sh | 6 +++++- script/init-master.sh | 26 ++++++++++++++++---------- script/reset.sh | 16 ++++++---------- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/script/common.sh b/script/common.sh index a1024c7..dba7881 100755 --- a/script/common.sh +++ b/script/common.sh @@ -107,15 +107,19 @@ function update_k8s_manifests(){ mkdir /data/kubernetes/manifests/ || rm -rf /data/kubernetes/manifests/* mkdir /data/kubernetes/addons/ || rm -rf /data/kubernetes/addons/* process_manifests + process_addons } function process_manifests(){ mkdir -p /data/kubernetes/manifests/ - mkdir -p /data/kubernetes/addons/ for f in ${K8S_HOME}/k8s/manifests/*; do name=$(basename ${f}) replace_vars ${f} /data/kubernetes/manifests/${name} done +} + +function process_addons(){ + mkdir -p /data/kubernetes/addons/ for addon in ${K8S_HOME}/k8s/addons/*; do addon_name=$(basename $addon) diff --git a/script/init-master.sh b/script/init-master.sh index 2a49c4c..5d2398a 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -7,21 +7,27 @@ source "${K8S_HOME}/script/common.sh" ensure_dir link_dynamic_dir +systemctl start docker +docker_login + init_token=$(get_or_gen_init_token) #retry kubeadm check --cloud-provider-name qingcloud --cloud-provider-config /etc/kubernetes/qingcloud.conf -kubeadm alpha phase certs selfsign --apiserver-advertise-address ${HOST_IP} --cert-altnames ${ENV_API_EXTERNAL_DOMAIN} -kubeadm alpha phase kubeconfig client-certs --client-name system:node:${HOST_INSTANCE_ID} --organization system:nodes --server https://${MASTER_IP}:6443 > /etc/kubernetes/kubelet.conf -kubeadm alpha phase kubeconfig client-certs --client-name system:kube-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/controller-manager.conf -kubeadm alpha phase kubeconfig client-certs --client-name system:cloud-controller-manager --server https://${MASTER_IP}:6443 > /etc/kubernetes/cloud-controller-manager.conf -kubeadm alpha phase kubeconfig client-certs --client-name system:kube-scheduler --server https://${MASTER_IP}:6443 > /etc/kubernetes/scheduler.conf -kubeadm alpha phase kubeconfig client-certs --client-name kubernetes-admin --organization system:masters --server https://${MASTER_IP}:6443 > /etc/kubernetes/admin.conf -docker_login +kubeadm alpha phase certs all --apiserver-advertise-address ${MASTER_IP} --apiserver-cert-extra-sans ${ENV_API_EXTERNAL_DOMAIN} --service-cidr 10.96.0.0/16 --service-dns-domain cluster.local +kubeadm alpha phase kubeconfig all --apiserver-advertise-address ${MASTER_IP} process_manifests -systemctl start docker systemctl start kubelet wait_kubelet wait_apiserver -train_master -retry kubeadm token create ${init_token} --token-ttl 0 + +kubeadm alpha phase mark-master ${MASTER_INSTANCE_ID} +retry kubeadm token create ${init_token} --token-ttl 0 --kubeconfig /etc/kubernetes/admin.conf +retry kubeadm alpha phase bootstrap-token node allow-post-csrs --kubeconfig /etc/kubernetes/admin.conf +retry kubeadm alpha phase bootstrap-token node allow-auto-approve --kubeconfig /etc/kubernetes/admin.conf +retry kubeadm alpha phase bootstrap-token cluster-info /etc/kubernetes/admin.conf --kubeconfig /etc/kubernetes/admin.conf +retry kubeadm alpha phase upload-config --kubeconfig /etc/kubernetes/admin.conf +retry kubeadm alpha phase apiconfig --kubeconfig /etc/kubernetes/admin.conf +process_addons + +#train_master diff --git a/script/reset.sh b/script/reset.sh index 5fbb41d..e79f1ed 100755 --- a/script/reset.sh +++ b/script/reset.sh @@ -7,28 +7,24 @@ source "${K8S_HOME}/script/common.sh" systemctl stop kubelet -echo "Unmounting directories in /var/lib/kubelet..." -cat /proc/mounts | awk '{print $2}' | grep '/var/lib/kubelet' | xargs umount +echo "Unmounting directories in /data/var/lib/kubelet..." +cat /proc/mounts | awk '{print $2}' | grep '/data/var/lib/kubelet' | xargs umount echo "clean config" unlink /etc/kubernetes -rm /data/kubernetes/kubelet.conf -rm /data/kubernetes/admin.conf -rm /data/kubernetes/init_token rm ${NODE_INIT_LOCK} -rm -rf /data/kubernetes/pki -rm -rf /data/kubernetes/manifests -rm -rf /data/kubernetes/addons +rm -rf /data/kubernetes/* echo "stop all container" docker ps -a | grep 'k8s_' | awk '{print $1}' | xargs docker rm --force --volumes rm -rf /data/etcd -rm -rf /var/lib/kubelet/* +rm -rf /data/var/lib/kubelet/* echo "flush iptables" iptables --flush -t nat iptables --flush -rm -rf /etc/cni +rm -rf /etc/cni +systemctl restart confd From 49370f174f6cc59000b778696a9bbbe3e042cb5e Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 17:51:19 +0800 Subject: [PATCH 059/167] update reset.sh --- script/reset.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/reset.sh b/script/reset.sh index e79f1ed..697507c 100755 --- a/script/reset.sh +++ b/script/reset.sh @@ -13,7 +13,10 @@ cat /proc/mounts | awk '{print $2}' | grep '/data/var/lib/kubelet' | xargs umoun echo "clean config" unlink /etc/kubernetes rm ${NODE_INIT_LOCK} -rm -rf /data/kubernetes/* +rm -f /data/kubernetes/*.conf +rm -rf /data/kubernetes/addons +rm -rf /data/kubernetes/manifests +rm -rf /data/kubernetes/pki echo "stop all container" docker ps -a | grep 'k8s_' | awk '{print $1}' | xargs docker rm --force --volumes From cbd777a0ad47ec65641cfde425fd7bc94e2470b9 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 17:57:02 +0800 Subject: [PATCH 060/167] fix init master script. --- script/init-master.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/init-master.sh b/script/init-master.sh index 5d2398a..fd2599a 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -22,7 +22,7 @@ wait_kubelet wait_apiserver kubeadm alpha phase mark-master ${MASTER_INSTANCE_ID} -retry kubeadm token create ${init_token} --token-ttl 0 --kubeconfig /etc/kubernetes/admin.conf +retry kubeadm token create ${init_token} --ttl 0 --description "the default kubeadm init token" --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token node allow-post-csrs --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token node allow-auto-approve --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token cluster-info /etc/kubernetes/admin.conf --kubeconfig /etc/kubernetes/admin.conf From 1959faeb0fea4099b3afdf10ad6c4eaefc309f12 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 18:04:02 +0800 Subject: [PATCH 061/167] update addons tolerations --- k8s/addons/dashboard/dashboard-deployment.yaml | 3 +-- k8s/addons/kube-dns/kube-dns-deployment.yaml | 3 +-- k8s/addons/kube-proxy/kube-proxy.yaml | 2 ++ k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/k8s/addons/dashboard/dashboard-deployment.yaml b/k8s/addons/dashboard/dashboard-deployment.yaml index d6b6781..f9283d3 100644 --- a/k8s/addons/dashboard/dashboard-deployment.yaml +++ b/k8s/addons/dashboard/dashboard-deployment.yaml @@ -41,8 +41,7 @@ spec: tolerations: - key: "CriticalAddonsOnly" operator: "Exists" - - key: "dedicated" - value: "master" + - key: "node-role.kubernetes.io/master" effect: NoSchedule nodeSelector: role: master \ No newline at end of file diff --git a/k8s/addons/kube-dns/kube-dns-deployment.yaml b/k8s/addons/kube-dns/kube-dns-deployment.yaml index 26c7030..96879ac 100644 --- a/k8s/addons/kube-dns/kube-dns-deployment.yaml +++ b/k8s/addons/kube-dns/kube-dns-deployment.yaml @@ -153,8 +153,7 @@ spec: tolerations: - key: "CriticalAddonsOnly" operator: Exists - - key: "dedicated" - value: "master" + - key: "node-role.kubernetes.io/master" effect: NoSchedule nodeSelector: role: master \ No newline at end of file diff --git a/k8s/addons/kube-proxy/kube-proxy.yaml b/k8s/addons/kube-proxy/kube-proxy.yaml index fa117fc..c86561f 100644 --- a/k8s/addons/kube-proxy/kube-proxy.yaml +++ b/k8s/addons/kube-proxy/kube-proxy.yaml @@ -52,6 +52,8 @@ spec: operator: "Exists" - key: "dedicated" operator: "Exists" + - key: "node-role.kubernetes.io/master" + effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized effect: NoSchedule value: "true" \ No newline at end of file diff --git a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml index a436b35..692d313 100644 --- a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml +++ b/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml @@ -54,6 +54,8 @@ spec: operator: "Exists" - key: "dedicated" operator: "Exists" + - key: "node-role.kubernetes.io/master" + effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized effect: NoSchedule value: "true" From 3fd7d2ef53ee77a05e59313266468ad284a4b19d Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 18:30:12 +0800 Subject: [PATCH 062/167] fix init-master --- script/init-master.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/init-master.sh b/script/init-master.sh index fd2599a..6b39044 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -26,7 +26,7 @@ retry kubeadm token create ${init_token} --ttl 0 --description "the default kube retry kubeadm alpha phase bootstrap-token node allow-post-csrs --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token node allow-auto-approve --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token cluster-info /etc/kubernetes/admin.conf --kubeconfig /etc/kubernetes/admin.conf -retry kubeadm alpha phase upload-config --kubeconfig /etc/kubernetes/admin.conf +#retry kubeadm alpha phase upload-config --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase apiconfig --kubeconfig /etc/kubernetes/admin.conf process_addons From 6f608cfd4067ae085f5e1b15c3bdcfd8a382c361 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 10 Aug 2017 19:18:04 +0800 Subject: [PATCH 063/167] update init-client --- script/init-client.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/init-client.sh b/script/init-client.sh index d6bdd58..6a3a82d 100755 --- a/script/init-client.sh +++ b/script/init-client.sh @@ -10,5 +10,3 @@ systemctl restart ssh ensure_dir join_node - -ln -fs /etc/kubernetes/kubelet.conf /root/.kube/config \ No newline at end of file From bd9bde87e3fe9c9e00154444c497ab8282d7c1b2 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 09:38:44 +0800 Subject: [PATCH 064/167] update kubelet config. --- confd/templates/k8s/kubelet.tmpl | 4 ++-- script/common.sh | 4 ++++ script/reset.sh | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index 8e14a78..de833ef 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -17,7 +17,7 @@ KUBELET_ADDRESS="--address=0.0.0.0" KUBELET_PORT="--port=10250" # You may leave this blank to use the actual hostname -KUBELET_HOSTNAME="--node-ip={{getv "/host/ip"}}" +KUBELET_HOSTNAME="--hostname-override={{getv "/host/instance_id"}} --node-ip={{getv "/host/ip"}} --provider-id={{getv "/host/instance_id"}}" KUBELET_NETWORK_ARGS="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" @@ -33,4 +33,4 @@ KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernete KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" -KUBELET_EXTRA_ARGS="--node-labels role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir /data/var/lib/kubelet" \ No newline at end of file +KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes" \ No newline at end of file diff --git a/script/common.sh b/script/common.sh index dba7881..b79e2a1 100755 --- a/script/common.sh +++ b/script/common.sh @@ -274,6 +274,10 @@ function link_dynamic_dir(){ then mkdir /data/var/lib/kubelet && ln -s /data/var/lib/kubelet /var/lib/kubelet fi + if [ ! -d "/data/var/run/kubernetes" ] + then + mkdir /data/var/run/kubernetes && ln -s /data/var/run/kubernetes /var/run/kubernetes + fi ln -fs /root/.docker /data/var/lib/kubelet/.docker } diff --git a/script/reset.sh b/script/reset.sh index 697507c..dc1ac3b 100755 --- a/script/reset.sh +++ b/script/reset.sh @@ -8,11 +8,11 @@ source "${K8S_HOME}/script/common.sh" systemctl stop kubelet echo "Unmounting directories in /data/var/lib/kubelet..." -cat /proc/mounts | awk '{print $2}' | grep '/data/var/lib/kubelet' | xargs umount +cat /proc/mounts | awk '{print $2}' | grep '/data/var/lib/kubelet' | xargs -d '\n' -r umount echo "clean config" unlink /etc/kubernetes -rm ${NODE_INIT_LOCK} +rm -f ${NODE_INIT_LOCK} rm -f /data/kubernetes/*.conf rm -rf /data/kubernetes/addons rm -rf /data/kubernetes/manifests From 9017c08e191677e4900c26e816e587110a291820 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 12:03:34 +0800 Subject: [PATCH 065/167] add adminconf to metadata --- script/get-metadata.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-metadata.sh b/script/get-metadata.sh index 7cda5e0..73085e1 100755 --- a/script/get-metadata.sh +++ b/script/get-metadata.sh @@ -7,4 +7,5 @@ source "${K8S_HOME}/script/common.sh" ensure_dir init_token=$(get_or_gen_init_token) -echo '{"init_token":"'${init_token}'"}' +adminconf=$(cat "/etc/kubernetes/admin.conf") +echo '{"init_token":"'${init_token}'", "adminconf":'${adminconf}'}' From dd10947dd80f060cb3e3d2cfa68dadcb9e746ca9 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 12:04:51 +0800 Subject: [PATCH 066/167] update metadata --- script/get-metadata.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-metadata.sh b/script/get-metadata.sh index 73085e1..0d7b5dd 100755 --- a/script/get-metadata.sh +++ b/script/get-metadata.sh @@ -8,4 +8,4 @@ ensure_dir init_token=$(get_or_gen_init_token) adminconf=$(cat "/etc/kubernetes/admin.conf") -echo '{"init_token":"'${init_token}'", "adminconf":'${adminconf}'}' +echo '{"init_token":"'${init_token}'", "adminconf":"'${adminconf}'"}' From 99f050a703ac95e162c55352de22d47638960524 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 12:09:25 +0800 Subject: [PATCH 067/167] update get metadata --- script/get-metadata.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-metadata.sh b/script/get-metadata.sh index 0d7b5dd..a12b5e3 100755 --- a/script/get-metadata.sh +++ b/script/get-metadata.sh @@ -7,5 +7,5 @@ source "${K8S_HOME}/script/common.sh" ensure_dir init_token=$(get_or_gen_init_token) -adminconf=$(cat "/etc/kubernetes/admin.conf") +adminconf=$(cat "/etc/kubernetes/admin.conf"|base64 -w 0) echo '{"init_token":"'${init_token}'", "adminconf":"'${adminconf}'"}' From 550f51dc594ff739cb95779f920fc4694f95de57 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 12:28:29 +0800 Subject: [PATCH 068/167] update install script. --- image/install-pkg.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/image/install-pkg.sh b/image/install-pkg.sh index ea8c200..d55549a 100755 --- a/image/install-pkg.sh +++ b/image/install-pkg.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash apt-get update + +apt-get install -y ebtables socat +apt-get install -y jq apt-transport-https bash-completion + apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D apt-add-repository 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' apt-get update apt-cache policy docker-engine apt-get install -y docker-engine -apt-get install -y ebtables socat -apt-get install -y jq apt-transport-https bash-completion apt-get remove network-manager apt-get upgrade -y From ed89ec1356851879446f5280804d98b5c7a412ac Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 16:29:42 +0800 Subject: [PATCH 069/167] update es image. --- docker/elasticsearch/build.sh | 6 +++--- k8s/addons/monitor/es-statefulset.yaml | 2 +- script/common.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/elasticsearch/build.sh b/docker/elasticsearch/build.sh index d0d98d2..14bb59d 100644 --- a/docker/elasticsearch/build.sh +++ b/docker/elasticsearch/build.sh @@ -1,5 +1,5 @@ #!/bin/sh -docker pull gcr.io/google_containers/elasticsearch:v5.5.1 -docker tag gcr.io/google_containers/elasticsearch:v5.5.1 dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 -docker push dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 +docker pull gcr.io/google-containers/elasticsearch:v5.5.1-1 +docker tag gcr.io/google-containers/elasticsearch:v5.5.1-1 dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1-1 +docker push dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1-1 diff --git a/k8s/addons/monitor/es-statefulset.yaml b/k8s/addons/monitor/es-statefulset.yaml index 7467b2a..15d0c07 100644 --- a/k8s/addons/monitor/es-statefulset.yaml +++ b/k8s/addons/monitor/es-statefulset.yaml @@ -28,7 +28,7 @@ spec: securityContext: privileged: true containers: - - image: dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 + - image: dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1-1 imagePullPolicy: IfNotPresent name: elasticsearch-logging resources: diff --git a/script/common.sh b/script/common.sh index b79e2a1..e7b8604 100755 --- a/script/common.sh +++ b/script/common.sh @@ -276,7 +276,7 @@ function link_dynamic_dir(){ fi if [ ! -d "/data/var/run/kubernetes" ] then - mkdir /data/var/run/kubernetes && ln -s /data/var/run/kubernetes /var/run/kubernetes + mkdir -p /data/var/run/kubernetes && ln -s /data/var/run/kubernetes /var/run/kubernetes fi ln -fs /root/.docker /data/var/lib/kubelet/.docker } From f2300dd56fd0fe1c4c8de0183db4329599f48f73 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 16:49:11 +0800 Subject: [PATCH 070/167] update reload-env. --- script/reload-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/reload-env.sh b/script/reload-env.sh index 3718236..fb678e5 100755 --- a/script/reload-env.sh +++ b/script/reload-env.sh @@ -8,5 +8,6 @@ source "${K8S_HOME}/script/common.sh" if [ "${HOST_ROLE}" == "master" ] && [ -d "/data/kubernetes/manifests/" ] then process_manifests +process_addons fi docker_login \ No newline at end of file From 4feb9bc897c8f7a1d8215f6954eb3ca0d29cfd8d Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:04:33 +0800 Subject: [PATCH 071/167] update kubeconfig confd --- confd/conf.d/k8s/kubeconfig.toml | 6 ++++++ confd/templates/k8s/kubeconfig.tmpl | 1 + image/install-qingcloud-agent.sh | 6 ++++++ 3 files changed, 13 insertions(+) create mode 100644 confd/conf.d/k8s/kubeconfig.toml create mode 100644 confd/templates/k8s/kubeconfig.tmpl diff --git a/confd/conf.d/k8s/kubeconfig.toml b/confd/conf.d/k8s/kubeconfig.toml new file mode 100644 index 0000000..cd7fc2c --- /dev/null +++ b/confd/conf.d/k8s/kubeconfig.toml @@ -0,0 +1,6 @@ +[template] +src = "k8s/kubeconfig.tmpl" +dest = "/root/.kube/config" +keys = [ + "/", +] diff --git a/confd/templates/k8s/kubeconfig.tmpl b/confd/templates/k8s/kubeconfig.tmpl new file mode 100644 index 0000000..c708841 --- /dev/null +++ b/confd/templates/k8s/kubeconfig.tmpl @@ -0,0 +1 @@ +{{range gets "/hosts/master/*/token"}}{{json .Value|base64Decode}}{{end}} diff --git a/image/install-qingcloud-agent.sh b/image/install-qingcloud-agent.sh index e36d4cb..eb75f54 100755 --- a/image/install-qingcloud-agent.sh +++ b/image/install-qingcloud-agent.sh @@ -12,5 +12,11 @@ cd /tmp rm -rf app-agent-linux-amd64/ rm app-agent-linux-amd64.tar.gz +echo "upgrade confd" +wget https://github.com/yunify/confd/releases/download/v0.13.10/confd-linux-amd64.tar.gz +tar -O -zxf confd-linux-amd64.tar.gz >/opt/qingcloud/app-agent/bin/confd +chmod +x /opt/qingcloud/app-agent/bin/confd +rm confd-linux-amd64.tar.gz + systemctl enable confd systemctl disable confd \ No newline at end of file From 297124d397796494901573296134c821208deeef Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:09:02 +0800 Subject: [PATCH 072/167] update confd config. --- confd/templates/k8s/interfaces.tmpl | 2 +- confd/templates/k8s/kubeconfig.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/confd/templates/k8s/interfaces.tmpl b/confd/templates/k8s/interfaces.tmpl index 3de12b0..078a7b7 100644 --- a/confd/templates/k8s/interfaces.tmpl +++ b/confd/templates/k8s/interfaces.tmpl @@ -1,4 +1,4 @@ -{{$role := getv "host/role"}}{{if eq $role "client"}} +{{$role := getv "/host/role"}}{{if eq $role "client"}} auto lo iface lo inet loopback {{$masters := getvs "/hosts/master/*/ip"}} diff --git a/confd/templates/k8s/kubeconfig.tmpl b/confd/templates/k8s/kubeconfig.tmpl index c708841..76ce38e 100644 --- a/confd/templates/k8s/kubeconfig.tmpl +++ b/confd/templates/k8s/kubeconfig.tmpl @@ -1 +1 @@ -{{range gets "/hosts/master/*/token"}}{{json .Value|base64Decode}}{{end}} +{{range gets "/hosts/master/*/token"}}{{$data := json .Value}}{{base64Decode $data.adminconf}}{{end}} From b84af7d3a86bb20569c1a8fe10b5a0b022f43d73 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:13:09 +0800 Subject: [PATCH 073/167] update interface.tmpl --- confd/templates/k8s/interfaces.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/confd/templates/k8s/interfaces.tmpl b/confd/templates/k8s/interfaces.tmpl index 078a7b7..c58ab3e 100644 --- a/confd/templates/k8s/interfaces.tmpl +++ b/confd/templates/k8s/interfaces.tmpl @@ -1,5 +1,4 @@ -{{$role := getv "/host/role"}}{{if eq $role "client"}} -auto lo +{{$role := getv "/host/role"}}{{if eq $role "client"}}auto lo iface lo inet loopback {{$masters := getvs "/hosts/master/*/ip"}} allow-hotplug eth0 From 406563cf6fc52d6a8926c13f1adfa5110b2332e8 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:16:27 +0800 Subject: [PATCH 074/167] update service ip range. --- k8s/manifests/kube-apiserver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/manifests/kube-apiserver.yaml b/k8s/manifests/kube-apiserver.yaml index e5d89b4..b82ded4 100644 --- a/k8s/manifests/kube-apiserver.yaml +++ b/k8s/manifests/kube-apiserver.yaml @@ -15,7 +15,7 @@ spec: - --v=${KUBE_LOG_LEVEL} - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-allowed-names=front-proxy-client - - --service-cluster-ip-range=10.96.0.0/12 + - --service-cluster-ip-range=10.96.0.0/16 - --secure-port=6443 - --insecure-port=8080 - --insecure-bind-address=127.0.0.1 From 16c8d833cd8e1f0129177a745b22e58ef29628d4 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:37:16 +0800 Subject: [PATCH 075/167] update interfaces tmpl --- confd/templates/k8s/interfaces.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/confd/templates/k8s/interfaces.tmpl b/confd/templates/k8s/interfaces.tmpl index c58ab3e..74687ab 100644 --- a/confd/templates/k8s/interfaces.tmpl +++ b/confd/templates/k8s/interfaces.tmpl @@ -1,13 +1,13 @@ -{{$role := getv "/host/role"}}{{if eq $role "client"}}auto lo +{{$role := getv "/host/role" -}}{{if eq $role "client" -}}auto lo iface lo inet loopback -{{$masters := getvs "/hosts/master/*/ip"}} +{{$masters := getvs "/hosts/master/*/ip" -}} allow-hotplug eth0 iface eth0 inet dhcp dns-search default.svc.cluster.local svc.cluster.local cluster.local dns-nameservers 10.96.0.10 - up route add -net 10.96.0.0 netmask 255.255.0.0 gw {{index $masters 0}}{{else}} + up route add -net 10.96.0.0 netmask 255.255.0.0 gw {{index $masters 0 -}}{{else}} auto lo iface lo inet loopback allow-hotplug eth0 -iface eth0 inet dhcp{{end}} \ No newline at end of file +iface eth0 inet dhcp{{end -}} \ No newline at end of file From ed4fa0bc2d2569877e7b992d674b86c15abef6ed Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:44:48 +0800 Subject: [PATCH 076/167] fix interface config new line. --- confd/templates/k8s/interfaces.tmpl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/confd/templates/k8s/interfaces.tmpl b/confd/templates/k8s/interfaces.tmpl index 74687ab..6d7fe73 100644 --- a/confd/templates/k8s/interfaces.tmpl +++ b/confd/templates/k8s/interfaces.tmpl @@ -1,13 +1,17 @@ -{{$role := getv "/host/role" -}}{{if eq $role "client" -}}auto lo +{{$role := getv "/host/role" -}} +{{if eq $role "client" -}} +auto lo iface lo inet loopback {{$masters := getvs "/hosts/master/*/ip" -}} allow-hotplug eth0 iface eth0 inet dhcp dns-search default.svc.cluster.local svc.cluster.local cluster.local dns-nameservers 10.96.0.10 - up route add -net 10.96.0.0 netmask 255.255.0.0 gw {{index $masters 0 -}}{{else}} + up route add -net 10.96.0.0 netmask 255.255.0.0 gw {{index $masters 0 -}} +{{- else -}} auto lo iface lo inet loopback allow-hotplug eth0 -iface eth0 inet dhcp{{end -}} \ No newline at end of file +iface eth0 inet dhcp +{{end -}} \ No newline at end of file From 08cead6d6535238b810e69f2ec69f0dd6989a79c Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 11 Aug 2017 18:45:06 +0800 Subject: [PATCH 077/167] update app config. --- app/cluster.json.mustache | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index d33f271..acf376f 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -2,15 +2,12 @@ "name": {{cluster.name}}, "description": {{cluster.description}}, "vxnet": {{cluster.vxnet}}, - "upgrade_policy": [ - "appv-i6bzyuxw" - ], "nodes": [ { "role": "master", "container": { "type": "kvm", - "image": "img-03nuflem", + "image": "img-tacg33kl", "zone": "pek3a" }, "instance_class": {{cluster.master.instance_class}}, @@ -44,9 +41,6 @@ "nodes_to_execute_on": 1, "post_stop_service": true, "cmd": "/opt/kubernetes/script/destroy-master.sh" - }, - "upgrade": { - "cmd": "/opt/kubernetes/script/upgrade-master.sh" } }, "custom_metadata": { @@ -90,7 +84,7 @@ "role": "node", "container": { "type": "kvm", - "image": "img-03nuflem", + "image": "img-tacg33kl", "zone": "pek3a" }, "instance_class": {{cluster.node.instance_class}}, @@ -131,9 +125,6 @@ "post_stop_service": false, "cmd": "/opt/kubernetes/script/destroy-node.sh", "timeout": 86400 - }, - "upgrade": { - "cmd": "/opt/kubernetes/script/upgrade-node.sh" } }, "health_check": { @@ -175,7 +166,7 @@ "role": "log", "container": { "type": "kvm", - "image": "img-03nuflem", + "image": "img-tacg33kl", "zone": "pek3a" }, "instance_class": {{cluster.log.instance_class}}, @@ -256,7 +247,7 @@ "role": "client", "container": { "type": "kvm", - "image": "img-8zk1b5oi", + "image": "img-tacg33kl", "zone": "pek3a" }, "instance_class": {{cluster.client.instance_class}}, @@ -267,7 +258,7 @@ "services": { "start": { "order": 2, - "cmd": "/usr/bin/init_config.sh" + "cmd": "/opt/kubernetes/script/init-client.sh" } } } From b3f66bd2a90c4815b4592e373497887f792f1a72 Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 17:31:32 +0800 Subject: [PATCH 078/167] add qingcloud-volume-provisioner config --- confd/templates/k8s/kubelet.tmpl | 2 +- image/update-k8s-bin.sh | 13 ++++-- .../qingcloud/qingcloud-storage-class.yaml | 9 ++++ .../qingcloud-volume-provisioner.yaml | 43 +++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 k8s/addons/qingcloud/qingcloud-storage-class.yaml create mode 100644 k8s/manifests/qingcloud-volume-provisioner.yaml diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index de833ef..e08d1f0 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -33,4 +33,4 @@ KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernete KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" -KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes" \ No newline at end of file +KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes --enable-controller-attach-detach=true" \ No newline at end of file diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index b369d5b..9000688 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -18,14 +18,21 @@ function download_k8s_bin() for bin in "${k8s_bins[@]}"; do local bin_url="$k8s_base_url/${bin}" echo "downloading ${bin_url}" - wget -c ${bin_url} - unlink "${k8s_bin_path}/${bin}" - ln -s "${K8S_HOME}/bin/${bin}" "${k8s_bin_path}/${bin}" + wget ${bin_url} + ln -fs "${K8S_HOME}/bin/${bin}" "${k8s_bin_path}/${bin}" done + wget "https://pek3a.qingstor.com/k8s-qingcloud/k8s/qingcloud/volume/qingcloud-flex-volume" chmod +x * + ln -fs "${K8S_HOME}/bin/qingcloud-flex-volume" "${k8s_bin_path}/qingcloud-flex-volume" + "${K8S_HOME}/bin/qingcloud-flex-volume" --install=true popd } +function download_flex_volume_driver() +{ + /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ +} + rm -rf ${K8S_HOME}/bin/* download_k8s_bin diff --git a/k8s/addons/qingcloud/qingcloud-storage-class.yaml b/k8s/addons/qingcloud/qingcloud-storage-class.yaml new file mode 100644 index 0000000..2c5e3d7 --- /dev/null +++ b/k8s/addons/qingcloud/qingcloud-storage-class.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: qingcloud-storageclass + labels: + addonmanager.kubernetes.io/mode: EnsureExists + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" +provisioner: qingcloud/volume-provisioner \ No newline at end of file diff --git a/k8s/manifests/qingcloud-volume-provisioner.yaml b/k8s/manifests/qingcloud-volume-provisioner.yaml new file mode 100644 index 0000000..926d4a5 --- /dev/null +++ b/k8s/manifests/qingcloud-volume-provisioner.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + component: qingcloud-volume-provisioner + tier: control-plane + name: qingcloud-volume-provisioner + namespace: kube-system +spec: + containers: + - command: + - /qingcloud-volume-provisioner + - --v=${KUBE_LOG_LEVEL} + - --kubeconfig=/etc/kubernetes/admin.conf + - --qingcloud_config=/etc/qingcloud/client.yaml + image: dockerhub.qingcloud.com/qingcloud/qingcloud-volume-provisioner + imagePullPolicy: IfNotPresent + name: qingcloud-cloud-controller-manager + resources: + requests: + cpu: 250m + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s + readOnly: true + - mountPath: /etc/ssl/certs + name: certs + - mountPath: /etc/qingcloud + name: qingcloud + readOnly: true + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes + name: k8s + - hostPath: + path: /etc/ssl/certs + name: certs + - hostPath: + path: /etc/qingcloud + name: qingcloud \ No newline at end of file From 68c041e18f23d61868e76ab0e7a17b4590884b50 Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 17:39:49 +0800 Subject: [PATCH 079/167] update install script. --- image/update-k8s-bin.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index 9000688..07fe93b 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -15,6 +15,7 @@ function download_k8s_bin() { mkdir -p ${K8S_HOME}/bin pushd ${K8S_HOME}/bin + rm * for bin in "${k8s_bins[@]}"; do local bin_url="$k8s_base_url/${bin}" echo "downloading ${bin_url}" From a0d0def8f08cbfa6f82c764813311eb0466fe0de Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 19:08:15 +0800 Subject: [PATCH 080/167] map plugins to controller-manager. --- k8s/manifests/kube-controller-manager.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 4568bb8..49980ca 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -47,6 +47,8 @@ spec: readOnly: true - mountPath: /etc/ssl/certs name: certs + - mountPath: /usr/libexec/kubernetes/kubelet-plugins + name: plugins hostNetwork: true volumes: - name: k8s @@ -54,4 +56,6 @@ spec: path: /etc/kubernetes - name: certs hostPath: - path: /etc/ssl/certs \ No newline at end of file + path: /etc/ssl/certs + - name: plugins + path: /usr/libexec/kubernetes/kubelet-plugins \ No newline at end of file From 3f6ef9c49e74f8bfa617474809c034358da0ff9f Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 19:14:58 +0800 Subject: [PATCH 081/167] fix controller-manager. --- k8s/manifests/kube-controller-manager.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 49980ca..06a89b2 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -58,4 +58,5 @@ spec: hostPath: path: /etc/ssl/certs - name: plugins + hostPath: path: /usr/libexec/kubernetes/kubelet-plugins \ No newline at end of file From ec318cecc9e65f46d3c851532f622fc7922e32ac Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 19:27:07 +0800 Subject: [PATCH 082/167] mount /etc/qingcloud to controller manager. --- k8s/manifests/kube-controller-manager.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 06a89b2..55ef957 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -49,6 +49,9 @@ spec: name: certs - mountPath: /usr/libexec/kubernetes/kubelet-plugins name: plugins + - mountPath: /etc/qingcloud + name: qingcloud + readOnly: true hostNetwork: true volumes: - name: k8s @@ -59,4 +62,7 @@ spec: path: /etc/ssl/certs - name: plugins hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins \ No newline at end of file + path: /usr/libexec/kubernetes/kubelet-plugins + - name: qingcloud + hostPath: + path: /etc/qingcloud \ No newline at end of file From aeb33bfc96d2b723df923f0aad9e2077eeef447f Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 19:37:14 +0800 Subject: [PATCH 083/167] add --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --- confd/templates/k8s/kubelet.tmpl | 2 +- k8s/manifests/kube-controller-manager.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index e08d1f0..fa08fb2 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -33,4 +33,4 @@ KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernete KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" -KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes --enable-controller-attach-detach=true" \ No newline at end of file +KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes --enable-controller-attach-detach=true --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" \ No newline at end of file diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 55ef957..ec268a3 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -25,6 +25,7 @@ spec: - --allocate-node-cidrs=false - --service-cluster-ip-range=10.96.0.0/16 - --cloud-provider=external + - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ - --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false image: dockerhub.qingcloud.com/google_containers/hyperkube-amd64:${HYPERKUBE_VERSION} imagePullPolicy: IfNotPresent From 3e0e58a6b3b2427bf80a7fcff71e646b39cfeef6 Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 15 Aug 2017 19:48:37 +0800 Subject: [PATCH 084/167] update --volume-plugin-dir --- confd/templates/k8s/kubelet.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index fa08fb2..03ba334 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -33,4 +33,4 @@ KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernete KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" -KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes --enable-controller-attach-detach=true --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" \ No newline at end of file +KUBELET_EXTRA_ARGS="--node-labels=role={{getv "/host/role"}},node_id={{getv "/host/node_id"}} --max-pods 60 --feature-gates=AllAlpha=true,DynamicKubeletConfig=false,RotateKubeletServerCertificate=false,RotateKubeletClientCertificate=false --root-dir=/data/var/lib/kubelet --cert-dir=/data/var/run/kubernetes --enable-controller-attach-detach=true --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" \ No newline at end of file From a766480289efccaef328cf0aae7f739a973fbc06 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 10:10:20 +0800 Subject: [PATCH 085/167] mount controller flex log dir to host --- k8s/manifests/kube-controller-manager.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index ec268a3..82b76a3 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -50,6 +50,8 @@ spec: name: certs - mountPath: /usr/libexec/kubernetes/kubelet-plugins name: plugins + - mountPath: /var/log/qingcloud-flex-volume/ + name: flexlog - mountPath: /etc/qingcloud name: qingcloud readOnly: true @@ -66,4 +68,7 @@ spec: path: /usr/libexec/kubernetes/kubelet-plugins - name: qingcloud hostPath: - path: /etc/qingcloud \ No newline at end of file + path: /etc/qingcloud + - name: flexlog + hostPath: + path: /var/log/qingcloud-flex-volume-controller-manager/ \ No newline at end of file From b8f34d21205516ad2a6b78276d69748c7c84eae4 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 11:55:45 +0800 Subject: [PATCH 086/167] update k8s bin script. --- image/update-k8s-bin.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/image/update-k8s-bin.sh b/image/update-k8s-bin.sh index 07fe93b..374fa9e 100755 --- a/image/update-k8s-bin.sh +++ b/image/update-k8s-bin.sh @@ -15,7 +15,6 @@ function download_k8s_bin() { mkdir -p ${K8S_HOME}/bin pushd ${K8S_HOME}/bin - rm * for bin in "${k8s_bins[@]}"; do local bin_url="$k8s_base_url/${bin}" echo "downloading ${bin_url}" @@ -29,14 +28,11 @@ function download_k8s_bin() popd } -function download_flex_volume_driver() -{ - /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ -} - rm -rf ${K8S_HOME}/bin/* download_k8s_bin +systemctl is-active kubelet >/dev/null 2>&1 && systemctl restart kubelet + kubectl completion bash >/etc/profile.d/kubectl.sh kubeadm completion bash >/etc/profile.d/kubeadm.sh source /etc/profile \ No newline at end of file From ab17d289bdedc7a6f196368e2272f361681d410f Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 18:26:29 +0800 Subject: [PATCH 087/167] update image build script. --- image/README.md | 11 +++++++++++ image/build-image.sh | 19 ------------------- image/build.sh | 19 +++++++++++++++++++ image/{pre-install.sh => pre-build.sh} | 4 ++-- ...cker-images.sh => update-docker-images.sh} | 0 image/{install-pkg.sh => update-pkg.sh} | 2 +- ...oud-agent.sh => update-qingcloud-agent.sh} | 0 image/update-sysconfig.sh | 4 +++- ...systemd-conf.sh => update-systemd-conf.sh} | 0 9 files changed, 36 insertions(+), 23 deletions(-) create mode 100644 image/README.md delete mode 100755 image/build-image.sh create mode 100755 image/build.sh rename image/{pre-install.sh => pre-build.sh} (73%) rename image/{pull-docker-images.sh => update-docker-images.sh} (100%) rename image/{install-pkg.sh => update-pkg.sh} (85%) rename image/{install-qingcloud-agent.sh => update-qingcloud-agent.sh} (100%) rename image/{install-systemd-conf.sh => update-systemd-conf.sh} (100%) diff --git a/image/README.md b/image/README.md new file mode 100644 index 0000000..24a9c44 --- /dev/null +++ b/image/README.md @@ -0,0 +1,11 @@ +#Build kubernetes vm image + +* base ubuntu 16.04 + +```bash +apt-get install -y git +git clone https://github.com/QingCloudAppcenter/kubernetes.git /opt/kubernetes +cd /opt/kubernetes/image + +build.sh +``` \ No newline at end of file diff --git a/image/build-image.sh b/image/build-image.sh deleted file mode 100755 index 4cfc24b..0000000 --- a/image/build-image.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) -K8S_HOME=$(dirname "${SCRIPTPATH}") - -set -o errexit -set -o nounset -set -o pipefail - -${K8S_HOME}/image/install-pkg.sh -${K8S_HOME}/image/install-qingcloud-agent.sh -${K8S_HOME}/image/update-confd.sh -${K8S_HOME}/image/update-sysconfig.sh -${K8S_HOME}/image/update-sshd-config.sh -${K8S_HOME}/image/update-k8s-bin.sh -${K8S_HOME}/image/update-cni.sh -${K8S_HOME}/image/update-logrotate.sh -${K8S_HOME}/image/install-systemd-conf.sh -${K8S_HOME}/image/pull-docker-images.sh \ No newline at end of file diff --git a/image/build.sh b/image/build.sh new file mode 100755 index 0000000..50d4791 --- /dev/null +++ b/image/build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +K8S_HOME=$(dirname "${SCRIPTPATH}") + +set -o errexit +set -o nounset +set -o pipefail + +source ${K8S_HOME}/image/update-pkg.sh +source ${K8S_HOME}/image/update-qingcloud-agent.sh +source ${K8S_HOME}/image/update-confd.sh +source ${K8S_HOME}/image/update-sysconfig.sh +source ${K8S_HOME}/image/update-sshd-config.sh +source ${K8S_HOME}/image/update-k8s-bin.sh +source ${K8S_HOME}/image/update-cni.sh +source ${K8S_HOME}/image/update-logrotate.sh +source ${K8S_HOME}/image/update-systemd-conf.sh +source ${K8S_HOME}/image/pull-docker-images.sh \ No newline at end of file diff --git a/image/pre-install.sh b/image/pre-build.sh similarity index 73% rename from image/pre-install.sh rename to image/pre-build.sh index f180b7f..c71d4f6 100755 --- a/image/pre-install.sh +++ b/image/pre-build.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -apt-get install -y git wget +apt-get install -y git git clone https://github.com/QingCloudAppcenter/kubernetes.git /opt/kubernetes cd /opt/kubernetes/image -./build-image.sh +build.sh diff --git a/image/pull-docker-images.sh b/image/update-docker-images.sh similarity index 100% rename from image/pull-docker-images.sh rename to image/update-docker-images.sh diff --git a/image/install-pkg.sh b/image/update-pkg.sh similarity index 85% rename from image/install-pkg.sh rename to image/update-pkg.sh index d55549a..3f04606 100755 --- a/image/install-pkg.sh +++ b/image/update-pkg.sh @@ -3,7 +3,7 @@ apt-get update apt-get install -y ebtables socat -apt-get install -y jq apt-transport-https bash-completion +apt-get install -y jq apt-transport-https bash-completion ntp wget apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D apt-add-repository 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' diff --git a/image/install-qingcloud-agent.sh b/image/update-qingcloud-agent.sh similarity index 100% rename from image/install-qingcloud-agent.sh rename to image/update-qingcloud-agent.sh diff --git a/image/update-sysconfig.sh b/image/update-sysconfig.sh index 6c5fcee..8e7f38c 100755 --- a/image/update-sysconfig.sh +++ b/image/update-sysconfig.sh @@ -18,4 +18,6 @@ iface lo inet loopback allow-hotplug eth0 iface eth0 inet dhcp -EOF \ No newline at end of file +EOF + +timedatectl set-timezone UTC \ No newline at end of file diff --git a/image/install-systemd-conf.sh b/image/update-systemd-conf.sh similarity index 100% rename from image/install-systemd-conf.sh rename to image/update-systemd-conf.sh From 9c0ebe2ae72b8094117656d98a492df76c6b8f39 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 18:28:23 +0800 Subject: [PATCH 088/167] remove client image script. --- client_image/client_image_build.sh | 37 ------------------- client_image/confd/conf.d/k8s/env.sh.toml | 6 --- .../confd/conf.d/k8s/init_token.metad.toml | 6 --- client_image/confd/conf.d/k8s/interfaces.toml | 7 ---- client_image/confd/templates/k8s/env.sh.tmpl | 10 ----- .../confd/templates/k8s/init_token.metad.tmpl | 1 - .../confd/templates/k8s/interfaces.tmpl | 8 ---- client_image/script/init_config.sh | 37 ------------------- client_image/script/reload_interface.sh | 2 - script/init-client.sh | 5 +-- 10 files changed, 1 insertion(+), 118 deletions(-) delete mode 100644 client_image/client_image_build.sh delete mode 100644 client_image/confd/conf.d/k8s/env.sh.toml delete mode 100644 client_image/confd/conf.d/k8s/init_token.metad.toml delete mode 100644 client_image/confd/conf.d/k8s/interfaces.toml delete mode 100644 client_image/confd/templates/k8s/env.sh.tmpl delete mode 100644 client_image/confd/templates/k8s/init_token.metad.tmpl delete mode 100644 client_image/confd/templates/k8s/interfaces.tmpl delete mode 100644 client_image/script/init_config.sh delete mode 100644 client_image/script/reload_interface.sh diff --git a/client_image/client_image_build.sh b/client_image/client_image_build.sh deleted file mode 100644 index 7f9d63e..0000000 --- a/client_image/client_image_build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -apt-get update -apt-get upgrade -y -apt-get install -y wget jq bash-completion -wget http://k8s-qingcloud.pek3a.qingstor.com/k8s/release/bin/linux/amd64/kubectl -O /usr/bin/kubectl -wget http://k8s-qingcloud.pek3a.qingstor.com/k8s/release/bin/linux/amd64/kubeadm -O /usr/bin/kubeadm - -chmod +x /usr/bin/kubectl -chmod +x /usr/bin/kubeadm - - - -pushd /tmp -wget https://pek3a.qingstor.com/appcenter/developer/packages/app-agent-linux-amd64.tar.gz -tar -zxvf app-agent-linux-amd64.tar.gz -cd app-agent-linux-amd64/ -./install.sh -cd .. -rm -rf app-agent-linux-amd64/ -rm app-agent-linux-amd64.tar.gz -popd - -cp -r confd/* /etc/confd/ -cp script/* /usr/bin/ -chmod +x /usr/bin/*.sh - -cat << EOF > /etc/network/interfaces -auto lo -iface lo inet loopback - -allow-hotplug eth0 -iface eth0 inet dhcp -EOF - -sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config -kubectl completion bash >>/etc/profile \ No newline at end of file diff --git a/client_image/confd/conf.d/k8s/env.sh.toml b/client_image/confd/conf.d/k8s/env.sh.toml deleted file mode 100644 index bf1645e..0000000 --- a/client_image/confd/conf.d/k8s/env.sh.toml +++ /dev/null @@ -1,6 +0,0 @@ -[template] -src = "k8s/env.sh.tmpl" -dest = "/data/kubernetes/env.sh" -keys = [ - "/", -] \ No newline at end of file diff --git a/client_image/confd/conf.d/k8s/init_token.metad.toml b/client_image/confd/conf.d/k8s/init_token.metad.toml deleted file mode 100644 index 607c53c..0000000 --- a/client_image/confd/conf.d/k8s/init_token.metad.toml +++ /dev/null @@ -1,6 +0,0 @@ -[template] -src = "k8s/init_token.metad.tmpl" -dest = "/data/kubernetes/init_token.metad" -keys = [ - "/", -] diff --git a/client_image/confd/conf.d/k8s/interfaces.toml b/client_image/confd/conf.d/k8s/interfaces.toml deleted file mode 100644 index df9c116..0000000 --- a/client_image/confd/conf.d/k8s/interfaces.toml +++ /dev/null @@ -1,7 +0,0 @@ -[template] -src = "k8s/interfaces.tmpl" -dest = "/etc/network/interfaces" -keys = [ - "/", -] -reload_cmd = "/usr/bin/reload_interface.sh" \ No newline at end of file diff --git a/client_image/confd/templates/k8s/env.sh.tmpl b/client_image/confd/templates/k8s/env.sh.tmpl deleted file mode 100644 index c6280e0..0000000 --- a/client_image/confd/templates/k8s/env.sh.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -{{range gets "/host/*"}} -HOST_{{base .Key| toUpper}}="{{.Value|js}}" -{{end}} -{{range gets "/hosts/master/*/*"}} -MASTER_{{base .Key| toUpper}}="{{.Value|js}}" -{{end}} -{{range gets "/env/*"}} -ENV_{{base .Key| toUpper}}="{{.Value|js}}" -{{end}} diff --git a/client_image/confd/templates/k8s/init_token.metad.tmpl b/client_image/confd/templates/k8s/init_token.metad.tmpl deleted file mode 100644 index 7b55106..0000000 --- a/client_image/confd/templates/k8s/init_token.metad.tmpl +++ /dev/null @@ -1 +0,0 @@ -{{range gets "/hosts/master/*/token"}}{{$data := json .Value}}{{$data.init_token}}{{end}} diff --git a/client_image/confd/templates/k8s/interfaces.tmpl b/client_image/confd/templates/k8s/interfaces.tmpl deleted file mode 100644 index ad920bd..0000000 --- a/client_image/confd/templates/k8s/interfaces.tmpl +++ /dev/null @@ -1,8 +0,0 @@ -auto lo -iface lo inet loopback -{{$masters := getvs "/hosts/master/*/ip"}} -allow-hotplug eth0 -iface eth0 inet dhcp - dns-search default.svc.cluster.local svc.cluster.local cluster.local - dns-nameservers 10.96.0.10 - up route add -net 10.96.0.0 netmask 255.255.0.0 gw {{index $masters 0}} \ No newline at end of file diff --git a/client_image/script/init_config.sh b/client_image/script/init_config.sh deleted file mode 100644 index 342a822..0000000 --- a/client_image/script/init_config.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -source "/data/kubernetes/env.sh" - -function fail { - echo $1 >&2 - exit 1 -} - -function retry { - local n=1 - local max=30 - local delay=10 - while true; do - "$@" && break || { - if [[ $n -lt $max ]]; then - ((n++)) - echo "Command failed. Attempt $n/$max:" - sleep $delay; - else - fail "The command has failed after $n attempts." - fi - } - done -} - -init_token=`cat /data/kubernetes/init_token.metad` - -while [ -z ${init_token} ] -do - echo "sleep for wait init_token for 2 second" - sleep 2 - init_token=`cat /data/kubernetes/init_token.metad` -done - -echo "master ip: ${MASTER_IP} init_token: ${init_token}" -retry kubeadm client-config ${MASTER_IP} --token ${init_token} --skip-preflight-checks diff --git a/client_image/script/reload_interface.sh b/client_image/script/reload_interface.sh deleted file mode 100644 index 9a49d9a..0000000 --- a/client_image/script/reload_interface.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -ifdown eth0 && ifup eth0 \ No newline at end of file diff --git a/script/init-client.sh b/script/init-client.sh index 6a3a82d..b3b6eb6 100755 --- a/script/init-client.sh +++ b/script/init-client.sh @@ -6,7 +6,4 @@ source "${K8S_HOME}/script/common.sh" echo "root:k8s" |chpasswd sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config -systemctl restart ssh - -ensure_dir -join_node +systemctl restart ssh \ No newline at end of file From db5a4c605f7103beaa5f2fbc66228db116afb8e3 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 18:31:36 +0800 Subject: [PATCH 089/167] update docker images script. --- image/update-docker-images.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/image/update-docker-images.sh b/image/update-docker-images.sh index 55eb34d..b135a9d 100755 --- a/image/update-docker-images.sh +++ b/image/update-docker-images.sh @@ -32,4 +32,6 @@ docker pull dockerhub.qingcloud.com/google_containers/addon-resizer:1.7 docker pull dockerhub.qingcloud.com/elastic/kibana:5.5.1 docker pull dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 docker pull dockerhub.qingcloud.com/qingcloud/elasticsearch-curator:5.1.1 -docker pull dockerhub.qingcloud.com/qingcloud/file-sync:0.7 \ No newline at end of file +docker pull dockerhub.qingcloud.com/qingcloud/file-sync:0.7 +docker pull dockerhub.qingcloud.com/qingcloud/qingcloud-volume-provisioner +docker pull dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager \ No newline at end of file From 8e36a5e39d9321f244a427d5aab777c8a3c69464 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 18:53:51 +0800 Subject: [PATCH 090/167] update image script. --- image/build.sh | 20 ++++++++++---------- image/update-pkg.sh | 8 +++++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/image/build.sh b/image/build.sh index 50d4791..8fc67dd 100755 --- a/image/build.sh +++ b/image/build.sh @@ -7,13 +7,13 @@ set -o errexit set -o nounset set -o pipefail -source ${K8S_HOME}/image/update-pkg.sh -source ${K8S_HOME}/image/update-qingcloud-agent.sh -source ${K8S_HOME}/image/update-confd.sh -source ${K8S_HOME}/image/update-sysconfig.sh -source ${K8S_HOME}/image/update-sshd-config.sh -source ${K8S_HOME}/image/update-k8s-bin.sh -source ${K8S_HOME}/image/update-cni.sh -source ${K8S_HOME}/image/update-logrotate.sh -source ${K8S_HOME}/image/update-systemd-conf.sh -source ${K8S_HOME}/image/pull-docker-images.sh \ No newline at end of file +${K8S_HOME}/image/update-pkg.sh +${K8S_HOME}/image/update-qingcloud-agent.sh +${K8S_HOME}/image/update-confd.sh +${K8S_HOME}/image/update-sysconfig.sh +${K8S_HOME}/image/update-sshd-config.sh +${K8S_HOME}/image/update-k8s-bin.sh +${K8S_HOME}/image/update-cni.sh +${K8S_HOME}/image/update-logrotate.sh +${K8S_HOME}/image/update-systemd-conf.sh +${K8S_HOME}/image/pull-docker-images.sh \ No newline at end of file diff --git a/image/update-pkg.sh b/image/update-pkg.sh index 3f04606..7df7153 100755 --- a/image/update-pkg.sh +++ b/image/update-pkg.sh @@ -2,16 +2,18 @@ apt-get update -apt-get install -y ebtables socat -apt-get install -y jq apt-transport-https bash-completion ntp wget +apt-get install -y ebtables socat jq apt-transport-https bash-completion ntp wget apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D apt-add-repository 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' + apt-get update + apt-cache policy docker-engine apt-get install -y docker-engine apt-get remove network-manager -apt-get upgrade -y +DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade + From d140cf49de396997d85ab8c577d53b1e0390685e Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 18:58:07 +0800 Subject: [PATCH 091/167] fix image build script. --- image/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/build.sh b/image/build.sh index 8fc67dd..a6c982e 100755 --- a/image/build.sh +++ b/image/build.sh @@ -16,4 +16,4 @@ ${K8S_HOME}/image/update-k8s-bin.sh ${K8S_HOME}/image/update-cni.sh ${K8S_HOME}/image/update-logrotate.sh ${K8S_HOME}/image/update-systemd-conf.sh -${K8S_HOME}/image/pull-docker-images.sh \ No newline at end of file +${K8S_HOME}/image/update-docker-images.sh \ No newline at end of file From 5db6cbe8c2139f58d54d25909f03b4b9e7a840e3 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 16 Aug 2017 19:03:41 +0800 Subject: [PATCH 092/167] add capacity storage class. --- .../qingcloud/qingcloud-storage-class-capacity.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml diff --git a/k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml b/k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml new file mode 100644 index 0000000..d61c63c --- /dev/null +++ b/k8s/addons/qingcloud/qingcloud-storage-class-capacity.yaml @@ -0,0 +1,11 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: qingcloud-storageclass-capacity + labels: + addonmanager.kubernetes.io/mode: EnsureExists + annotations: + storageclass.beta.kubernetes.io/is-default-class: "false" +provisioner: qingcloud/volume-provisioner +parameters: + type: "2" \ No newline at end of file From f5d91c1d699b56a7bc5ae8c92023f7282451e1cd Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 17 Aug 2017 11:02:50 +0800 Subject: [PATCH 093/167] update app config. --- app/cluster.json.mustache | 10 +++++----- app/config.json | 10 ---------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index acf376f..fc64535 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -7,7 +7,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-tacg33kl", + "image": "img-neffdxwj", "zone": "pek3a" }, "instance_class": {{cluster.master.instance_class}}, @@ -84,7 +84,7 @@ "role": "node", "container": { "type": "kvm", - "image": "img-tacg33kl", + "image": "img-neffdxwj", "zone": "pek3a" }, "instance_class": {{cluster.node.instance_class}}, @@ -166,7 +166,7 @@ "role": "log", "container": { "type": "kvm", - "image": "img-tacg33kl", + "image": "img-neffdxwj", "zone": "pek3a" }, "instance_class": {{cluster.log.instance_class}}, @@ -247,11 +247,11 @@ "role": "client", "container": { "type": "kvm", - "image": "img-tacg33kl", + "image": "img-neffdxwj", "zone": "pek3a" }, "instance_class": {{cluster.client.instance_class}}, - "count": {{cluster.client.count}}, + "count": 1, "cpu": {{cluster.client.cpu}}, "memory": {{cluster.client.memory}}, "user_access":true, diff --git a/app/config.json b/app/config.json index 4312bf5..66127d9 100644 --- a/app/config.json +++ b/app/config.json @@ -271,16 +271,6 @@ ], "required": "yes" }, - { - "key": "count", - "label": "count", - "description": "Number of client nodes for the cluster to create", - "type": "integer", - "default": 1, - "min":1, - "max":1, - "required": "yes" - }, { "key": "instance_class", "label": "resource type", From 81bacafb3515e41063999ceeb9c7c4e02c3e9519 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 17 Aug 2017 12:18:47 +0800 Subject: [PATCH 094/167] add leader-elector sample. --- sample/leader-elector-deployment.yaml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 sample/leader-elector-deployment.yaml diff --git a/sample/leader-elector-deployment.yaml b/sample/leader-elector-deployment.yaml new file mode 100644 index 0000000..e5c1e5d --- /dev/null +++ b/sample/leader-elector-deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: leader-elector +spec: + replicas: 3 + template: + metadata: + labels: + app: leader-elector + spec: + serviceAccountName: leader-elector-example + containers: + - name: web + image: dockerhub.qingcloud.com/google_containers/leader-elector:0.5 + ports: + - name: web + containerPort: 80 + protocol: TCP +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: leader-elector-example +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: leader-elector +rules: +- apiGroups: + - "" + resources: + - "endpoints" + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: leader-elector +subjects: +- kind: ServiceAccount + name: leader-elector-example + apiGroup: "" +roleRef: + kind: ClusterRole + name: leader-elector + apiGroup: "" From ef275e451f152a2b31da8c93bfcae48dbaa052b7 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 17 Aug 2017 12:25:38 +0800 Subject: [PATCH 095/167] update leader elector --- sample/leader-elector-deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sample/leader-elector-deployment.yaml b/sample/leader-elector-deployment.yaml index e5c1e5d..a414e01 100644 --- a/sample/leader-elector-deployment.yaml +++ b/sample/leader-elector-deployment.yaml @@ -13,6 +13,8 @@ spec: containers: - name: web image: dockerhub.qingcloud.com/google_containers/leader-elector:0.5 + args: + - --election=example ports: - name: web containerPort: 80 @@ -42,6 +44,7 @@ metadata: subjects: - kind: ServiceAccount name: leader-elector-example + namespace: default apiGroup: "" roleRef: kind: ClusterRole From f7b4f1929c570173c73849965b8b1f9dfd63eebd Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 17 Aug 2017 12:30:37 +0800 Subject: [PATCH 096/167] update leader-elector --- sample/leader-elector-deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sample/leader-elector-deployment.yaml b/sample/leader-elector-deployment.yaml index a414e01..73a8fcb 100644 --- a/sample/leader-elector-deployment.yaml +++ b/sample/leader-elector-deployment.yaml @@ -15,6 +15,7 @@ spec: image: dockerhub.qingcloud.com/google_containers/leader-elector:0.5 args: - --election=example + - --http=0.0.0.0:4040 ports: - name: web containerPort: 80 From d647d7ca4964d09535aafbe798d5bbd5eaa802e7 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 18 Aug 2017 11:01:49 +0800 Subject: [PATCH 097/167] supported docker option registry-mirrors. --- app/cluster.json.mustache | 1 + app/config.json | 8 ++++++++ confd/templates/k8s/daemon.json.tmpl | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index fc64535..48e1e19 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -268,6 +268,7 @@ "secret_access_key": {{env.secret_access_key}}, "api_external_domain": {{env.api_external_domain}}, "insecure-registries": {{env.insecure-registries}}, + "registry-mirrors": {{env.registry-mirrors}}, "pod_vxnets":{{env.pod_vxnets}}, "dockerhub_username":{{env.dockerhub_username}}, "dockerhub_password":{{env.dockerhub_password}}, diff --git a/app/config.json b/app/config.json index 66127d9..5e4d5ee 100644 --- a/app/config.json +++ b/app/config.json @@ -317,6 +317,14 @@ "changeable": false, "required": "no" }, + { + "key": "registry-mirrors", + "label": "Registry mirrors", + "description": "The docker hub registry mirror, use a blank to split multi registry", + "type": "string", + "default": "https://registry.docker-cn.com", + "required": "no" + }, { "key": "insecure-registries", "label": "Insecure registries", diff --git a/confd/templates/k8s/daemon.json.tmpl b/confd/templates/k8s/daemon.json.tmpl index 9077a5c..47bdf12 100644 --- a/confd/templates/k8s/daemon.json.tmpl +++ b/confd/templates/k8s/daemon.json.tmpl @@ -5,5 +5,9 @@ {{if ne $registries ""}} "insecure-registries": {{split $registries " "|toJson}}, {{end}} + {{$registry-mirrors := getv "/env/registry-mirrors"}} + {{if ne $registry-mirrors ""}} + "registry-mirrors": {{split $registry-mirrors " "|toJson}}, + {{end}} "storage-driver": "overlay2" } \ No newline at end of file From cf70e9266cd8f337e83fb5e612147caf0f820e8c Mon Sep 17 00:00:00 2001 From: jolestar Date: Mon, 21 Aug 2017 11:39:36 +0800 Subject: [PATCH 098/167] remove train_master when start. --- script/init-master.sh | 1 - script/start-master.sh | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/script/init-master.sh b/script/init-master.sh index 6b39044..e67412a 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -30,4 +30,3 @@ retry kubeadm alpha phase bootstrap-token cluster-info /etc/kubernetes/admin.con retry kubeadm alpha phase apiconfig --kubeconfig /etc/kubernetes/admin.conf process_addons -#train_master diff --git a/script/start-master.sh b/script/start-master.sh index 9da7366..a06c00d 100755 --- a/script/start-master.sh +++ b/script/start-master.sh @@ -10,5 +10,4 @@ ensure_dir systemctl start docker systemctl start kubelet wait_kubelet -wait_apiserver -train_master \ No newline at end of file +wait_apiserver \ No newline at end of file From d685fc7f04c45d05fe3ca94d98293c9ff077c441 Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 22 Aug 2017 16:08:26 +0800 Subject: [PATCH 099/167] set kubelet authmode to AlwaysAllow --- confd/templates/k8s/kubelet.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confd/templates/k8s/kubelet.tmpl b/confd/templates/k8s/kubelet.tmpl index 03ba334..d4dc0c8 100644 --- a/confd/templates/k8s/kubelet.tmpl +++ b/confd/templates/k8s/kubelet.tmpl @@ -29,7 +29,7 @@ KUBELET_DNS_ARGS="--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" KUBE_CLOUD_PROVIDER="--cloud-provider=external" -KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt" +KUBELET_AUTHZ_ARGS="--authorization-mode=AlwaysAllow --client-ca-file=/etc/kubernetes/pki/ca.crt" KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs" From 4f5c693c856f7d1f1705b6202c68696d5d209e7a Mon Sep 17 00:00:00 2001 From: jolestar Date: Tue, 22 Aug 2017 23:16:31 +0800 Subject: [PATCH 100/167] set /etc/security/limits.conf limit. --- image/update-sysconfig.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/image/update-sysconfig.sh b/image/update-sysconfig.sh index 8e7f38c..3b8784b 100755 --- a/image/update-sysconfig.sh +++ b/image/update-sysconfig.sh @@ -20,4 +20,9 @@ allow-hotplug eth0 iface eth0 inet dhcp EOF -timedatectl set-timezone UTC \ No newline at end of file +timedatectl set-timezone UTC + +cat << EOF > /etc/security/limits.conf +root soft nofile 200000 +root hard nofile 200000 +EOF \ No newline at end of file From bb1f8c4b6c9bc1514d4302a58eedc3d2036756c9 Mon Sep 17 00:00:00 2001 From: jolestar Date: Wed, 23 Aug 2017 09:36:11 +0800 Subject: [PATCH 101/167] update dashboard service account. --- .../dashboard/dashboard-deployment.yaml | 6 ++++- k8s/addons/dashboard/dashboard-sa.yaml | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 k8s/addons/dashboard/dashboard-sa.yaml diff --git a/k8s/addons/dashboard/dashboard-deployment.yaml b/k8s/addons/dashboard/dashboard-deployment.yaml index f9283d3..4706bcc 100644 --- a/k8s/addons/dashboard/dashboard-deployment.yaml +++ b/k8s/addons/dashboard/dashboard-deployment.yaml @@ -8,6 +8,8 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: + replicas: 1 + revisionHistoryLimit: 10 selector: matchLabels: k8s-app: kubernetes-dashboard @@ -18,9 +20,10 @@ spec: labels: k8s-app: kubernetes-dashboard spec: + serviceAccountName: kubernetes-dashboard containers: - name: kubernetes-dashboard - image: dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.6.1 + image: dockerhub.qingcloud.com/google_containers/kubernetes-dashboard-amd64:v1.6.3 imagePullPolicy: IfNotPresent resources: # keep request = limit to keep this container in guaranteed class @@ -32,6 +35,7 @@ spec: memory: 100Mi ports: - containerPort: 9090 + protocol: TCP livenessProbe: httpGet: path: / diff --git a/k8s/addons/dashboard/dashboard-sa.yaml b/k8s/addons/dashboard/dashboard-sa.yaml new file mode 100644 index 0000000..801669f --- /dev/null +++ b/k8s/addons/dashboard/dashboard-sa.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + name: kubernetes-dashboard + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system \ No newline at end of file From e7a2a612f20cec8b513415af779329fa5884667b Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:25:02 +0800 Subject: [PATCH 102/167] fluent bit support forward server config. --- confd/conf.d/k8s/fluentbit-extend.conf.toml | 7 ++ .../templates/k8s/fluentbit-extend.conf.tmpl | 9 +++ k8s/addons/monitor/fluentbit-cm-extend.yaml | 13 ++++ k8s/addons/monitor/fluentbit-cm.yaml | 64 +++++++++++++++++++ ...t-elasticsearch.yaml => fluentbit-ds.yaml} | 29 ++++++++- script/common.sh | 10 +++ script/reload-fluentbit.sh | 8 +++ 7 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 confd/conf.d/k8s/fluentbit-extend.conf.toml create mode 100644 confd/templates/k8s/fluentbit-extend.conf.tmpl create mode 100644 k8s/addons/monitor/fluentbit-cm-extend.yaml create mode 100644 k8s/addons/monitor/fluentbit-cm.yaml rename k8s/addons/monitor/{fluent-bit-daemonset-elasticsearch.yaml => fluentbit-ds.yaml} (66%) create mode 100755 script/reload-fluentbit.sh diff --git a/confd/conf.d/k8s/fluentbit-extend.conf.toml b/confd/conf.d/k8s/fluentbit-extend.conf.toml new file mode 100644 index 0000000..5a202d5 --- /dev/null +++ b/confd/conf.d/k8s/fluentbit-extend.conf.toml @@ -0,0 +1,7 @@ +[template] +src = "k8s/fluentbit-extend.conf.tmpl" +dest = "/data/kubernetes/fluentbit/extend.conf" +keys = [ + "/env", +] +reload_cmd = "/opt/kubernetes/script/reload-fluentbit.sh" \ No newline at end of file diff --git a/confd/templates/k8s/fluentbit-extend.conf.tmpl b/confd/templates/k8s/fluentbit-extend.conf.tmpl new file mode 100644 index 0000000..75aea16 --- /dev/null +++ b/confd/templates/k8s/fluentbit-extend.conf.tmpl @@ -0,0 +1,9 @@ +{{- $server := getv "/env/fluent-forward-server" -}} +{{- if ne $server "" -}} +{{- parts := split $server ":" -}} +[OUTPUT] + Name forward + Match * + Host {{index $parts 0}} + Port {{index $parts 1}} +{{- end -}} \ No newline at end of file diff --git a/k8s/addons/monitor/fluentbit-cm-extend.yaml b/k8s/addons/monitor/fluentbit-cm-extend.yaml new file mode 100644 index 0000000..8d3ed51 --- /dev/null +++ b/k8s/addons/monitor/fluentbit-cm-extend.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit-extend + namespace: kube-system + labels: + # this config should be EnsureExists, allow change by api. + addonmanager.kubernetes.io/mode: EnsureExists +data: + extend.conf: | + # empty config + + diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml new file mode 100644 index 0000000..fcf3f57 --- /dev/null +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + fluent-bit.conf: | + [SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser docker + DB /var/log/flb_kube.db + Mem_Buf_Limit 5MB + + [INPUT] + Name systemd + Tag host.* + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Merge_JSON_Log On + + [OUTPUT] + Name es + Match * + Host ${FLUENT_ELASTICSEARCH_HOST} + Port ${FLUENT_ELASTICSEARCH_PORT} + Index fluentbit + Type log + Logstash_Format On + + @INCLUDE extend.conf + parsers.conf: | + [PARSER] + Name json-test + Format json + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z + + [PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + diff --git a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml b/k8s/addons/monitor/fluentbit-ds.yaml similarity index 66% rename from k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml rename to k8s/addons/monitor/fluentbit-ds.yaml index 692d313..ad9b4b1 100644 --- a/k8s/addons/monitor/fluent-bit-daemonset-elasticsearch.yaml +++ b/k8s/addons/monitor/fluentbit-ds.yaml @@ -8,17 +8,26 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: + selector: + matchLabels: + k8s-app: fluent-bit-logging + updateStrategy: + type: RollingUpdate template: metadata: labels: k8s-app: fluent-bit-logging annotations: scheduler.alpha.kubernetes.io/critical-pod: '' + qingcloud.com/update-time: "" spec: serviceAccountName: fluentd-es containers: - name: fluent-bit - image: dockerhub.qingcloud.com/fluent/fluent-bit-kubernetes-daemonset:0.11.13 + image: dockerhub.qingcloud.com/fluent/fluent-bit:0.12 + command: + - /fluent-bit/bin/fluent-bit + - -c /fluent-bit/etc/fluent-bit.conf imagePullPolicy: IfNotPresent env: - name: FLUENT_ELASTICSEARCH_HOST @@ -37,6 +46,15 @@ spec: - name: varlibdockercontainers mountPath: /data/var/lib/docker/containers readOnly: true + - name: journallog + mountPath: /run/log/journal/ + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc/ + readOnly: true + - name: fluent-bit-extend-config + mountPath: /fluent-bit/etc/ + readOnly: true ports: - containerPort: 24224 name: ui @@ -49,6 +67,15 @@ spec: - name: varlibdockercontainers hostPath: path: /data/var/lib/docker/containers + - name: journallog + hostPath: + path: /run/log/journal/ + - name: fluent-bit-config + configMap: + name: fluent-bit + - name: fluent-bit-extend-config + configMap: + name: fluent-bit-extend tolerations: - key: "CriticalAddonsOnly" operator: "Exists" diff --git a/script/common.sh b/script/common.sh index e7b8604..b3f0fb9 100755 --- a/script/common.sh +++ b/script/common.sh @@ -322,4 +322,14 @@ function upgrade_docker(){ ln -s /data/var/lib/docker /var/lib/docker ln -s /data/var/lib/kubelet /var/lib/kubelet return 0 +} + +function update_fluent_config(){ + if [ "${HOST_ROLE}" == "master" ] + then + kubectl create configmap --dry-run -o yaml fluent-bit-extend -n kube-system --from-file /etc/kubernetes/fluentbit/extend.conf | kubectl replace -f - + date=$(date +%s) + sed -i 's/qingcloud\.com\/update-time:.*/qingcloud\.com\/update-time: "'${date}'"/g' /etc/kubernetes/addons/fluentbit-ds.yaml + kubectl apply -f /etc/kubernetes/addons/fluentbit-ds.yaml + fi } \ No newline at end of file diff --git a/script/reload-fluentbit.sh b/script/reload-fluentbit.sh new file mode 100755 index 0000000..4fb46c1 --- /dev/null +++ b/script/reload-fluentbit.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +K8S_HOME=$(dirname "${SCRIPTPATH}") + +source "${K8S_HOME}/script/common.sh" + +systemctl is-active kubelet >/dev/null 2>&1 && update_fluent_config \ No newline at end of file From 9a82a73af720243fc1f45feaceac55963e43d284 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:26:10 +0800 Subject: [PATCH 103/167] update logrotate config. --- logrotate/conf | 12 ------------ logrotate/docker-containers | 4 ++-- logrotate/flex-volume | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 14 deletions(-) delete mode 100644 logrotate/conf create mode 100644 logrotate/flex-volume diff --git a/logrotate/conf b/logrotate/conf deleted file mode 100644 index 7228708..0000000 --- a/logrotate/conf +++ /dev/null @@ -1,12 +0,0 @@ -/var/log/containers/*.log { - rotate 5 - copytruncate - missingok - notifempty - compress - maxsize 100M - daily - dateext - dateformat -%Y%m%d-%s - create 0644 root root -} diff --git a/logrotate/docker-containers b/logrotate/docker-containers index 0b9d2ae..419bf78 100644 --- a/logrotate/docker-containers +++ b/logrotate/docker-containers @@ -1,5 +1,5 @@ - /var/lib/docker/containers/*/*-json.log { - rotate 5 +/data/var/lib/docker/containers/*/*-json.log { + rotate 1 copytruncate missingok notifempty diff --git a/logrotate/flex-volume b/logrotate/flex-volume new file mode 100644 index 0000000..bf4064b --- /dev/null +++ b/logrotate/flex-volume @@ -0,0 +1,21 @@ +/var/log/qingcloud-flex-volume/* { + rotate 1 + copytruncate + missingok + notifempty + compress + maxsize 10M + daily + create 0644 root root +} + +/var/log/qingcloud-flex-volume-controller-manager/* { + rotate 1 + copytruncate + missingok + notifempty + compress + maxsize 10M + daily + create 0644 root root +} \ No newline at end of file From 9c7ba6c5785c11fe0c98200262c330a8f2a43f6b Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:27:23 +0800 Subject: [PATCH 104/167] manifests update. --- k8s/manifests/kube-addon-manager.yaml | 12 +++--------- k8s/manifests/kube-controller-manager.yaml | 10 ++++++++-- k8s/manifests/qingcloud-volume-provisioner.yaml | 12 ++++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/k8s/manifests/kube-addon-manager.yaml b/k8s/manifests/kube-addon-manager.yaml index b978238..3018f68 100644 --- a/k8s/manifests/kube-addon-manager.yaml +++ b/k8s/manifests/kube-addon-manager.yaml @@ -20,7 +20,7 @@ spec: command: - /bin/bash - -c - - /opt/kube-addons.sh 1>>/var/log/kube-addon-manager.log 2>&1 + - /opt/kube-addons.sh resources: requests: cpu: 5m @@ -29,13 +29,7 @@ spec: - mountPath: /etc/kubernetes/ name: addons readOnly: true - - mountPath: /var/log - name: varlog - readOnly: false volumes: - - hostPath: + - name: addons + hostPath: path: /etc/kubernetes/ - name: addons - - hostPath: - path: /var/log - name: varlog \ No newline at end of file diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 82b76a3..0bc39ea 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -10,7 +10,8 @@ metadata: namespace: kube-system spec: containers: - - command: + - name: kube-controller-manager + command: - kube-controller-manager - --v=${KUBE_LOG_LEVEL} - --kubeconfig=/etc/kubernetes/controller-manager.conf @@ -38,7 +39,6 @@ spec: scheme: HTTP initialDelaySeconds: 15 timeoutSeconds: 15 - name: kube-controller-manager resources: requests: cpu: 200m @@ -55,6 +55,12 @@ spec: - mountPath: /etc/qingcloud name: qingcloud readOnly: true + - name: flexlog + image: dockerhub.qingcloud.com/busybox:1.27.1 + args: [/bin/sh, -c, 'tail -n+1 -F --retry /var/log/qingcloud-flex-volume/flex-volume.INFO'] + volumeMounts: + - name: flexlog + mountPath: /var/log/qingcloud-flex-volume hostNetwork: true volumes: - name: k8s diff --git a/k8s/manifests/qingcloud-volume-provisioner.yaml b/k8s/manifests/qingcloud-volume-provisioner.yaml index 926d4a5..b0226bf 100644 --- a/k8s/manifests/qingcloud-volume-provisioner.yaml +++ b/k8s/manifests/qingcloud-volume-provisioner.yaml @@ -32,12 +32,12 @@ spec: readOnly: true hostNetwork: true volumes: - - hostPath: + - name: k8s + hostPath: path: /etc/kubernetes - name: k8s - - hostPath: + - name: certs + hostPath: path: /etc/ssl/certs - name: certs - - hostPath: + - name: qingcloud + hostPath: path: /etc/qingcloud - name: qingcloud \ No newline at end of file From 513ea759f942dd01c1747cd7385e0b479e05383a Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:28:27 +0800 Subject: [PATCH 105/167] update pull docker image script. --- image/update-docker-images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/image/update-docker-images.sh b/image/update-docker-images.sh index b135a9d..95c7d1e 100755 --- a/image/update-docker-images.sh +++ b/image/update-docker-images.sh @@ -33,5 +33,6 @@ docker pull dockerhub.qingcloud.com/elastic/kibana:5.5.1 docker pull dockerhub.qingcloud.com/google_containers/elasticsearch:v5.5.1 docker pull dockerhub.qingcloud.com/qingcloud/elasticsearch-curator:5.1.1 docker pull dockerhub.qingcloud.com/qingcloud/file-sync:0.7 +docker pull dockerhub.qingcloud.com/busybox:1.27.1 docker pull dockerhub.qingcloud.com/qingcloud/qingcloud-volume-provisioner docker pull dockerhub.qingcloud.com/qingcloud/qingcloud-cloud-controller-manager \ No newline at end of file From 6acf17cfa95968c534b41752e2071a6e03e9d8bd Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:28:44 +0800 Subject: [PATCH 106/167] rename kube-proxy --- k8s/addons/kube-proxy/{kube-proxy.yaml => kube-proxy-ds.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename k8s/addons/kube-proxy/{kube-proxy.yaml => kube-proxy-ds.yaml} (100%) diff --git a/k8s/addons/kube-proxy/kube-proxy.yaml b/k8s/addons/kube-proxy/kube-proxy-ds.yaml similarity index 100% rename from k8s/addons/kube-proxy/kube-proxy.yaml rename to k8s/addons/kube-proxy/kube-proxy-ds.yaml From 23a3821175c664e0f1e5e00e2f0f8a055ac27123 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:37:31 +0800 Subject: [PATCH 107/167] fix controller manager flexlog --- k8s/manifests/kube-controller-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/manifests/kube-controller-manager.yaml b/k8s/manifests/kube-controller-manager.yaml index 0bc39ea..84d5952 100644 --- a/k8s/manifests/kube-controller-manager.yaml +++ b/k8s/manifests/kube-controller-manager.yaml @@ -57,7 +57,7 @@ spec: readOnly: true - name: flexlog image: dockerhub.qingcloud.com/busybox:1.27.1 - args: [/bin/sh, -c, 'tail -n+1 -F --retry /var/log/qingcloud-flex-volume/flex-volume.INFO'] + args: [/bin/sh, -c, 'tail -n+1 -F /var/log/qingcloud-flex-volume/flex-volume.INFO'] volumeMounts: - name: flexlog mountPath: /var/log/qingcloud-flex-volume From af5b421468ff9bff9eb5e138b8fb3a3633e39d19 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:42:54 +0800 Subject: [PATCH 108/167] fix fluent-bit ds --- k8s/addons/monitor/fluentbit-ds.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/addons/monitor/fluentbit-ds.yaml b/k8s/addons/monitor/fluentbit-ds.yaml index ad9b4b1..acc9ce0 100644 --- a/k8s/addons/monitor/fluentbit-ds.yaml +++ b/k8s/addons/monitor/fluentbit-ds.yaml @@ -72,10 +72,10 @@ spec: path: /run/log/journal/ - name: fluent-bit-config configMap: - name: fluent-bit + name: fluent-bit - name: fluent-bit-extend-config configMap: - name: fluent-bit-extend + name: fluent-bit-extend tolerations: - key: "CriticalAddonsOnly" operator: "Exists" From fcac49149682b283729f8528c0f1a1440649409f Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 17:58:12 +0800 Subject: [PATCH 109/167] set extend config path. --- k8s/addons/monitor/fluentbit-ds.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/addons/monitor/fluentbit-ds.yaml b/k8s/addons/monitor/fluentbit-ds.yaml index acc9ce0..bf5d8a2 100644 --- a/k8s/addons/monitor/fluentbit-ds.yaml +++ b/k8s/addons/monitor/fluentbit-ds.yaml @@ -76,6 +76,9 @@ spec: - name: fluent-bit-extend-config configMap: name: fluent-bit-extend + items: + - key: extend.conf + path: extend.conf tolerations: - key: "CriticalAddonsOnly" operator: "Exists" From b043073dfc7b7c0a451d4f621b157c6f533d6aa3 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 18:07:47 +0800 Subject: [PATCH 110/167] update fluentbit config --- k8s/addons/monitor/fluentbit-cm.yaml | 2 +- k8s/addons/monitor/fluentbit-ds.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index fcf3f57..874057e 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -47,7 +47,7 @@ data: Type log Logstash_Format On - @INCLUDE extend.conf + @INCLUDE /fluent-bit/etc.d/extend.conf parsers.conf: | [PARSER] Name json-test diff --git a/k8s/addons/monitor/fluentbit-ds.yaml b/k8s/addons/monitor/fluentbit-ds.yaml index bf5d8a2..533c0e5 100644 --- a/k8s/addons/monitor/fluentbit-ds.yaml +++ b/k8s/addons/monitor/fluentbit-ds.yaml @@ -53,7 +53,7 @@ spec: mountPath: /fluent-bit/etc/ readOnly: true - name: fluent-bit-extend-config - mountPath: /fluent-bit/etc/ + mountPath: /fluent-bit/etc.d/ readOnly: true ports: - containerPort: 24224 From 4964c4005b3290628d8c6ff0dfbd70b9d489c0d2 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 18:20:51 +0800 Subject: [PATCH 111/167] fix fluentbit config. --- k8s/addons/monitor/fluentbit-cm.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index 874057e..da383f9 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -8,10 +8,10 @@ metadata: data: fluent-bit.conf: | [SERVICE] - Flush 1 - Daemon Off - Log_Level info - Parsers_File parsers.conf + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf [INPUT] Name tail @@ -47,7 +47,7 @@ data: Type log Logstash_Format On - @INCLUDE /fluent-bit/etc.d/extend.conf + parsers.conf: | [PARSER] Name json-test From 79ebe5e46861de81b743dbaf798ffacebd29e644 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 18:37:10 +0800 Subject: [PATCH 112/167] update fluentbit. --- k8s/addons/monitor/fluentbit-cm.yaml | 2 +- k8s/addons/monitor/fluentbit-ds.yaml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index da383f9..045f8b5 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -47,7 +47,7 @@ data: Type log Logstash_Format On - + parsers.conf: | [PARSER] Name json-test diff --git a/k8s/addons/monitor/fluentbit-ds.yaml b/k8s/addons/monitor/fluentbit-ds.yaml index 533c0e5..168d59b 100644 --- a/k8s/addons/monitor/fluentbit-ds.yaml +++ b/k8s/addons/monitor/fluentbit-ds.yaml @@ -51,10 +51,8 @@ spec: readOnly: true - name: fluent-bit-config mountPath: /fluent-bit/etc/ - readOnly: true - name: fluent-bit-extend-config mountPath: /fluent-bit/etc.d/ - readOnly: true ports: - containerPort: 24224 name: ui From d15916c683a3513279858b1359a77d59be9687e6 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 18:49:18 +0800 Subject: [PATCH 113/167] fix fluentbit config --- k8s/addons/monitor/fluentbit-ds.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/addons/monitor/fluentbit-ds.yaml b/k8s/addons/monitor/fluentbit-ds.yaml index 168d59b..f2a3adb 100644 --- a/k8s/addons/monitor/fluentbit-ds.yaml +++ b/k8s/addons/monitor/fluentbit-ds.yaml @@ -27,7 +27,8 @@ spec: image: dockerhub.qingcloud.com/fluent/fluent-bit:0.12 command: - /fluent-bit/bin/fluent-bit - - -c /fluent-bit/etc/fluent-bit.conf + - -c + - /fluent-bit/etc/fluent-bit.conf imagePullPolicy: IfNotPresent env: - name: FLUENT_ELASTICSEARCH_HOST From 1cf8ed39215a8e2ff9529bafc52fcce3c3e12d94 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 18:53:02 +0800 Subject: [PATCH 114/167] support include extend conf --- k8s/addons/monitor/fluentbit-cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index 045f8b5..10b44a9 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -47,7 +47,7 @@ data: Type log Logstash_Format On - + @INCLUDE /fluent-bit/etc.d/extend.conf parsers.conf: | [PARSER] Name json-test From 90bc310b5c872c9ea345defa63f1ce601aaa50ec Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 18:57:38 +0800 Subject: [PATCH 115/167] fix confd --- confd/templates/k8s/daemon.json.tmpl | 6 +++--- confd/templates/k8s/fluentbit-extend.conf.tmpl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/confd/templates/k8s/daemon.json.tmpl b/confd/templates/k8s/daemon.json.tmpl index 47bdf12..7cc6127 100644 --- a/confd/templates/k8s/daemon.json.tmpl +++ b/confd/templates/k8s/daemon.json.tmpl @@ -5,9 +5,9 @@ {{if ne $registries ""}} "insecure-registries": {{split $registries " "|toJson}}, {{end}} - {{$registry-mirrors := getv "/env/registry-mirrors"}} - {{if ne $registry-mirrors ""}} - "registry-mirrors": {{split $registry-mirrors " "|toJson}}, + {{$mirrors := getv "/env/registry-mirrors"}} + {{if ne $mirrors ""}} + "registry-mirrors": {{split $mirrors " "|toJson}}, {{end}} "storage-driver": "overlay2" } \ No newline at end of file diff --git a/confd/templates/k8s/fluentbit-extend.conf.tmpl b/confd/templates/k8s/fluentbit-extend.conf.tmpl index 75aea16..8b068fc 100644 --- a/confd/templates/k8s/fluentbit-extend.conf.tmpl +++ b/confd/templates/k8s/fluentbit-extend.conf.tmpl @@ -1,6 +1,6 @@ {{- $server := getv "/env/fluent-forward-server" -}} {{- if ne $server "" -}} -{{- parts := split $server ":" -}} +{{- $parts := split $server ":" -}} [OUTPUT] Name forward Match * From 57a6669064eb15175d1d25494b561aa76be2717a Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 19:05:34 +0800 Subject: [PATCH 116/167] fix reload fluentbit --- script/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/common.sh b/script/common.sh index b3f0fb9..680b6b5 100755 --- a/script/common.sh +++ b/script/common.sh @@ -329,7 +329,7 @@ function update_fluent_config(){ then kubectl create configmap --dry-run -o yaml fluent-bit-extend -n kube-system --from-file /etc/kubernetes/fluentbit/extend.conf | kubectl replace -f - date=$(date +%s) - sed -i 's/qingcloud\.com\/update-time:.*/qingcloud\.com\/update-time: "'${date}'"/g' /etc/kubernetes/addons/fluentbit-ds.yaml - kubectl apply -f /etc/kubernetes/addons/fluentbit-ds.yaml + sed -i 's/qingcloud\.com\/update-time:.*/qingcloud\.com\/update-time: "'${date}'"/g' /etc/kubernetes/addons/monitor/fluentbit-ds.yaml + kubectl apply -f /etc/kubernetes/addons/monitor/fluentbit-ds.yaml fi } \ No newline at end of file From 82fbf0dfd6027c0b952efd8fcea45aa8b7e265f6 Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 19:30:10 +0800 Subject: [PATCH 117/167] update fluentbit config --- k8s/addons/monitor/fluentbit-cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index 10b44a9..106dbb4 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -47,7 +47,9 @@ data: Type log Logstash_Format On + @INCLUDE /fluent-bit/etc.d/extend.conf + parsers.conf: | [PARSER] Name json-test From 7ecb1a644da4afde2fb719dbebd6dc0515d83e2f Mon Sep 17 00:00:00 2001 From: jolestar Date: Thu, 24 Aug 2017 20:02:19 +0800 Subject: [PATCH 118/167] update config --- k8s/addons/monitor/fluentbit-cm.yaml | 5 +---- script/common.sh | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index 106dbb4..e333648 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -7,6 +7,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile data: fluent-bit.conf: | + @INCLUDE /fluent-bit/etc.d/extend.conf [SERVICE] Flush 1 Daemon Off @@ -46,10 +47,6 @@ data: Index fluentbit Type log Logstash_Format On - - - @INCLUDE /fluent-bit/etc.d/extend.conf - parsers.conf: | [PARSER] Name json-test diff --git a/script/common.sh b/script/common.sh index 680b6b5..63d53aa 100755 --- a/script/common.sh +++ b/script/common.sh @@ -327,7 +327,7 @@ function upgrade_docker(){ function update_fluent_config(){ if [ "${HOST_ROLE}" == "master" ] then - kubectl create configmap --dry-run -o yaml fluent-bit-extend -n kube-system --from-file /etc/kubernetes/fluentbit/extend.conf | kubectl replace -f - + kubectl create configmap --dry-run -o yaml fluent-bit-extend -n kube-system --from-file /etc/kubernetes/fluentbit/extend.conf | kubectl replace -n kube-system -f - date=$(date +%s) sed -i 's/qingcloud\.com\/update-time:.*/qingcloud\.com\/update-time: "'${date}'"/g' /etc/kubernetes/addons/monitor/fluentbit-ds.yaml kubectl apply -f /etc/kubernetes/addons/monitor/fluentbit-ds.yaml From db2451d341c1c8601f1273de56fe75492318f4b4 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 10:14:01 +0800 Subject: [PATCH 119/167] update fluentbit config --- k8s/addons/monitor/fluentbit-cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/addons/monitor/fluentbit-cm.yaml b/k8s/addons/monitor/fluentbit-cm.yaml index e333648..c3a6796 100644 --- a/k8s/addons/monitor/fluentbit-cm.yaml +++ b/k8s/addons/monitor/fluentbit-cm.yaml @@ -47,6 +47,8 @@ data: Index fluentbit Type log Logstash_Format On + Include_Tag_Key On + Tag_Key flb-log-tag parsers.conf: | [PARSER] Name json-test From b3383cab3f61f0791bab7011af2028c746fe8456 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 10:42:34 +0800 Subject: [PATCH 120/167] enhancement health check. --- script/check-master.sh | 2 +- script/check-node.sh | 8 +++++++- script/common.sh | 5 +++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/script/check-master.sh b/script/check-master.sh index 5da0736..5c480ba 100755 --- a/script/check-master.sh +++ b/script/check-master.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -if systemctl is-active kubelet +if systemctl is-active kubelet && [ $(get_node_status) == "True" ] then if [ "$(curl --silent --fail http://localhost:8080/healthz)" = "ok" ] then diff --git a/script/check-node.sh b/script/check-node.sh index 84c93c0..9895b35 100755 --- a/script/check-node.sh +++ b/script/check-node.sh @@ -1,2 +1,8 @@ #!/usr/bin/env bash -systemctl is-active kubelet \ No newline at end of file + +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +K8S_HOME=$(dirname "${SCRIPTPATH}") + +source "${K8S_HOME}/script/common.sh" + +systemctl is-active kubelet && $(get_node_status) == "True" \ No newline at end of file diff --git a/script/common.sh b/script/common.sh index 63d53aa..312fa0a 100755 --- a/script/common.sh +++ b/script/common.sh @@ -332,4 +332,9 @@ function update_fluent_config(){ sed -i 's/qingcloud\.com\/update-time:.*/qingcloud\.com\/update-time: "'${date}'"/g' /etc/kubernetes/addons/monitor/fluentbit-ds.yaml kubectl apply -f /etc/kubernetes/addons/monitor/fluentbit-ds.yaml fi +} + +function get_node_status(){ + status=$(kubectl get nodes/${HOST_INSTANCE_ID} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') + echo status } \ No newline at end of file From d713953831002d85ea400875db57fff9d00acb40 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 10:47:24 +0800 Subject: [PATCH 121/167] enhancement --- script/check-master.sh | 14 ++++++++++++-- script/check-node.sh | 10 +++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/script/check-master.sh b/script/check-master.sh index 5c480ba..1222e49 100755 --- a/script/check-master.sh +++ b/script/check-master.sh @@ -1,9 +1,19 @@ #!/usr/bin/env bash -if systemctl is-active kubelet && [ $(get_node_status) == "True" ] + +SCRIPTPATH=$( cd $(dirname $0) ; pwd -P ) +K8S_HOME=$(dirname "${SCRIPTPATH}") + +source "${K8S_HOME}/script/common.sh" + +if systemctl is-active kubelet then if [ "$(curl --silent --fail http://localhost:8080/healthz)" = "ok" ] then - exit 0 + status=$(get_node_status) + if [ "${status}" == "True" ] + then + exit 0 + fi fi fi exit 1 \ No newline at end of file diff --git a/script/check-node.sh b/script/check-node.sh index 9895b35..5eed5ce 100755 --- a/script/check-node.sh +++ b/script/check-node.sh @@ -5,4 +5,12 @@ K8S_HOME=$(dirname "${SCRIPTPATH}") source "${K8S_HOME}/script/common.sh" -systemctl is-active kubelet && $(get_node_status) == "True" \ No newline at end of file +if systemctl is-active kubelet +then + status=$(get_node_status) + if [ "${status}" == "True" ] + then + exit 0 + fi +fi +exit 1 \ No newline at end of file From 77013073db7e4d4edac63a3c854ef622b187279c Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 10:50:02 +0800 Subject: [PATCH 122/167] enhancement health check. --- script/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/common.sh b/script/common.sh index 312fa0a..50c9567 100755 --- a/script/common.sh +++ b/script/common.sh @@ -336,5 +336,5 @@ function update_fluent_config(){ function get_node_status(){ status=$(kubectl get nodes/${HOST_INSTANCE_ID} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') - echo status + echo ${status} } \ No newline at end of file From 812ca04203ade4236562729b8543ae643694d7bc Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 10:55:40 +0800 Subject: [PATCH 123/167] replace mykubectl to kubectl --- script/common.sh | 34 +++++++++++++++------------------- script/reload-curator.sh | 4 ++-- script/scale-in.sh | 2 +- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/script/common.sh b/script/common.sh index 50c9567..06c7c79 100755 --- a/script/common.sh +++ b/script/common.sh @@ -37,10 +37,6 @@ timestamp() { date +"%s" } -function mykubectl(){ - kubectl --kubeconfig='/etc/kubernetes/kubelet.conf' $* -} - function ensure_dir(){ if [ ! -d /root/.kube ]; then mkdir /root/.kube @@ -132,7 +128,7 @@ function process_addons(){ } function scale_es(){ - retry mykubectl scale --replicas=$1 statefulsets/elasticsearch-logging-v1 -n kube-system + retry kubeclt scale --replicas=$1 statefulsets/elasticsearch-logging-v1 -n kube-system } function join_node(){ @@ -175,47 +171,47 @@ function wait_apiserver(){ } function wait_system_pod(){ - while [ "$(mykubectl get pods -o custom-columns=STATUS:.status.phase --no-headers=true -n kube-system|uniq)" != "Running" ] + while [ "$(kubeclt get pods -o custom-columns=STATUS:.status.phase --no-headers=true -n kube-system|uniq)" != "Running" ] do echo "wait all kube-system pods running, no ready pods: " - mykubectl get pods --no-headers=true -n kube-system |grep -v Running + kubeclt get pods --no-headers=true -n kube-system |grep -v Running sleep 2 done } function train_master(){ - retry mykubectl taint nodes ${MASTER_INSTANCE_ID} --overwrite dedicated=master:NoSchedule + retry kubeclt taint nodes ${MASTER_INSTANCE_ID} --overwrite dedicated=master:NoSchedule } function train_node(){ if [ "${HOST_ROLE}" == "log" ] then - retry mykubectl taint nodes ${HOST_INSTANCE_ID} --overwrite dedicated=log:NoSchedule + retry kubeclt taint nodes ${HOST_INSTANCE_ID} --overwrite dedicated=log:NoSchedule fi } function cordon_all(){ for node in $(kubectl get nodes --no-headers=true -o custom-columns=name:.metadata.name) do - mykubectl cordon $node + kubeclt cordon $node done } function cordon_node(){ - mykubectl cordon ${HOST_INSTANCE_ID} + kubeclt cordon ${HOST_INSTANCE_ID} return $? } function uncordon_all(){ for node in $(kubectl get nodes --no-headers=true -o custom-columns=name:.metadata.name) do - mykubectl uncordon $node + kubeclt uncordon $node done } function clean_addons(){ - echo "stop addons-manager" && rm /data/kubernetes/manifests/kube-addon-manager.yaml && mykubectl delete --ignore-not-found=true "pods/kube-addon-manager-${MASTER_INSTANCE_ID}" -n kube-system - mykubectl delete --timeout=60s --force --now -R -f /data/kubernetes/addons/ + echo "stop addons-manager" && rm /data/kubernetes/manifests/kube-addon-manager.yaml && kubeclt delete --ignore-not-found=true "pods/kube-addon-manager-${MASTER_INSTANCE_ID}" -n kube-system + kubeclt delete --timeout=60s --force --now -R -f /data/kubernetes/addons/ echo "clean addons" && rm -rf /data/kubernetes/addons } @@ -232,23 +228,23 @@ function clean_static_pod(){ function clean_pod(){ clean_addons - for namespace in $(mykubectl get namespaces --no-headers=true -o custom-columns=name:.metadata.name) + for namespace in $(kubeclt get namespaces --no-headers=true -o custom-columns=name:.metadata.name) do if [ "${namespace}" != "kube-system" ] then - mykubectl delete --force --now --all --timeout=60s pods -n ${namespace} + kubeclt delete --force --now --all --timeout=60s pods -n ${namespace} fi done local n=1 local max=6 - while mykubectl get pods --no-headers=true --all-namespaces |grep Terminating + while kubeclt get pods --no-headers=true --all-namespaces |grep Terminating do if [[ $n -lt $max ]]; then echo "break wait terminating." break fi echo "wait all pods terminating:" - mykubectl get pods --no-headers=true --all-namespaces |grep Terminating + kubeclt get pods --no-headers=true --all-namespaces |grep Terminating sleep 5 ((n++)) done @@ -256,7 +252,7 @@ function clean_pod(){ } function drain_node(){ - mykubectl drain --delete-local-data=true --ignore-daemonsets=true --force $1 + kubeclt drain --delete-local-data=true --ignore-daemonsets=true --force $1 return $? } diff --git a/script/reload-curator.sh b/script/reload-curator.sh index 711281f..1c89a32 100755 --- a/script/reload-curator.sh +++ b/script/reload-curator.sh @@ -8,10 +8,10 @@ source "${K8S_HOME}/script/common.sh" systemctl is-active kubelet >/dev/null 2>&1 && { if [ "${HOST_ROLE}" == "master" ] then - mykubectl delete job -l k8s-app=clean-log -n kube-system + kubeclt delete job -l k8s-app=clean-log -n kube-system JOB_ID=$(timestamp) sed 's/${JOB_ID}/'"${JOB_ID}"'/g' /etc/kubernetes/addons/qingcloud/clean-log-job.yaml > /tmp/clean-log-job-${JOB_ID}.yaml - retry mykubectl create -f /tmp/clean-log-job-${JOB_ID}.yaml + retry kubeclt create -f /tmp/clean-log-job-${JOB_ID}.yaml rm /tmp/clean-log-job-${JOB_ID}.yaml fi } diff --git a/script/scale-in.sh b/script/scale-in.sh index 8cdc2ae..f6f8a2d 100755 --- a/script/scale-in.sh +++ b/script/scale-in.sh @@ -11,7 +11,7 @@ do if [ "$n" != "" ] then drain_node ${n} - mykubectl delete node/${n} + kubeclt delete node/${n} fi done exit 0 \ No newline at end of file From fd32c8022d598ff651bbc8a67f4d04397e2df396 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 11:00:50 +0800 Subject: [PATCH 124/167] clear unused base function. --- script/common.sh | 50 +------------------------------------------ script/init-master.sh | 2 +- 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/script/common.sh b/script/common.sh index 06c7c79..40e6793 100755 --- a/script/common.sh +++ b/script/common.sh @@ -170,17 +170,8 @@ function wait_apiserver(){ done; } -function wait_system_pod(){ - while [ "$(kubeclt get pods -o custom-columns=STATUS:.status.phase --no-headers=true -n kube-system|uniq)" != "Running" ] - do - echo "wait all kube-system pods running, no ready pods: " - kubeclt get pods --no-headers=true -n kube-system |grep -v Running - sleep 2 - done -} - function train_master(){ - retry kubeclt taint nodes ${MASTER_INSTANCE_ID} --overwrite dedicated=master:NoSchedule + retry kubeadm alpha phase mark-master ${MASTER_INSTANCE_ID} } function train_node(){ @@ -217,38 +208,6 @@ function clean_addons(){ function clean_static_pod(){ echo "clean static pod" && rm -rf /data/kubernetes/manifests - sleep 10 - if [ "$(docker ps -aq)" != "" ] - then - echo "wait all containers to be rm:" - docker ps -a - sleep 10 - fi -} - -function clean_pod(){ - clean_addons - for namespace in $(kubeclt get namespaces --no-headers=true -o custom-columns=name:.metadata.name) - do - if [ "${namespace}" != "kube-system" ] - then - kubeclt delete --force --now --all --timeout=60s pods -n ${namespace} - fi - done - local n=1 - local max=6 - while kubeclt get pods --no-headers=true --all-namespaces |grep Terminating - do - if [[ $n -lt $max ]]; then - echo "break wait terminating." - break - fi - echo "wait all pods terminating:" - kubeclt get pods --no-headers=true --all-namespaces |grep Terminating - sleep 5 - ((n++)) - done - clean_static_pod } function drain_node(){ @@ -293,13 +252,6 @@ function flush_iptables(){ iptables --flush } -function wait_qingcloudvolume_detach(){ - while df |grep "qingcloud-volume" > /dev/null; - do - echo "waiting qingcloud-volume detach" && df |grep "qingcloud-volume" && sleep 2 - done -} - function docker_login(){ if [ ! -z "${ENV_DOCKERHUB_USERNAME}" ] && [ ! -z "${ENV_DOCKERHUB_PASSWORD}" ] then diff --git a/script/init-master.sh b/script/init-master.sh index e67412a..78ae0e2 100755 --- a/script/init-master.sh +++ b/script/init-master.sh @@ -21,7 +21,7 @@ systemctl start kubelet wait_kubelet wait_apiserver -kubeadm alpha phase mark-master ${MASTER_INSTANCE_ID} +train_master retry kubeadm token create ${init_token} --ttl 0 --description "the default kubeadm init token" --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token node allow-post-csrs --kubeconfig /etc/kubernetes/admin.conf retry kubeadm alpha phase bootstrap-token node allow-auto-approve --kubeconfig /etc/kubernetes/admin.conf From 1b88484f72c9f95cedf22e8406307aa747a18248 Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 11:49:23 +0800 Subject: [PATCH 125/167] update hyperkube_version --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index c734f72..35f214e 100644 --- a/version +++ b/version @@ -1 +1 @@ -HYPERKUBE_VERSION="v1.7.3" +HYPERKUBE_VERSION="v1.7.4" From 53c7070a6a7f4da4fcc0cab41bccfb9add69f97b Mon Sep 17 00:00:00 2001 From: jolestar Date: Fri, 25 Aug 2017 12:03:00 +0800 Subject: [PATCH 126/167] update heapster version. --- docker/addon-manager/CHANGELOG.md | 47 ---- docker/addon-manager/Dockerfile | 21 -- docker/addon-manager/Makefile | 58 ----- docker/addon-manager/README.md | 61 ----- docker/addon-manager/kube-addons.sh | 233 ------------------ docker/addon-manager/namespace.yaml | 4 - docker/fluentbit/README.md | 48 ---- docker/fluentbit/build.sh | 5 + docker/fluentbit/docker-image/Dockerfile | 8 - docker/fluentbit/docker-image/fluent-bit.conf | 33 --- docker/fluentbit/docker-image/parsers.conf | 45 ---- docker/heapster/build.sh | 6 +- image/update-docker-images.sh | 6 +- k8s/addons/monitor/heapster-deployment.yaml | 16 +- 14 files changed, 19 insertions(+), 572 deletions(-) delete mode 100644 docker/addon-manager/CHANGELOG.md delete mode 100644 docker/addon-manager/Dockerfile delete mode 100644 docker/addon-manager/Makefile delete mode 100644 docker/addon-manager/README.md delete mode 100755 docker/addon-manager/kube-addons.sh delete mode 100644 docker/addon-manager/namespace.yaml delete mode 100644 docker/fluentbit/README.md create mode 100644 docker/fluentbit/build.sh delete mode 100644 docker/fluentbit/docker-image/Dockerfile delete mode 100644 docker/fluentbit/docker-image/fluent-bit.conf delete mode 100644 docker/fluentbit/docker-image/parsers.conf diff --git a/docker/addon-manager/CHANGELOG.md b/docker/addon-manager/CHANGELOG.md deleted file mode 100644 index c95c973..0000000 --- a/docker/addon-manager/CHANGELOG.md +++ /dev/null @@ -1,47 +0,0 @@ -### Version 6.4-beta.1 (Wed March 8 2017 Zihong Zheng ) - - Create EnsureExists class addons before Reconcile class addons. - -### Version 6.4-alpha.3 (Fri February 24 2017 Zihong Zheng ) - - Support 'ensure exist' class addon and use addon-manager specific label. - -### Version 6.4-alpha.2 (Wed February 16 2017 Zihong Zheng ) - - Update kubectl to v1.6.0-alpha.2 to use HPA in autoscaling/v1 instead of extensions/v1beta1. - -### Version 6.4-alpha.1 (Wed February 1 2017 Zihong Zheng ) - - Update kubectl to v1.6.0-alpha.1 for supporting optional ConfigMap. - -### Version 6.3 (Fri January 27 2017 Lucas Käldström ) - - Updated the arm base image to `armhf/busybox` and now using qemu v2.7 for emulation. - -### Version 6.2 (Thu January 12 2017 Zihong Zheng ) - - Update kubectl to the stable version. - -### Version 6.1 (Tue November 29 2016 Zihong Zheng ) - - Support pruning old Deployments. - -### Version 6.0 (Fri November 18 2016 Zihong Zheng ) - - Upgrade Addon Manager to use `kubectl apply`. - -### Version 5.2 (Wed October 26 2016 Zihong Zheng ) - - Added support for ConfigMap and upgraded kubectl version to v1.4.4 (pr #35255) - -### Version 5.1 (Mon Jul 4 2016 Marek Grabowski ) - - Fixed the way addon-manager handles non-namespaced objects - -### Version 5 (Fri Jun 24 2016 Jerzy Szczepkowski @jszczepkowski) - - Added PetSet support to addon manager - -### Version 4 (Tue Jun 21 2016 Mike Danese @mikedanese) - - Increased addon check interval - -### Version 3 (Sun Jun 19 2016 Lucas Käldström @luxas) - - Bumped up addon-manager to v3 - -### Version 2 (Fri May 20 2016 Lucas Käldström @luxas) - - Removed deprecated kubectl command, added support for DaemonSets - -### Version 1 (Thu May 5 2016 Mike Danese @mikedanese) - - Run kube-addon-manager in a pod - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/addon-manager/CHANGELOG.md?pixel)]() diff --git a/docker/addon-manager/Dockerfile b/docker/addon-manager/Dockerfile deleted file mode 100644 index c8961bd..0000000 --- a/docker/addon-manager/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM BASEIMAGE - -ADD kube-addons.sh /opt/ -ADD namespace.yaml /opt/ -ADD kubectl /usr/local/bin/ - -CMD ["/opt/kube-addons.sh"] diff --git a/docker/addon-manager/Makefile b/docker/addon-manager/Makefile deleted file mode 100644 index b49dd27..0000000 --- a/docker/addon-manager/Makefile +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -IMAGE=dockerhub.qingcloud.com/google_containers/kube-addon-manager -ARCH?=amd64 -TEMP_DIR:=$(shell mktemp -d) -VERSION=v1.5.4.qingcloud -KUBECTL_VERSION?=v1.5.4 - -ifeq ($(ARCH),amd64) - BASEIMAGE?=bashell/alpine-bash -endif -ifeq ($(ARCH),arm) - BASEIMAGE?=armhf/debian -endif -ifeq ($(ARCH),arm64) - BASEIMAGE?=aarch64/debian -endif -ifeq ($(ARCH),ppc64le) - BASEIMAGE?=ppc64le/debian -endif -ifeq ($(ARCH),s390x) - BASEIMAGE?=s390x/debian -endif - -.PHONY: build push - -all: build - -build: - cp ./* $(TEMP_DIR) - curl -sSL --retry 5 https://pek3a.qingstor.com/jolestar/k8s/release/bin/linux/amd64/kubectl > $(TEMP_DIR)/kubectl - chmod +x $(TEMP_DIR)/kubectl - cd $(TEMP_DIR) && sed -i.back "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile - docker build --pull -t $(IMAGE)-$(ARCH):$(VERSION) $(TEMP_DIR) - -push: build - docker -- push $(IMAGE)-$(ARCH):$(VERSION) -ifeq ($(ARCH),amd64) - # Backward compatibility. TODO: deprecate this image tag - docker rmi $(IMAGE):$(VERSION) || true - docker tag $(IMAGE)-$(ARCH):$(VERSION) $(IMAGE):$(VERSION) - gcloud docker -- push $(IMAGE):$(VERSION) -endif - -clean: - docker rmi -f $(IMAGE)-$(ARCH):$(VERSION) diff --git a/docker/addon-manager/README.md b/docker/addon-manager/README.md deleted file mode 100644 index b9da703..0000000 --- a/docker/addon-manager/README.md +++ /dev/null @@ -1,61 +0,0 @@ -### Addon-manager - -addon-manager manages two classes of addons with given template files. -- Addons with label `addonmanager.kubernetes.io/mode=Reconcile` will be periodically -reconciled. Direct manipulation to these addons through apiserver is discouraged because -addon-manager will bring them back to the original state. In particular: - - Addon will be re-created if it is deleted. - - Addon will be reconfigured to the state given by the supplied fields in the template - file periodically. - - Addon will be deleted when its manifest file is deleted. -- Addons with label `addonmanager.kubernetes.io/mode=EnsureExists` will be checked for -existence only. Users can edit these addons as they want. In particular: - - Addon will only be created/re-created with the given template file when there is no - instance of the resource with that name. - - Addon will not be deleted when the manifest file is deleted. - -Notes: -- Label `kubernetes.io/cluster-service=true` is deprecated (only for Addon Manager). -In future release (after one year), Addon Manager may not respect it anymore. Addons -have this label but without `addonmanager.kubernetes.io/mode=EnsureExists` will be -treated as "reconcile class addons" for now. -- Resources under $ADDON_PATH (default `/etc/kubernetes/addons/`) needs to have either one -of these two labels. Meanwhile namespaced resources need to be in `kube-system` namespace. -Otherwise it will be omitted. -- The above label and namespace rule does not stand for `/opt/namespace.yaml` and -resources under `/etc/kubernetes/admission-controls/`. addon-manager will attempt to -create them regardless during startup. - -#### How to release - -The `addon-manager` is built for multiple architectures. - -1. Change something in the source -2. Bump `VERSION` in the `Makefile` -3. Bump `KUBECTL_VERSION` in the `Makefile` if required -4. Build the `amd64` image and test it on a cluster -5. Push all images - -```console -# Build for linux/amd64 (default) -$ make push ARCH=amd64 -# ---> gcr.io/google-containers/kube-addon-manager-amd64:VERSION -# ---> gcr.io/google-containers/kube-addon-manager:VERSION (image with backwards-compatible naming) - -$ make push ARCH=arm -# ---> gcr.io/google-containers/kube-addon-manager-arm:VERSION - -$ make push ARCH=arm64 -# ---> gcr.io/google-containers/kube-addon-manager-arm64:VERSION - -$ make push ARCH=ppc64le -# ---> gcr.io/google-containers/kube-addon-manager-ppc64le:VERSION - -$ make push ARCH=s390x -# ---> gcr.io/google-containers/kube-addon-manager-s390x:VERSION -``` - -If you don't want to push the images, run `make` or `make build` instead - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/addon-manager/README.md?pixel)]() diff --git a/docker/addon-manager/kube-addons.sh b/docker/addon-manager/kube-addons.sh deleted file mode 100755 index ec29602..0000000 --- a/docker/addon-manager/kube-addons.sh +++ /dev/null @@ -1,233 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# LIMITATIONS -# 1. Exit code is probably not always correct. -# 2. There are no unittests. -# 3. Will not work if the total length of paths to addons is greater than -# bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE. - -# cosmetic improvements to be done -# 1. Improve the log function; add timestamp, file name, etc. -# 2. Logging doesn't work from files that print things out. -# 3. Kubectl prints the output to stderr (the output should be captured and then -# logged) - -# The business logic for whether a given object should be created -# was already enforced by salt, and /etc/kubernetes/addons is the -# managed result is of that. Start everything below that directory. -KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl} -KUBECTL_OPTS=${KUBECTL_OPTS:-} - -ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-60} -ADDON_PATH=${ADDON_PATH:-/etc/kubernetes/addons} - -SYSTEM_NAMESPACE=kube-system - -# Addons could use this label with two modes: -# - ADDON_MANAGER_LABEL=Reconcile -# - ADDON_MANAGER_LABEL=EnsureExists -ADDON_MANAGER_LABEL="addonmanager.kubernetes.io/mode" -# This label is deprecated (only for Addon Manager). In future release -# addon-manager may not respect it anymore. Addons with -# CLUSTER_SERVICE_LABEL=true and without ADDON_MANAGER_LABEL=EnsureExists -# will be reconciled for now. -CLUSTER_SERVICE_LABEL="kubernetes.io/cluster-service" - -# Remember that you can't log from functions that print some output (because -# logs are also printed on stdout). -# $1 level -# $2 message -function log() { - # manage log levels manually here - - # add the timestamp if you find it useful - case $1 in - DB3 ) -# echo "$1: $2" - ;; - DB2 ) -# echo "$1: $2" - ;; - DBG ) -# echo "$1: $2" - ;; - INFO ) - echo "$1: $2" - ;; - WRN ) - echo "$1: $2" - ;; - ERR ) - echo "$1: $2" - ;; - * ) - echo "INVALID_LOG_LEVEL $1: $2" - ;; - esac -} - -# $1 filename of addon to start. -# $2 count of tries to start the addon. -# $3 delay in seconds between two consecutive tries -# $4 namespace -function start_addon() { - local -r addon_filename=$1; - local -r tries=$2; - local -r delay=$3; - local -r namespace=$4 - - create_resource_from_string "$(cat ${addon_filename})" "${tries}" "${delay}" "${addon_filename}" "${namespace}" -} - -# $1 string with json or yaml. -# $2 count of tries to start the addon. -# $3 delay in seconds between two consecutive tries -# $4 name of this object to use when logging about it. -# $5 namespace for this object -function create_resource_from_string() { - local -r config_string=$1; - local tries=$2; - local -r delay=$3; - local -r config_name=$4; - local -r namespace=$5; - while [ ${tries} -gt 0 ]; do - echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} --namespace="${namespace}" apply -f - && \ - log INFO "== Successfully started ${config_name} in namespace ${namespace} at $(date -Is)" && \ - return 0; - let tries=tries-1; - log WRN "== Failed to start ${config_name} in namespace ${namespace} at $(date -Is). ${tries} tries remaining. ==" - sleep ${delay}; - done - return 1; -} - -# $1 resource type. -function annotate_addons() { - local -r obj_type=$1; - - # Annotate to objects already have this annotation should fail. - # Only try once for now. - ${KUBECTL} ${KUBECTL_OPTS} annotate ${obj_type} --namespace=${SYSTEM_NAMESPACE} -l ${CLUSTER_SERVICE_LABEL}=true \ - kubectl.kubernetes.io/last-applied-configuration='' --overwrite=false - - if [[ $? -eq 0 ]]; then - log INFO "== Annotate resources completed successfully at $(date -Is) ==" - else - log WRN "== Annotate resources completed with errors at $(date -Is) ==" - fi -} - -# $1 enable --prune or not. -function reconcile_addons() { - local -r enable_prune=$1; - - # TODO: Remove the first command in future release. - # Adding this for backward compatibility. Old addons have CLUSTER_SERVICE_LABEL=true and don't have - # ADDON_MANAGER_LABEL=EnsureExists will still be reconciled. - # Filter out `configured` message to not noisily log. - # `created`, `pruned` and errors will be logged. - log INFO "== Reconciling with deprecated label ==" - ${KUBECTL} ${KUBECTL_OPTS} apply --namespace=${SYSTEM_NAMESPACE} -f ${ADDON_PATH} \ - -l ${CLUSTER_SERVICE_LABEL}=true,${ADDON_MANAGER_LABEL}!=EnsureExists \ - --prune=${enable_prune} --recursive | grep -v configured - - log INFO "== Reconciling with addon-manager label ==" - ${KUBECTL} ${KUBECTL_OPTS} apply --namespace=${SYSTEM_NAMESPACE} -f ${ADDON_PATH} \ - -l ${CLUSTER_SERVICE_LABEL}!=true,${ADDON_MANAGER_LABEL}=Reconcile \ - --prune=${enable_prune} --recursive | grep -v configured - - log INFO "== Kubernetes addon reconcile completed at $(date -Is) ==" -} - -function ensure_addons() { - # Create objects already exist should fail. - # Filter out `AlreadyExists` message to not noisily log. - ${KUBECTL} ${KUBECTL_OPTS} apply --namespace=${SYSTEM_NAMESPACE} -f ${ADDON_PATH} \ - -l ${ADDON_MANAGER_LABEL}=EnsureExists --recursive 2>&1 | grep -v AlreadyExists - - log INFO "== Kubernetes addon ensure completed at $(date -Is) ==" -} - -# The business logic for whether a given object should be created -# was already enforced by salt, and /etc/kubernetes/addons is the -# managed result is of that. Start everything below that directory. -log INFO "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} ==" - -# Create the namespace that will be used to host the cluster-level add-ons. -start_addon /opt/namespace.yaml 100 10 "" & - -# Wait for the default service account to be created in the kube-system namespace. -token_found="" -while [ -z "${token_found}" ]; do - sleep .5 - token_found=$(${KUBECTL} ${KUBECTL_OPTS} get --namespace="${SYSTEM_NAMESPACE}" serviceaccount default -o go-template="{{with index .secrets 0}}{{.name}}{{end}}") - if [[ $? -ne 0 ]]; then - token_found=""; - log WRN "== Error getting default service account, retry in 0.5 second ==" - fi -done - -log INFO "== Default service account in the ${SYSTEM_NAMESPACE} namespace has token ${token_found} ==" - -# Create admission_control objects if defined before any other addon services. If the limits -# are defined in a namespace other than default, we should still create the limits for the -# default namespace. -for obj in $(find /etc/kubernetes/admission-controls \( -name \*.yaml -o -name \*.json \)); do - start_addon "${obj}" 100 10 default & - log INFO "++ obj ${obj} is created ++" -done - -# TODO: The annotate and spin up parts should be removed after 1.6 is released. - -# Fake the "kubectl.kubernetes.io/last-applied-configuration" annotation on old resources -# in order to clean them up by `kubectl apply --prune`. -# RCs have to be annotated for 1.4->1.5+ upgrade, because we migrated from RCs to deployments for all default addons in 1.5. -# Other types resources will also need this fake annotation if their names are changed, -# otherwise they would be leaked during upgrade. -log INFO "== Annotating the old addon resources at $(date -Is) ==" -annotate_addons ReplicationController -annotate_addons Deployment - -# Create new addon resources by apply (with --prune=false). -# The old RCs will not fight for pods created by new Deployments with the same label because the `controllerRef` feature. -# The new Deployments will not fight for pods created by old RCs with the same label because the additional `pod-template-hash` label. -# Apply will fail if some fields are modified but not are allowed, in that case should bump up addon version and name (e.g. handle externally). -log INFO "== Executing apply to spin up new addon resources at $(date -Is) ==" -ensure_addons -reconcile_addons false - -# Wait for new addons to be spinned up before delete old resources -log INFO "== Wait for addons to be spinned up at $(date -Is) ==" -sleep ${ADDON_CHECK_INTERVAL_SEC} - -# Start the apply loop. -# Check if the configuration has changed recently - in case the user -# created/updated/deleted the files on the master. -log INFO "== Entering periodical apply loop at $(date -Is) ==" -while true; do - start_sec=$(date +"%s") - # Only print stderr for the readability of logging - ensure_addons - reconcile_addons true - end_sec=$(date +"%s") - len_sec=$((${end_sec}-${start_sec})) - # subtract the time passed from the sleep time - if [[ ${len_sec} -lt ${ADDON_CHECK_INTERVAL_SEC} ]]; then - sleep_time=$((${ADDON_CHECK_INTERVAL_SEC}-${len_sec})) - sleep ${sleep_time} - fi -done diff --git a/docker/addon-manager/namespace.yaml b/docker/addon-manager/namespace.yaml deleted file mode 100644 index 986f4b4..0000000 --- a/docker/addon-manager/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kube-system diff --git a/docker/fluentbit/README.md b/docker/fluentbit/README.md deleted file mode 100644 index c6e1d90..0000000 --- a/docker/fluentbit/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Help to Test! - -[Fluent Bit](http://fluentbit.io) v0.11 was just released at the end of March 2017 and it comes with support for Kubernetes logging needs: it can consume logs and enrich them with proper Kubernetes metadata, retrieved from the API server of course. - -Our goal is to make logging cheaper in terms of memory consumption. In order to accomplish this we need to deal with different factors, not only processing incoming logs but also _formatting_ the data for our backends like Elasticsearch, which can be a little expensive as it requires a JSON representation. - -> Note: Fluent Bit uses a binary representation for logs, converting this to JSON requires enough memory for the process. - -## Getting started - -Fluent Bit must be deployed as a DaemonSet, on that way it will be available on every node of your Kubernetes cluster. - -This repository contains two Yaml DaemonSet files: - -| Yaml file | Description | -|-----------|-------------| -| [fluent-bit-daemonset-elasticsearch](fluent-bit-daemonset-elasticsearch.yaml) | deploys a stable release of Fluent Bit. | - -The current DaemonSet points to this specific Docker Hub image: - -[0.11](https://hub.docker.com/r/fluent/fluent-bit-kubernetes-daemonset/tags/) fluent/fluent-bit-kubernetes-daemonset:0.11 - -### Steps - -1. Make sure your Elasticsearch backend is running and can be reach through the hostname _elasticsearch-logging_. This value can be changed in the Yaml file - -2. Deploy the daemonset file from this repository: - -```bash -$ kubectl apply -f fluent-bit-daemonset-elasticsearch.yaml -``` - -## Details - -The default configuration of Fluent Bit makes sure of the following: - -- Consume all containers logs from the running Node. -- The [Tail input plugin](http://fluentbit.io/documentation/0.11/input/tail.html) will not append more than __5MB__ into the engine until they are flushed to the Elasticsearch backend. This limit aims to provide a workaround for [backpressure](http://fluentbit.io/documentation/0.11/configuration/backpressure.html) scenarios. -- The Kubernetes filter will enrigh the logs with Kubernetes metadata, specifically _labels_ and _annotations_. The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache. -- The default backend in the configuration is Elasticsearch set by the [Elasticsearch Ouput Plugin](http://fluentbit.io/documentation/0.11/output/elasticsearch.html). It uses the Logstash format to ingest the logs. If you need a different Index and Type, please refer to the plugin option and do your own adjustments. -- There is an option called __Retry_Limit__ set to False, that means if Fluent Bit cannot flush the records to Elasticsearch it will re-try indefinitely until it succeed. - -## Get back to us! - -Your contribution on testing is highly appreciated, we aim to make logging cheaper for everybody so your feedback is fundamental, try to get back to us on: - -- [Mailing List / Google Group](https://groups.google.com/forum/#!forum/fluent-bit) -- [Slack Channel #fluent-bit](http://slack.fluentd.org) diff --git a/docker/fluentbit/build.sh b/docker/fluentbit/build.sh new file mode 100644 index 0000000..92f2c3d --- /dev/null +++ b/docker/fluentbit/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +docker pull fluent/fluent-bit:0.12 +docker tag fluent/fluent-bit:0.12 dockerhub.qingcloud.com/fluent/fluent-bit:0.12 +docker push dockerhub.qingcloud.com/fluent/fluent-bit:0.12 \ No newline at end of file diff --git a/docker/fluentbit/docker-image/Dockerfile b/docker/fluentbit/docker-image/Dockerfile deleted file mode 100644 index f01d7b6..0000000 --- a/docker/fluentbit/docker-image/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM fluent/fluent-bit:0.11.13 -USER root - -COPY fluent-bit.conf /fluent-bit/etc/ -COPY parsers.conf /fluent-bit/etc/ -EXPOSE 24224 - -CMD ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/fluent-bit.conf"] diff --git a/docker/fluentbit/docker-image/fluent-bit.conf b/docker/fluentbit/docker-image/fluent-bit.conf deleted file mode 100644 index 9133a34..0000000 --- a/docker/fluentbit/docker-image/fluent-bit.conf +++ /dev/null @@ -1,33 +0,0 @@ -[SERVICE] - Flush 1 - Daemon Off - Log_Level info - Parsers_File parsers.conf - -[INPUT] - Name tail - Tag kube.* - Path /var/log/containers/*.log - Parser docker - DB /var/log/flb_kube.db - Mem_Buf_Limit 5MB - -[INPUT] - Name forward - Listen 0.0.0.0 - Port 24224 - -[FILTER] - Name kubernetes - Match kube.* - Kube_URL https://kubernetes.default.svc:443 - Merge_JSON_Log On - -[OUTPUT] - Name es - Match * - Host ${FLUENT_ELASTICSEARCH_HOST} - Port ${FLUENT_ELASTICSEARCH_PORT} - Index fluentbit - Type docker - Logstash_Format On diff --git a/docker/fluentbit/docker-image/parsers.conf b/docker/fluentbit/docker-image/parsers.conf deleted file mode 100644 index bffaab7..0000000 --- a/docker/fluentbit/docker-image/parsers.conf +++ /dev/null @@ -1,45 +0,0 @@ -[PARSER] - Name apache - Format regex - Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?