diff --git a/docs/JenkinsUpgradeGuide.md b/docs/JenkinsUpgradeGuide.md index 3527f3d4f..b046c1814 100644 --- a/docs/JenkinsUpgradeGuide.md +++ b/docs/JenkinsUpgradeGuide.md @@ -1,67 +1,96 @@ -# Upgrade Notes for Jenkins +## Upgrade Notes for Jenkins -This document only for who enabled DevOps in Kubesphere. If this is the first time that -you install Jenkins via enable DevOps, then you don't need this document. +This document only for who enabled DevOps in Kubesphere. If this is the first time that you install Jenkins via enable +DevOps, then you don't need this document. -# Background +## Background -Considering Jenkins just has the filesystem as its backend storage. There's not a official -upgrade guide about how to deal with the configuration files from the community. And users -might install other plugins by themselves, it's very hard to provide a automatic way to +Considering Jenkins just has the filesystem as its backend storage. There's not an official +upgrade guide about how to deal with the configuration files from the community. And users might install other plugins +by themselves, it's very hard to provide an automatic way to upgrade [Kubesphere Jenkins](https://github.com/kubesphere/ks-jenkins). -# Upgrade to v3.0.1 +**Each section covers the upgrade from the previous latest release**, the section on v3.0.1 covers the upgrade from +v3.0.0. -> Please notice, v3.0.1 has not been released yet. +## Upgrade to v3.1.1 -## Backup -Firstly, please backup your Jenkins. You can do it by [thin-backup-plugin](https://github.com/jenkinsci/thin-backup-plugin). -Jenkins backup plugins are not quite active, but this one was still maintained this year. +> Please note that v3.1.1 has not been released yet. -Basicly, you can backup everything in the Jenkins home directory. The home directory in the Jenkins pod is `/var/jenkins_home`. +### Migrate `ks-devops-config` configmap -If you've installed other plugins by yourself, or you've upgraded some plugins. You need to export a list of these plugins. +If you customized `ks-devops-config` configmap before, you might need to migrate it into new namespace by executing the +following command: + +```shell +kubectl -n kubesphere-devops-system get configmap ks-devops-agent -oyaml | sed s/namespace:\ kubesphere-devops-system/namespace:\ kubesphere-devops-worker/ | kubectl apply -f - +``` + +## Upgrade to v3.0.1 + +### Backup + +Firstly, please backup your Jenkins. You can do it +by [thin-backup-plugin](https://github.com/jenkinsci/thin-backup-plugin). Jenkins backup plugins are not quite active, +but this one was still maintained this year. + +Basically, you can backup everything in the Jenkins home directory. The home directory in the Jenkins pod +is `/var/jenkins_home`. + +If you've installed other plugins by yourself, or you've upgraded some plugins. You need to export a list of these +plugins. Before you do that, please install and config the [Jenkins CLI](https://github.com/jenkins-zh/jenkins-cli). -You can get the token of Jenkins via: `kubectl get cm kubesphere-config -n kubesphere-system -o jsonpath={.data.kubesphere\\.yaml} | grep devops -A 2 | grep password`. +You can get the token of Jenkins via: -Then export the plugins list via: `jcli plugin formula > jenkins.yaml`. This file is similar to [formula.yaml](https://github.com/kubesphere/ks-jenkins/blob/master/formula.yaml) which comes from [ks-jenkins](https://github.com/kubesphere/ks-jenkins). +```shell +kubectl get cm kubesphere-config -n kubesphere-system -o jsonpath={.data.kubesphere\\.yaml} | grep devops -A 2 | grep password +``` -## Upgrade +Then export the plugins list via the below command. -### Step1: +```shell +jcli plugin formula > jenkins.yaml +``` + +This file is similar to [formula.yaml](https://github.com/kubesphere/ks-jenkins/blob/master/formula.yaml) which comes +from [ks-jenkins](https://github.com/kubesphere/ks-jenkins). + +### Upgrade + +#### Step1: Update the image from deploy to `kubespheredev/ks-jenkins:2.249.1` -``` +```shell kubectl -n kubesphere-devops-system patch deploy ks-jenkins --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "kubespheredev/ks-jenkins:2.249.1"}]' kubectl -n kubesphere-devops-system patch deploy ks-jenkins --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/initContainers/0/image", "value": "kubespheredev/ks-jenkins:2.249.1"}]' ``` -### Step2: +#### Step2: `uc-jenkins-update-center` was removed from `v3.0.1`. So you can just remove it. Do it via the following command: -``` +```shell kubectl -n kubesphere-devops-system delete deploy uc-jenkins-update-center kubectl -n kubesphere-devops-system delete service uc-jenkins-update-center kubectl -n kubesphere-devops-system patch configmap ks-jenkins --type='json' -p='[{"op": "remove", "path": "/data/plugins.txt"}]' -kubectl -n kubesphere-devops-system patch configmap ks-jenkins --type='json' -p='[{"op": "replace", "path": "/data/apply_config.sh", "value":"mkdir -p /usr/share/jenkins/ref/secrets/;\n -echo false > /usr/share/jenkins/ref/secrets/slave-to-master-security-kill-switch;\n -cp --no-clobber /var/jenkins_config/config.xml /var/jenkins_home;\n -cp --no-clobber /var/jenkins_config/jenkins.CLI.xml /var/jenkins_home;\n -cp --no-clobber /var/jenkins_config/jenkins.model.JenkinsLocationConfiguration.xml /var/jenkins_home;\n -mkdir -p /var/jenkins_home/init.groovy.d/;\n +kubectl -n kubesphere-devops-system patch configmap ks-jenkins --type='json' -p='[{"op": "replace", "path": "/data/apply_config.sh", "value":"mkdir -p /usr/share/jenkins/ref/secrets/ +echo false > /usr/share/jenkins/ref/secrets/slave-to-master-security-kill-switch +cp --no-clobber /var/jenkins_config/config.xml /var/jenkins_home +cp --no-clobber /var/jenkins_config/jenkins.CLI.xml /var/jenkins_home +cp --no-clobber /var/jenkins_config/jenkins.model.JenkinsLocationConfiguration.xml /var/jenkins_home +mkdir -p /var/jenkins_home/init.groovy.d/ yes | cp -i /var/jenkins_config/*.groovy /var/jenkins_home/init.groovy.d/"}]' ``` -### Step3: +#### Step3: Restart `ks-installer` -``` +```shell kubectl -n kubesphere-system scale deploy ks-installer --replicas=0 kubectl -n kubesphere-system scale deploy ks-installer --replicas=1 ``` @@ -70,19 +99,22 @@ Then you can check the logs via: `kubectl -n kubesphere-system logs deploy/ks-in It's ready if you can see something like below from the logs output: -``` +```text ##################################################### ### Welcome to KubeSphere! ### ##################################################### ``` -### Step4 (Optional): +#### Step4 (Optional): -Normally, you don't need to this step. But in order to make sure everything is ok. Please check the plugin list after -you upgrade Jenkins. If you found out there're part of them missed, please install these plugins by the following command: +Normally, you don't need to this step. But in order to make sure everything is ok. Please check the plugin list after +you upgrade Jenkins. If you found out there're part of them missed, please install these plugins by the following +command: -`jcli plugin install --formula jenkins.yaml` +```shell +jcli plugin install --formula jenkins.yaml +``` -### Verify +#### Verify Do some tests to make sure everything works well as you expected. diff --git a/roles/common/tasks/init-namespaces.yaml b/roles/common/tasks/init-namespaces.yaml index b2e80368c..2e9a773d6 100644 --- a/roles/common/tasks/init-namespaces.yaml +++ b/roles/common/tasks/init-namespaces.yaml @@ -22,12 +22,15 @@ {%- endif %} {%- if devops.enabled is defined and devops.enabled %} kubesphere-devops-system + kubesphere-devops-worker {%- endif %} {%- if harbor.enabled is defined and harbor.enabled %} kubesphere-devops-system + kubesphere-devops-worker {%- endif %} {%- if gitlab.enabled is defined and gitlab.enabled %} kubesphere-devops-system + kubesphere-devops-worker {%- endif %} {%- if servicemesh.enabled is defined and servicemesh.enabled %} istio-system diff --git a/roles/ks-devops/jenkins/defaults/main.yaml b/roles/ks-devops/jenkins/defaults/main.yaml index 740010e58..a0f39ac71 100644 --- a/roles/ks-devops/jenkins/defaults/main.yaml +++ b/roles/ks-devops/jenkins/defaults/main.yaml @@ -4,7 +4,6 @@ devops: jenkinsMemoryLim: "{{ jenkins_memory_lim |default('8Gi') }}" jenkinsMemoryReq: "{{ jenkins_memory_req |default('4Gi') }}" -Java_Opts: -Xms3g -Xmx6g -XX:MaxRAM=8g -Dhudson.slaves.NodeProvisioner.initialDelay=20 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85 -Dhudson.model.LoadStatistics.clock=5000 -Dhudson.model.LoadStatistics.decay=0.2 -Dhudson.slaves.NodeProvisioner.recurrencePeriod=5000 -verbose:gc -Xloggc:/var/jenkins_home/gc-%t.log -XX:NumberOfGCLogFiles=2 -XX:+UseGCLogFileRotation -XX:GCLogFileSize=100m -XX:+PrintGC -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCCause -XX:+PrintTenuringDistribution -XX:+PrintReferenceGC -XX:+PrintAdaptiveSizePolicy -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:+DisableExplicitGC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions JavaOpts: > -Xms{{ devops.jenkinsJavaOpts_Xms | default('3g') }} -Xmx{{ devops.jenkinsJavaOpts_Xmx | default('6g') }} diff --git a/roles/ks-devops/jenkins/tasks/main.yaml b/roles/ks-devops/jenkins/tasks/main.yaml index 7203a0684..e71fd5ab3 100644 --- a/roles/ks-devops/jenkins/tasks/main.yaml +++ b/roles/ks-devops/jenkins/tasks/main.yaml @@ -30,15 +30,15 @@ - block: - - name: ks-devops | Getting sonarqube host - shell: > - {{ bin_dir }}/kubectl get cm -n kubesphere-system kubesphere-config -o jsonpath='{.data.kubesphere\.yaml}' | grep "sonarQube:" -A 2 | grep "host" | awk '{print $2}' - register: sonarqube_host - - - name: ks-devops | Getting sonarqube token - shell: > - {{ bin_dir }}/kubectl get cm -n kubesphere-system kubesphere-config -o jsonpath='{.data.kubesphere\.yaml}' | grep "sonarQube:" -A 2 | grep "token" | awk '{print $2}' - register: sonarqube_token + - name: ks-devops | Getting sonarqube host + shell: > + {{ bin_dir }}/kubectl get cm -n kubesphere-system kubesphere-config -o jsonpath='{.data.kubesphere\.yaml}' | grep "sonarQube:" -A 2 | grep "host" | awk '{print $2}' + register: sonarqube_host + + - name: ks-devops | Getting sonarqube token + shell: > + {{ bin_dir }}/kubectl get cm -n kubesphere-system kubesphere-config -o jsonpath='{.data.kubesphere\.yaml}' | grep "sonarQube:" -A 2 | grep "token" | awk '{print $2}' + register: sonarqube_token when: - devops.sonarqube is not defined @@ -64,7 +64,8 @@ with_items: - { name: custom-values-jenkins, file: custom-values-jenkins.yaml } - { name: jenkins-casc-config, file: jenkins-casc-config.yml } - - { name: jenkins-agent-config, file: jenkins-agent-config.yaml} + - { name: jenkins-agent-config, file: jenkins-agent-config.yaml } + - { name: worker-resourcequota, file: worker-resourcequota.yaml } - name: ks-devops | Configuring Jenkins host and port @@ -74,8 +75,11 @@ - name: ks-devops | Creating Jenkins Agent Config File shell: > - {{ bin_dir }}/kubectl apply -f {{ kubesphere_dir }}/jenkins/jenkins-agent-config.yaml --namespace kubesphere-devops-system + {{ bin_dir }}/kubectl apply -f {{ kubesphere_dir }}/jenkins/jenkins-agent-config.yaml --namespace kubesphere-devops-worker +- name: ks-devops | Create resource quota for worker namespace + shell: > + {{ bin_dir }}/kubectl apply -f {{ kubesphere_dir }}/jenkins/worker-resourcequota.yaml - name: ks-devops | Deploying Jenkins shell: > diff --git a/roles/ks-devops/jenkins/templates/jenkins-casc-config.yml.j2 b/roles/ks-devops/jenkins/templates/jenkins-casc-config.yml.j2 index c426c21a4..f310bd00f 100644 --- a/roles/ks-devops/jenkins/templates/jenkins-casc-config.yml.j2 +++ b/roles/ks-devops/jenkins/templates/jenkins-casc-config.yml.j2 @@ -14,17 +14,17 @@ data: name: "kubernetes" serverUrl: "https://kubernetes.default" skipTlsVerify: true - namespace: "kubesphere-devops-system" + namespace: "kubesphere-devops-worker" credentialsId: "k8s-service-account" jenkinsUrl: "http://ks-jenkins.kubesphere-devops-system:80" jenkinsTunnel: "ks-jenkins-agent.kubesphere-devops-system:50000" - containerCapStr: "100" + containerCapStr: "2" connectTimeout: "60" readTimeout: "60" maxRequestsPerHostStr: "32" templates: - name: "base" - namespace: "kubesphere-devops-system" + namespace: "kubesphere-devops-worker" label: "base" nodeUsageMode: "NORMAL" idleMinutes: 0 @@ -43,6 +43,7 @@ data: command: "jenkins-slave" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" + resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: @@ -57,7 +58,7 @@ data: mountPath: "/root/.sonar/cache" yaml: "spec:\r\n affinity:\r\n nodeAffinity:\r\n preferredDuringSchedulingIgnoredDuringExecution:\r\n - weight: 1\r\n preference:\r\n matchExpressions:\r\n - key: node-role.kubernetes.io/worker\r\n operator: In\r\n values:\r\n - ci\r\n tolerations:\r\n - key: \"node.kubernetes.io/ci\"\r\n operator: \"Exists\"\r\n effect: \"NoSchedule\"\r\n - key: \"node.kubernetes.io/ci\"\r\n operator: \"Exists\"\r\n effect: \"PreferNoSchedule\"\r\n containers:\r\n - name: \"base\"\r\n resources:\r\n requests:\r\n ephemeral-storage: \"1Gi\"\r\n limits:\r\n ephemeral-storage: \"10Gi\"\r\n securityContext:\r\n fsGroup: 1000\r\n " - name: "nodejs" - namespace: "kubesphere-devops-system" + namespace: "kubesphere-devops-worker" label: "nodejs" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 @@ -76,6 +77,7 @@ data: command: "jenkins-slave" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" + resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: @@ -96,7 +98,7 @@ data: mountPath: "/root/.sonar/cache" yaml: "spec:\r\n affinity:\r\n nodeAffinity:\r\n preferredDuringSchedulingIgnoredDuringExecution:\r\n - weight: 1\r\n preference:\r\n matchExpressions:\r\n - key: node-role.kubernetes.io/worker\r\n operator: In\r\n values:\r\n - ci\r\n tolerations:\r\n - key: \"node.kubernetes.io/ci\"\r\n operator: \"Exists\"\r\n effect: \"NoSchedule\"\r\n - key: \"node.kubernetes.io/ci\"\r\n operator: \"Exists\"\r\n effect: \"PreferNoSchedule\"\r\n containers:\r\n - name: \"nodejs\"\r\n resources:\r\n requests:\r\n ephemeral-storage: \"1Gi\"\r\n limits:\r\n ephemeral-storage: \"10Gi\"\r\n securityContext:\r\n fsGroup: 1000\r\n " - name: "maven" - namespace: "kubesphere-devops-system" + namespace: "kubesphere-devops-worker" label: "maven" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 @@ -115,6 +117,7 @@ data: command: "jenkins-slave" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" + resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: @@ -132,7 +135,7 @@ data: mountPath: "/root/.sonar/cache" yaml: "spec:\r\n affinity:\r\n nodeAffinity:\r\n preferredDuringSchedulingIgnoredDuringExecution:\r\n - weight: 1\r\n preference:\r\n matchExpressions:\r\n - key: node-role.kubernetes.io/worker\r\n operator: In\r\n values:\r\n - ci\r\n tolerations:\r\n - key: \"node.kubernetes.io/ci\"\r\n operator: \"Exists\"\r\n effect: \"NoSchedule\"\r\n - key: \"node.kubernetes.io/ci\"\r\n operator: \"Exists\"\r\n effect: \"PreferNoSchedule\"\r\n containers:\r\n - name: \"maven\"\r\n resources:\r\n requests:\r\n ephemeral-storage: \"1Gi\"\r\n limits:\r\n ephemeral-storage: \"10Gi\"\r\n volumeMounts:\r\n - name: config-volume\r\n mountPath: /opt/apache-maven-3.5.3/conf/settings.xml\r\n subPath: settings.xml\r\n volumes:\r\n - name: config-volume\r\n configMap:\r\n name: ks-devops-agent\r\n items:\r\n - key: MavenSetting\r\n path: settings.xml\r\n securityContext:\r\n fsGroup: 1000\r\n " - name: "go" - namespace: "kubesphere-devops-system" + namespace: "kubesphere-devops-worker" label: "go" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 @@ -151,6 +154,7 @@ data: command: "jenkins-slave" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" + resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: diff --git a/roles/ks-devops/jenkins/templates/worker-resourcequota.yaml.j2 b/roles/ks-devops/jenkins/templates/worker-resourcequota.yaml.j2 new file mode 100644 index 000000000..eb2296fb9 --- /dev/null +++ b/roles/ks-devops/jenkins/templates/worker-resourcequota.yaml.j2 @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: kubesphere-devops-worker + namespace: kubesphere-devops-worker +spec: + hard: + limits.cpu: "9" + limits.memory: 19Gi diff --git a/roles/ks-monitor/files/prometheus/devops/prometheus-jenkinsAlertRules.yaml b/roles/ks-monitor/files/prometheus/devops/prometheus-jenkinsAlertRules.yaml new file mode 100644 index 000000000..70f8ed1d3 --- /dev/null +++ b/roles/ks-monitor/files/prometheus/devops/prometheus-jenkinsAlertRules.yaml @@ -0,0 +1,50 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + custom-alerting-rule-level: cluster + role: thanos-alerting-rules + thanosruler: thanos-ruler + name: prometheus-devops-rules + namespace: kubesphere-monitoring-system +spec: + groups: + - name: devops.jenkins.rules + rules: + - alert: jenkins-cpu-overcommit + annotations: + summary: >- + Jenkins({{ $labels.instance }}) process has overcommitted CPU resource requests. + expr: > + sum(rate(process_cpu_seconds_total{container="ks-jenkins", namespace="kubesphere-devops-system"}[5m])) by (container, instance) + > 0.8 + for: 5m + labels: + alerttype: metric + severity: warning + - alert: jenkins-down + annotations: + message: Jenkins has disppeared from Prometheus target discovery. + expr: absent(default_jenkins_up) + for: 5m + labels: + alerttype: metric + severity: critical + - alert: jenkins-too-many-stuck-jobs + annotations: + summary: Jenkins has too many stuck jobs. + message: Jenkins has more than 10 stuck jobs({{ $value | humanize }}) in the past 5 minutes. Please have an inspection about jenkins. + expr: avg_over_time(jenkins_queue_size_value[5m]) > 10 + for: 5m + labels: + alerttype: metric + severity: warning + - alert: jenkins-too-many-failure-jobs + annotations: + summary: Jenkins has too many failed jobs. + message: Jenkins has more than 5 failed jobs({{ $value | humanize }}) in the past 5 minutes. + expr: increase(jenkins_runs_failure_total{namespace='kubesphere-devops-system', container='ks-jenkins', job='ks-jenkins'}[5m]) > 5 + for: 5m + labels: + alerttype: metric + severity: warning diff --git a/roles/ks-monitor/files/prometheus/devops/prometheus-serviceMonitorJenkins.yaml b/roles/ks-monitor/files/prometheus/devops/prometheus-serviceMonitorJenkins.yaml new file mode 100644 index 000000000..e35a283ef --- /dev/null +++ b/roles/ks-monitor/files/prometheus/devops/prometheus-serviceMonitorJenkins.yaml @@ -0,0 +1,21 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + k8s-app: ks-jenkins + app.kubernetes.io/vendor: kubesphere + name: ks-jenkins + namespace: kubesphere-monitoring-system +spec: + endpoints: + - honorLabels: true + interval: 1m + port: http + path: /prometheus/ + jobLabel: k8s-app + namespaceSelector: + matchNames: + - kubesphere-devops-system + selector: + matchLabels: + app: ks-jenkins diff --git a/scripts/kubesphere-delete.sh b/scripts/kubesphere-delete.sh index dc4a3d31d..ff772b146 100755 --- a/scripts/kubesphere-delete.sh +++ b/scripts/kubesphere-delete.sh @@ -4,7 +4,7 @@ function delete_sure(){ cat << eof $(echo -e "\033[1;36mNote:\033[0m") -Delete the KubeSphere cluster, including the module kubesphere-system kubesphere-devops-system kubesphere-monitoring-system kubesphere-logging-system openpitrix-system. +Delete the KubeSphere cluster, including the module kubesphere-system kubesphere-devops-system kubesphere-devops-worker kubesphere-monitoring-system kubesphere-logging-system openpitrix-system. eof read -p "Please reconfirm that you want to delete the KubeSphere cluster. (yes/no) " ans @@ -178,7 +178,7 @@ do done # delete relevance ns -for ns in kubesphere-alerting-system kubesphere-controls-system kubesphere-devops-system kubesphere-logging-system kubesphere-monitoring-system kubesphere-monitoring-federated openpitrix-system kubesphere-system +for ns in kubesphere-alerting-system kubesphere-controls-system kubesphere-devops-system kubesphere-devops-worker kubesphere-logging-system kubesphere-monitoring-system kubesphere-monitoring-federated openpitrix-system kubesphere-system do kubectl delete ns $ns 2>/dev/null done