update kafka 2.6.1 add ssl

ansible/group_vars/all.yml

@@ -0,0 +1,4 @@
+app_version: 2.1.0
+local_cache_path: "~/.ansible/cache"
+cmak_version: "3.0.0.5"
+

ansible/make.yml

@@ -4,19 +4,20 @@
   strategy: free
   vars:
     target_env: "{{ lookup('env', 'target') }}"
-    local_cache_path: "~/.ansible/cache"
   tasks:
   - include_role:
       name: "{{ service_name }}"
     loop:
     - disable-apt-jobs-1.0.0
     - disable-motd-1.0.0
-    - app-agent-1.0.1
-    - appctl-1.0.9
-    - arping-1.0.0
-    - caddy-1.0.6
+    - app-agent-1.0.6
+    - appctl-1.2.5
+    - arping-1.0.5
+    - caddy-1.1.8
+    - base
     - kafka
     - kafka-manager
+#    - cmak
     - zabbix-agent
     loop_control:
       loop_var: service_name

ansible/requirements.yml

@@ -1,6 +1,6 @@
-- src: https://qingcloudappcenter.github.io/ansible-roles/app-agent-1.0.1.tar.gz
-- src: https://qingcloudappcenter.github.io/ansible-roles/appctl-1.0.9.tar.gz
-- src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.0.tar.gz
-- src: https://qingcloudappcenter.github.io/ansible-roles/caddy-1.0.6.tar.gz
-- src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.0.2.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/app-agent-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/appctl-1.2.5.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.5.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/caddy-1.1.8.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.0.9.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.5.tar.gz

ansible/roles/base/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: install tools
+  apt:
+    update_cache: yes
+    name: ['jq']
+    state: present
+

ansible/roles/cmak/files/etc/confd/conf.d/cmak.sh.toml

@@ -0,0 +1,8 @@
+[template]
+src = "cmak.sh.tmpl"
+dest = "/opt/app/current/bin/tmpl/cmak.sh"
+mode = "0700"
+keys = [
+  "/",
+]
+reload_cmd = "/opt/app/current/bin/tmpl/cmak.sh; appctl reload cmak"

ansible/roles/cmak/files/etc/confd/templates/cmak.sh/21.cmak.conf.tmpl

@@ -0,0 +1,47 @@
+{{- $roleMap := map "kafka" "kafka" "client" "kafka-manager" }}
+{{- $myRole := index $roleMap (getv "/host/role") }}
+{{- $myIp := getv "/host/ip" }}
+{{- $zkPort := getv "/links/zk_service/cluster/endpoints/client/port" "2181" }}
+
+{{- if eq $myRole "kafka-manager" }}
+
+encrypted=$(echo -n {{ getv "/cluster/cluster_id" }}{{ getv "/cluster/user_id" }} | sha256sum | base64)
+ZK_HOSTS="{{ range $i, $p := getvs "/links/zk_service/hosts/*/ip" }}
+  {{- if $i }},{{ end }}
+  {{- . }}:{{ $zkPort }}
+{{- end }}/kafka/{{ getv "/cluster/cluster_id" }}"
+
+flush /opt/app/current/conf/cmak/application.conf << APP_CONF_EOF
+play.http.secret.key="${encrypted:0:65}"
+play.http.secret.key=\${?APPLICATION_SECRET}
+
+# The application languages
+# ~~~~~
+play.i18n.langs=["en"]
+
+play.application.loader=loader.KafkaManagerLoader
+play.http.requestHandler = "play.http.DefaultHttpRequestHandler"
+play.http.context = "/"
+
+# dependency
+kafka-manager.consumer.properties.file=/opt/app/current/conf/cmak/consumer.properties
+kafka-manager.zkhosts="${ZK_HOSTS}"
+cmak.zkhosts="${ZK_HOSTS}"
+kafka-manager.base-zk-path="/kafka-manager"
+pinned-dispatcher.type="PinnedDispatcher"
+pinned-dispatcher.executor="thread-pool-executor"
+application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"]
+
+akka {
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
+  loglevel = "WARNING"
+}
+
+{{- range gets "/env/kafka-manager.basic*" }}
+{{ replace (base .Key) "kafka-manager." "" -1 }}="{{ .Value }}"
+{{- end }}
+basicAuthentication.realm="Kafka-Manager"
+http.port={{ getv "/env/kafka-manager.port" "9000" }}
+APP_CONF_EOF
+
+{{- end }}

ansible/roles/cmak/files/etc/confd/templates/cmak.sh/22.cmak.consumer.properties.tmpl

@@ -0,0 +1,10 @@
+{{- if eq $myRole "kafka-manager" }}
+
+flush /opt/app/current/conf/cmak/consumer.properties << CONSUMER_PROP_EOF
+security.protocol=PLAINTEXT
+key.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
+value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
+group.id={{ getv "/cluster/cluster_id" }}{{ getv "/host/sid" }}
+CONSUMER_PROP_EOF
+
+{{- end }}

ansible/roles/cmak/files/opt/app/current/conf/cmak/.env

@@ -0,0 +1 @@
+KAFKA_MANAGER_OPTS="-Dapplication.home=/data/kafka-manager -java-home /usr/lib/jvm/java-11-openjdk-amd64 -Dconfig.file=/opt/app/current/conf/cmak/application.conf -Dlogger.file=/opt/app/current/conf/cmak/logback.xml -Dpidfile.path=/var/run/cmak/cmak.pid"

ansible/roles/cmak/files/opt/app/current/conf/systemd/cmak.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Yahoo Kafka Manager
+Documentation=http://kafka.apache.org/documentation.html
+Requires=network.target
+After=network.target
+
+[Service]
+RuntimeDirectory=cmak
+PrivateTmp=true
+Type=simple
+User=kafka
+Group=svc
+LimitNOFILE=500000
+WorkingDirectory=/data/kafka-manager
+EnvironmentFile=/opt/app/current/conf/cmak/.env
+ExecStart=/opt/cmak/current/bin/cmak $KAFKA_MANAGER_OPTS
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/cmak/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: install confd files
+  include_role:
+    name: confd-files-1.0.9
+
+- name: copy binaries
+  copy:
+    src: "{{ role_path }}/files/opt/app/"
+    dest: /opt/app/
+    owner: root
+    group: svc
+    mode: preserve
+    directory_mode: u=rwx,g=rx,o=
+
+- name: install manager
+  include_role:
+    name: install-1.0.5
+  vars:
+    opts:
+      pkg_name: "{{ item  }}"
+      pkg_version: "{{ cmak_version }}"
+      pkg_type: zip
+      pkg_url: "https://github.com/yahoo/CMAK/releases/download/{{ cmak_version }}/cmak-{{ cmak_version }}.zip"
+      extracts: yes
+      target_owner: kafka
+      target_group: svc
+      creates: "bin/{{ item }}"
+      bin_path:
+  with_items: 
+    - cmak
+
+- name: copy binaries
+  copy:
+    src: "/opt/cmak/current/cmak-{{ cmak_version }}/"
+    dest: "/opt/cmak/current/"
+    owner: root
+    group: svc
+    mode: preserve
+    directory_mode: u=rwx,g=rx,o=
+    remote_src: yes
+
+- name: Remove old files
+  file: 
+    path: "/opt/cmak/current/cmak-{{ cmak_version }}"
+    state: absent

ansible/roles/kafka-manager/files/etc/confd/conf.d/kafka-manager.sh.toml

@@ -1,8 +1,8 @@
 [template]
 src = "kafka-manager.sh.tmpl"
-dest = "/opt/app/bin/tmpl/kafka-manager.sh"
+dest = "/opt/app/current/bin/tmpl/kafka-manager.sh"
 mode = "0700"
 keys = [
   "/",
 ]
-reload_cmd = "/opt/app/bin/tmpl/kafka-manager.sh; appctl reload kafka-manager"
+reload_cmd = "/opt/app/current/bin/tmpl/kafka-manager.sh; appctl reload kafka-manager"

ansible/roles/kafka-manager/files/etc/confd/templates/kafka-manager.sh/21.kafka-manager.conf.tmpl

@@ -7,7 +7,7 @@
 
 encrypted=$(echo -n {{ getv "/cluster/cluster_id" }}{{ getv "/cluster/user_id" }} | sha256sum | base64)
 
-flush > /opt/app/conf/kafka-manager/application.conf << APP_CONF_EOF
+flush  /opt/app/current/conf/kafka-manager/application.conf << APP_CONF_EOF
 play.http.secret.key="${encrypted:0:65}"
 play.http.secret.key=\${?APPLICATION_SECRET}
 
@@ -20,7 +20,7 @@ play.http.requestHandler = "play.http.DefaultHttpRequestHandler"
 play.http.context = "/"
 
 # dependency
-kafka-manager.consumer.properties.file=/opt/app/conf/kafka-manager/consumer.properties
+kafka-manager.consumer.properties.file=/opt/app/current/conf/kafka-manager/consumer.properties
 kafka-manager.zkhosts="{{ range $i, $p := getvs "/links/zk_service/hosts/*/ip" }}
   {{- if $i }},{{ end }}
   {{- . }}:{{ $zkPort }}

ansible/roles/kafka-manager/files/etc/confd/templates/kafka-manager.sh/22.kafka-manager.consumer.properties.tmpl

@@ -1,6 +1,6 @@
 {{- if eq $myRole "kafka-manager" }}
 
-flush > /opt/app/conf/kafka-manager/consumer.properties << CONSUMER_PROP_EOF
+flush /opt/app/current/conf/kafka-manager/consumer.properties << CONSUMER_PROP_EOF
 security.protocol=PLAINTEXT
 key.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer
 value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer

ansible/roles/kafka-manager/files/opt/app/current/conf/kafka-manager/.env

@@ -0,0 +1 @@
+KAFKA_MANAGER_OPTS="-Dapplication.home=/data/kafka-manager -java-home /usr/lib/jvm/java-11-openjdk-amd64 -Dconfig.file=/opt/app/current/conf/kafka-manager/application.conf -Dlogger.file=/opt/app/current/conf/kafka-manager/logback.xml -Dpidfile.path=/var/run/kafka-manager/kafka-manager.pid"

ansible/roles/kafka-manager/files/opt/app/current/conf/kafka-manager/logback.xml

@@ -0,0 +1,39 @@
+<!--
+  ~ Copyright (C) 2009-2015 Typesafe Inc. <http://www.typesafe.com>
+  -->
+<!-- The default logback configuration that Play uses if no other configuration is provided -->
+<configuration>
+
+    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>/data/kafka-manager/logs/application.log</file>
+        <encoder>
+           <pattern>%date - [%level] - from %logger in %thread %n%message%n%xException%n</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>/data/kafka-manager/logs/application-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <maxFileSize>2MB</maxFileSize>
+            <totalSizeCap>100MB</totalSizeCap>
+            <maxHistory>50</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <appender name="ASYNCFILE" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="FILE" />
+    </appender>
+
+    <logger name="play" level="INFO" />
+    <logger name="application" level="INFO" />
+    <logger name="kafka.manager" level="INFO" />
+
+    <!-- Off these ones as they are annoying, and anyway we manage configuration ourself -->
+    <logger name="com.avaje.ebean.config.PropertyMapLoader" level="OFF" />
+    <logger name="com.avaje.ebeaninternal.server.core.XmlConfigLoader" level="OFF" />
+    <logger name="com.avaje.ebeaninternal.server.lib.BackgroundThread" level="OFF" />
+    <logger name="com.gargoylesoftware.htmlunit.javascript" level="OFF" />
+    <logger name="org.apache.zookeeper" level="INFO"/>
+
+    <root level="INFO">
+        <appender-ref ref="ASYNCFILE" />
+    </root>
+
+</configuration>

ansible/roles/kafka-manager/files/lib/systemd/system/kafka-manager.service → ansible/roles/kafka-manager/files/opt/app/current/conf/systemd/kafka-manager.service

@@ -9,10 +9,10 @@ RuntimeDirectory=kafka-manager
 PrivateTmp=true
 Type=simple
 User=kafka
-Group=kafka
+Group=svc
 LimitNOFILE=500000
 WorkingDirectory=/data/kafka-manager
-EnvironmentFile=/opt/app/conf/kafka-manager/.env
+EnvironmentFile=/opt/app/current/conf/kafka-manager/.env
 ExecStart=/opt/kafka-manager/current/bin/kafka-manager $KAFKA_MANAGER_OPTS
 Restart=on-failure
 

ansible/roles/kafka-manager/tasks/main.yml

@@ -6,54 +6,25 @@
 
 - name: install confd files
   include_role:
-    name: confd-files-1.0.2
-
-- name: copy systemd files
-  copy: #拷贝service文件
-    src: files/{{ item }}
-    dest: /{{ item }}
-    directory_mode: yes
-  with_items:
-    - lib/systemd/system/
-
-- name: disable auto start
-  systemd:
-    name: "{{ svc_name }}"
-    state: stopped
-    masked: yes
-  loop:
-  - kafka-manager
-  loop_control:
-    loop_var: svc_name
+    name: confd-files-1.0.9
 
 - name: copy binaries
-  copy: #拷贝service文件
-    src: files/opt/
-    dest: /opt
+  copy:
+    src: "{{ role_path }}/files/opt/app/"
+    dest: /opt/app/
     owner: root
-    group: kafka
+    group: svc
     mode: preserve
     directory_mode: u=rwx,g=rx,o=
 
 - name: prepare service directories
   file:
     path: /opt/{{ item }}
     owner: kafka
-    group: kafka
+    group: svc
     state: directory
   with_items:
-    - kafka-manager/{{ kafka_manager_version }}
-
-- name: create symbolic links
-  file:
-    src: /opt/{{ item.name }}/{{ item.version }}
-    dest: /opt/{{ item.name }}/current
-    owner: kafka
-    group: kafka
-    state: link
-  with_items:
-    - name: kafka-manager
-      version: "{{ kafka_manager_version }}"
+    - kafka-manager
 
 - name: install manager 
   include_role:
@@ -84,24 +55,37 @@
     src: ~/.ansible/cache/kafka-manager/kafka-manager-{{ kafka_manager_version }}/target/universal/kafka-manager-{{ kafka_manager_version }}.zip
     dest: /tmp/kafka-manager-{{ kafka_manager_version }}.zip
     owner: kafka
-    group: kafka
+    group: svc
     mode: preserve
     directory_mode: u=rwx,g=rx,o=
 
 - name: install kafka manager binaries
   unarchive:
     src: /tmp/kafka-manager-{{ kafka_manager_version }}.zip
-    dest: /opt/kafka-manager/{{ kafka_manager_version }}
+    dest: "/opt/kafka-manager"
     owner: kafka
-    group: kafka
-    creates: /opt/kafka-manager/{{ kafka_manager_version }}/bin/kafka-manager
+    group: svc
     remote_src: yes
 
+- name: prepare service directories
+  shell: mv  /opt/kafka-manager/kafka-manager-{{ kafka_manager_version }} /opt/kafka-manager/{{ kafka_manager_version }}
+
+- name: create symbolic links
+  file:
+    src: /opt/{{ item.name }}/{{ item.version }}
+    dest: /opt/{{ item.name }}/current
+    owner: kafka
+    group: svc
+    state: link
+  with_items:
+    - name: kafka-manager
+      version: "{{ kafka_manager_version }}"
+
 - name: update permissions of service directories
   file:
     path: /opt/{{ item }}
     owner: kafka
-    group: kafka
+    group: svc
     recurse: yes
     state: directory
   with_items: