From e060511a6d16cec5c3f871e611c596d3b270ba95 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Sun, 21 Oct 2018 23:24:05 +0800
Subject: [PATCH 01/10] Add Storage Node Role

---
 1.1.1/app/cluster.json.mustache               | 52 +++++++++--
 1.1.1/app/config.json                         | 90 +++++++++++++++++--
 1.1.1/app/locale/en.json                      |  5 +-
 1.1.1/app/locale/zh-cn.json                   | 10 ++-
 .../confd/templates/registry.config.yml.tmpl  | 13 ++-
 1.1.1/vm-image/opt/harbor/bin/app.sh          | 31 +++++++
 6 files changed, 176 insertions(+), 25 deletions(-)
 create mode 100755 1.1.1/vm-image/opt/harbor/bin/app.sh

diff --git a/1.1.1/app/cluster.json.mustache b/1.1.1/app/cluster.json.mustache
index d77de25..0b147ea 100644
--- a/1.1.1/app/cluster.json.mustache
+++ b/1.1.1/app/cluster.json.mustache
@@ -1,14 +1,15 @@
 {
   "name": {{cluster.name}},
   "description": {{cluster.description}},
-  "upgrade_policy": ["appv-sn0liyi6","appv-9we6rg37","appv-32b0fb93","appv-kilcnvix"],
+  "upgrade_policy": [ "appv-t2kn8lrz", "appv-sn0liyi6", "appv-9we6rg37", "appv-32b0fb93", "appv-kilcnvix" ],
+  "upgrade_roles": { "add": [ "storage_node" ] },
   "vxnet": {{cluster.vxnet}},
   "nodes": [{
     "role": "log_node",
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-hpp0n6xj"
+      "image": "img-mmjbwk2g"
     },
     "instance_class": {{cluster.log_node.instance_class}},
     "count": 1,
@@ -35,13 +36,47 @@
         "cmd": "/opt/harbor/bin/manager.sh restart /opt/harbor/bin/docker-compose.log.yml"
       }
     }
-  },
-  {
+  }, {
+    "role": "storage_node",
+    "container": {
+      "type": "kvm",
+      "zone": "pek3a",
+      "image": "img-mmjbwk2g"
+    },
+    "instance_class": {{cluster.storage_node.instance_class}},
+    "count": {{cluster.storage_node.count}},
+    "cpu": {{cluster.storage_node.cpu}},
+    "memory": {{cluster.storage_node.memory}},
+    "volume": {
+      "size": {{cluster.storage_node.volume_size}},
+      "mount_point": "/data/storage",
+      "mount_options": "defaults,noatime",
+      "filesystem": "ext4",
+      "class": {{cluster.storage_node.volume_class}}
+    },
+    "user_access": false,
+    "services": {
+      "init": {
+        "cmd": "/opt/harbor/bin/app.sh storage init"
+      },
+      "start": {
+        "order": 2,
+        "cmd": "/opt/harbor/bin/app.sh storage start"
+      },
+      "stop": {
+        "order": 3,
+        "cmd": "/opt/harbor/bin/app.sh storage stop"
+      },
+      "restart": {
+        "cmd": "/opt/harbor/bin/app.sh storage restart"
+      }
+    }
+  }, {
     "role": "db_node",
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-hpp0n6xj"
+      "image": "img-mmjbwk2g"
     },
     "instance_class": {{cluster.db_node.instance_class}},
     "count": 1,
@@ -74,7 +109,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-hpp0n6xj"
+      "image": "img-mmjbwk2g"
     },
     "instance_class": {{cluster.cache_node.instance_class}},
     "count": 1,
@@ -99,7 +134,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-hpp0n6xj"
+      "image": "img-mmjbwk2g"
     },
     "instance_class": {{cluster.job_node.instance_class}},
     "count": {{cluster.job_node.count}},
@@ -128,6 +163,7 @@
       "QS_SECRET": {{env.QS_SECRET}},
       "QS_BUCKET": {{env.QS_BUCKET}},
       "QS_ZONE": {{env.QS_ZONE}},
+      "QS_URL": {{env.QS_URL}},
       "QS_ROOT_DIRECTORY": {{env.QS_ROOT_DIRECTORY}}
     }
   },{
@@ -135,7 +171,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-hpp0n6xj"
+      "image": "img-mmjbwk2g"
     },
     "instance_class": {{cluster.web_node.instance_class}},
     "count": {{cluster.web_node.count}},
diff --git a/1.1.1/app/config.json b/1.1.1/app/config.json
index 54a7469..0e4519b 100644
--- a/1.1.1/app/config.json
+++ b/1.1.1/app/config.json
@@ -190,6 +190,80 @@
         ],
         "required": "yes"
       }]
+    }, {
+      "key": "storage_node",
+      "description": "",
+      "label": "storage_node",
+      "type": "array",
+      "properties": [{
+        "key": "cpu",
+        "label": "CPU",
+        "description": "CPUs of each node",
+        "type": "integer",
+        "default": 2,
+        "range": [
+          1,
+          2,
+          4,
+          8
+        ],
+        "required": "yes"
+      }, {
+        "key": "memory",
+        "label": "Memory",
+        "description": "Memory of each node",
+        "type": "integer",
+        "default": 1024,
+        "range": [
+          1024,
+          2048,
+          4096,
+          8192
+        ],
+        "required": "yes"
+      }, {
+        "key": "count",
+        "label": "Count",
+        "description": "If QingStor will be used as Harbor's backend, then DO NOT create this node; otherwise please create one; default to 0 meaning not to create",
+        "type": "integer",
+        "default": 0,
+        "max": 1,
+        "min": 0,
+        "changeable": false,
+        "required": "yes"
+      }, {
+        "key": "instance_class",
+        "label": "Instance Class",
+        "description": "The instance type for the cluster to run, such as high performance, high performance plus",
+        "type": "integer",
+        "default": 0,
+        "range": [
+          0,
+          1
+        ],
+        "required": "yes"
+      }, {
+        "key": "volume_class",
+        "label": "Volume Class",
+        "description": "The volume type for each instance,such as high performance,high performance plus",
+        "type": "integer",
+        "default": 0,
+        "range": [
+          0,
+          3
+        ],
+        "required": "yes"
+      }, {
+        "key": "volume_size",
+        "label": "Volume Size",
+        "description": "The volume size for each instance",
+        "type": "integer",
+        "default": 50,
+        "min": 10,
+        "max": 1000,
+        "step": 10,
+        "required": "yes"
+      }]
     }, {
       "key": "job_node",
       "description": "",
@@ -323,34 +397,34 @@
     }, {
       "key": "QS_ACCESS",
       "label": "QS_ACCESS",
-      "description": "Provide the accesskey to connect QingStor for storing Image data",
+      "description": "Provide the accesskey to connect QingStor for storing Image data; keep untouched if not using QingStor",
       "type": "string",
       "changeable": true,
-      "default": "",
+      "default": "accesskey",
       "required": "yes"
     }, {
       "key": "QS_SECRET",
       "label": "QS_SECRET",
-      "description": "Provide the secretkey to connect QingStor for storing Image data",
+      "description": "Provide the secretkey to connect QingStor for storing Image data; keep untouched if not using QingStor",
       "type": "password",
       "changeable": true,
-      "default": "",
+      "default": "secretkey",
       "required": "yes"
     }, {
       "key": "QS_BUCKET",
       "label": "QS_BUCKET",
-      "description": "Provide the bucket name to use",
+      "description": "Provide the bucket name to use; keep untouched if not using QingStor",
       "type": "string",
       "changeable": false,
-      "default": "",
+      "default": "harbor",
       "required": "yes"
     }, {
       "key": "QS_ROOT_DIRECTORY",
       "label": "QS_ROOT_DIRECTORY",
-      "description": "The root direcotry or prefix for Harbor image storage",
+      "description": "The root direcotry or prefix for Harbor image storage; keep untouched if not using QingStor",
       "type": "string",
       "changeable": false,
-      "default": "",
+      "default": "registry-root",
       "required": "yes"
     }, {
       "key": "QS_ZONE",
diff --git a/1.1.1/app/locale/en.json b/1.1.1/app/locale/en.json
index 1cc9ee0..107b4f4 100644
--- a/1.1.1/app/locale/en.json
+++ b/1.1.1/app/locale/en.json
@@ -28,14 +28,10 @@
   "The instance type for the cluster to run, such as high performance, high performance plus": "The instance type for the cluster to run, such as high performance, high performance plus",
   "The volume type for each instance,such as high performance,high performance plus":"The volume type for each instance,such as high performance,high performance plus",
   "access_key_id": "access_key_id",
-  "Provide the secretkey to connect QingStor for storing Image data": "Provide the Secret_Access_Key to connect QingStor for storing Image data",
   "secret_access_key": "secret_access_key",
-  "Provide the accesskey to connect QingStor for storing Image data": "Provide the Access_Key_ID to connect QingStor for storing Image data",
   "zone": "zone",
   "Provide the zone of bucket to use": "Provide the zone of bucket to use, see detail from User-Guide",
   "bucket": "bucket",
-  "Provide the bucket name to use": "Provide the bucket name to use",
-  "The root direcotry or prefix for Harbor image storage":"The root direcotry or prefix for Harbor image storage",
   "access_address": "access_address",
   "volume size": "volume size",
   "Volume Size":"Volume Size",
@@ -47,6 +43,7 @@
   "Number of nodes for replica cluster to create":"Number of nodes for replica cluster to create, unworkable when set 0",
   "web_node":"Main Service Node",
   "log_node":"Log Node",
+  "storage_node":"Storage Node",
   "Configuration properties":"Configuration properties",
   "HARBOR_HOST":"Harbor Host",
   "HARBOR_MYSQL_PW":"MySQL Password",
diff --git a/1.1.1/app/locale/zh-cn.json b/1.1.1/app/locale/zh-cn.json
index f0a6b32..2e3d9ae 100644
--- a/1.1.1/app/locale/zh-cn.json
+++ b/1.1.1/app/locale/zh-cn.json
@@ -26,16 +26,16 @@
   "The instance type for the cluster to run, such as high performance, high performance plus": "节点实例类型，比如性能型与超高性能型",
   "The volume type for each instance,such as high performance,high performance plus":"磁盘类型，比如性能型与超高性能型",
   "access_key_id": "access_key_id",
-  "Provide the secretkey to connect QingStor for storing Image data": "提供可用于访问 QingStor 的 Secret_Access_Key，用于 Harbor 的镜像存储",
+  "Provide the secretkey to connect QingStor for storing Image data; keep untouched if not using QingStor": "提供可用于访问 QingStor 的 Secret Key，用于 Harbor 的镜像存储；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
   "Provide an access_key_id to use": "提供要使用的 access_key_id",
   "secret_access_key": "secret_access_key",
-  "Provide the accesskey to connect QingStor for storing Image data": "提供可用于访问 QingStor 的 Access_Key_ID，用于 Harbor 的镜像存储",
+  "Provide the accesskey to connect QingStor for storing Image data; keep untouched if not using QingStor": "提供可用于访问 QingStor 的 Access Key，用于 Harbor 的镜像存储；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
   "Provide the Address of QingStor": "提供QingStor的地址",
   "zone": "区域",
   "Provide the zone of bucket to use": "提供要使用的存储空间区域",
-  "The root direcotry or prefix for Harbor image storage":"提供用于存储镜像的根目录名称",
+  "The root direcotry or prefix for Harbor image storage; keep untouched if not using QingStor":"提供用于存储镜像的根目录名称；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
   "bucket": "存储区域",
-  "Provide the bucket name to use": "提供要使用的存储 Bucket 名称，需要提前创建，详情见《Harbor on QingCloud AppCenter 用户手册》",
+  "Provide the bucket name to use; keep untouched if not using QingStor": "提供要使用的存储 Bucket 名称，需要提前创建，详情见《Harbor on QingCloud AppCenter 用户手册》；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
   "volume size": "磁盘大小",
   "Volume Size": "磁盘大小",
   "Volume Class":"磁盘类型",
@@ -47,6 +47,8 @@
   "Number of nodes for replica cluster to create":"镜像复制服务节点数，如果为0，则“复制”功能会失效，可通过增加节点生效服务",
   "web_node":"主服务节点",
   "log_node":"日志节点",
+  "storage_node":"存储节点",
+  "If QingStor will be used as Harbor's backend, then DO NOT create this node; otherwise please create one; default to 0 meaning not to create": "如果使用 QingStor 做 Harbor 的后端存储则不要创建此节点，否则需要创建；默认为 0 表示不创建",
   "Configuration properties":"服务配置参数",
   "HARBOR_HOST":"Harbor 地址",
   "HARBOR_MYSQL_PW":"MySQL 密码",
diff --git a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl b/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
index 8b0ffaf..501980b 100644
--- a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
+++ b/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
@@ -5,12 +5,22 @@ log:
     service: registry
 
 storage:
+{{- if gt (len (ls "/hosts/storage_node")) 0 }}
+    s3:
+        accesskey: admin
+        secretkey: password
+        bucket: harbor
+        region: default
+        regionendpoint: http://{{ range getvs "/hosts/storage_node/*/ip" }}{{ . }}{{ end }}
+        rootdirectory: registry-root
+{{- else }}
     qs:
         accesskey: {{getv "/env/QS_ACCESS"}}
         secretkey: {{getv "/env/QS_SECRET"}}
         bucket: {{getv "/env/QS_BUCKET"}}
         zone: {{getv "/env/QS_ZONE"}}
         rootdirectory: {{getv "/env/QS_ROOT_DIRECTORY" "registry"}}
+{{- end }}
     cache:
         layerinfo: redis
     maintenance:
@@ -48,4 +58,5 @@ redis:
       idletimeout: 300s
     dialtimeout: 10ms
     readtimeout: 10ms
-    writetimeout: 10ms
\ No newline at end of file
+    writetimeout: 10ms
+
diff --git a/1.1.1/vm-image/opt/harbor/bin/app.sh b/1.1.1/vm-image/opt/harbor/bin/app.sh
new file mode 100755
index 0000000..778a7a5
--- /dev/null
+++ b/1.1.1/vm-image/opt/harbor/bin/app.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+minioVersion="RELEASE.2018-10-06T00-15-16Z"
+
+storageInit() {
+  mkdir -p /data/storage/harbor
+}
+
+storageStart() {
+  docker run -d \
+    --name minio \
+    -e "MINIO_ACCESS_KEY=admin" \
+    -e "MINIO_SECRET_KEY=password" \
+    -v /data/storage:/data \
+    -p 80:9000 \
+    --restart always \
+    minio/minio:${minioVersion} server /data
+}
+
+storageStop() {
+  if [ "$(docker ps -q -a -f name=minio)" ]; then
+    docker rm -f minio
+  fi
+}
+
+storageRestart() {
+  storageStop
+  storageStart
+}
+
+"$1${2^}"

From bc8f9897ab6592e8a7cd275b9f27f37f82c90e9f Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Mon, 22 Oct 2018 12:33:40 +0800
Subject: [PATCH 02/10] Use Generated Random Token for Minio

---
 1.1.1/app/cluster.json.mustache               | 18 +++++++--------
 .../vm-image/etc/confd/templates/env.sh.tmpl  | 14 ++++++++++--
 .../confd/templates/registry.config.yml.tmpl  |  7 ++++--
 1.1.1/vm-image/opt/harbor/bin/app.sh          | 22 +++++++++++++++----
 4 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/1.1.1/app/cluster.json.mustache b/1.1.1/app/cluster.json.mustache
index 0b147ea..f461f67 100644
--- a/1.1.1/app/cluster.json.mustache
+++ b/1.1.1/app/cluster.json.mustache
@@ -9,7 +9,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-mmjbwk2g"
+      "image": "img-aqakypwt"
     },
     "instance_class": {{cluster.log_node.instance_class}},
     "count": 1,
@@ -41,7 +41,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-mmjbwk2g"
+      "image": "img-aqakypwt"
     },
     "instance_class": {{cluster.storage_node.instance_class}},
     "count": {{cluster.storage_node.count}},
@@ -55,10 +55,10 @@
       "class": {{cluster.storage_node.volume_class}}
     },
     "user_access": false,
+    "custom_metadata": {
+      "cmd": "/opt/harbor/bin/app.sh storage generateToken"
+    },
     "services": {
-      "init": {
-        "cmd": "/opt/harbor/bin/app.sh storage init"
-      },
       "start": {
         "order": 2,
         "cmd": "/opt/harbor/bin/app.sh storage start"
@@ -76,7 +76,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-mmjbwk2g"
+      "image": "img-aqakypwt"
     },
     "instance_class": {{cluster.db_node.instance_class}},
     "count": 1,
@@ -109,7 +109,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-mmjbwk2g"
+      "image": "img-aqakypwt"
     },
     "instance_class": {{cluster.cache_node.instance_class}},
     "count": 1,
@@ -134,7 +134,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-mmjbwk2g"
+      "image": "img-aqakypwt"
     },
     "instance_class": {{cluster.job_node.instance_class}},
     "count": {{cluster.job_node.count}},
@@ -171,7 +171,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-mmjbwk2g"
+      "image": "img-aqakypwt"
     },
     "instance_class": {{cluster.web_node.instance_class}},
     "count": {{cluster.web_node.count}},
diff --git a/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl b/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
index f9d33af..c9d6a1e 100644
--- a/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
+++ b/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
@@ -1,3 +1,13 @@
 #!/bin/bash
-EXT_ENDPOINT={{getv "/env/HARBOR_HOST"}}
-QS_URL={{getv "/env/QS_URL"}}
+
+EXT_ENDPOINT={{ range getvs "/env/HARBOR_HOST" }}{{ . }}{{ end }}
+QS_URL={{ range getvs "/env/QS_URL" }}{{ . }}{{ end }}
+
+{{- range getvs "/host/role" | filter "storage_node" }}
+NODE_ROLE={{ . }}
+{{- range getvs "/host/token" }}
+{{- $token := json . }}
+MINIO_ACCESS_KEY={{ $token.accessKey }}
+MINIO_SECRET_KEY={{ $token.secretKey }}
+{{- end }}
+{{- end }}
diff --git a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl b/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
index 501980b..1d6da74 100644
--- a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
+++ b/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
@@ -6,13 +6,16 @@ log:
 
 storage:
 {{- if gt (len (ls "/hosts/storage_node")) 0 }}
+{{- range getvs "/hosts/storage_node/*/token" }}
+{{- $token := json . }}
     s3:
-        accesskey: admin
-        secretkey: password
+        accesskey: {{ $token.accessKey }}
+        secretkey: {{ $token.secretKey }}
         bucket: harbor
         region: default
         regionendpoint: http://{{ range getvs "/hosts/storage_node/*/ip" }}{{ . }}{{ end }}
         rootdirectory: registry-root
+{{- end }}
 {{- else }}
     qs:
         accesskey: {{getv "/env/QS_ACCESS"}}
diff --git a/1.1.1/vm-image/opt/harbor/bin/app.sh b/1.1.1/vm-image/opt/harbor/bin/app.sh
index 778a7a5..9db4074 100755
--- a/1.1.1/vm-image/opt/harbor/bin/app.sh
+++ b/1.1.1/vm-image/opt/harbor/bin/app.sh
@@ -1,16 +1,30 @@
 #!/usr/bin/env bash
 
+source /opt/harbor/bin/env.sh
 minioVersion="RELEASE.2018-10-06T00-15-16Z"
 
-storageInit() {
-  mkdir -p /data/storage/harbor
+[[ "$NODE_ROLE" == "$1_node" ]] || exit 0
+
+genTokenWithLength() {
+  < /dev/urandom tr -dc A-Za-z0-9 | head -c${1:-$1}
+}
+
+storageGenerateToken() {
+  accessKey=$(genTokenWithLength 21)
+  secretKey=$(genTokenWithLength 41)
+
+  cat << EOF
+{"accessKey":"$accessKey","secretKey":"$secretKey"}
+EOF
 }
 
 storageStart() {
+  mkdir -p /data/storage/harbor
+
   docker run -d \
     --name minio \
-    -e "MINIO_ACCESS_KEY=admin" \
-    -e "MINIO_SECRET_KEY=password" \
+    -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
+    -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
     -v /data/storage:/data \
     -p 80:9000 \
     --restart always \

From 29d73281037ee2a40becb4ca5e26b11152cb2a13 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Mon, 22 Oct 2018 19:56:22 +0800
Subject: [PATCH 03/10] Fix Script Bug

---
 1.1.1/app/cluster.json.mustache          | 12 +++---
 1.1.1/vm-image/opt/harbor/bin/manager.sh | 48 +++++++++++++-----------
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/1.1.1/app/cluster.json.mustache b/1.1.1/app/cluster.json.mustache
index f461f67..f5120c9 100644
--- a/1.1.1/app/cluster.json.mustache
+++ b/1.1.1/app/cluster.json.mustache
@@ -9,7 +9,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-aqakypwt"
+      "image": "img-p3ekx81u"
     },
     "instance_class": {{cluster.log_node.instance_class}},
     "count": 1,
@@ -41,7 +41,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-aqakypwt"
+      "image": "img-p3ekx81u"
     },
     "instance_class": {{cluster.storage_node.instance_class}},
     "count": {{cluster.storage_node.count}},
@@ -76,7 +76,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-aqakypwt"
+      "image": "img-p3ekx81u"
     },
     "instance_class": {{cluster.db_node.instance_class}},
     "count": 1,
@@ -109,7 +109,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-aqakypwt"
+      "image": "img-p3ekx81u"
     },
     "instance_class": {{cluster.cache_node.instance_class}},
     "count": 1,
@@ -134,7 +134,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-aqakypwt"
+      "image": "img-p3ekx81u"
     },
     "instance_class": {{cluster.job_node.instance_class}},
     "count": {{cluster.job_node.count}},
@@ -171,7 +171,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-aqakypwt"
+      "image": "img-p3ekx81u"
     },
     "instance_class": {{cluster.web_node.instance_class}},
     "count": {{cluster.web_node.count}},
diff --git a/1.1.1/vm-image/opt/harbor/bin/manager.sh b/1.1.1/vm-image/opt/harbor/bin/manager.sh
index 2a70a5f..9296e96 100644
--- a/1.1.1/vm-image/opt/harbor/bin/manager.sh
+++ b/1.1.1/vm-image/opt/harbor/bin/manager.sh
@@ -21,6 +21,28 @@ then
   exit 1
 fi
 
+function fail {
+  echo $1 >&2
+  exit 1
+}
+
+function retry {
+  local n=0
+  local max=10
+  local delay=1
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempted $n/$max:"
+        sleep $delay
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
 setEndPoint() {
   echo "Set endpoint ${EXT_ENDPOINT}"
   sed "s#EXT_ENDPOINT=.*#EXT_ENDPOINT=${EXT_ENDPOINT}#" -i /opt/harbor/bin/harbor/common/config/adminserver/env
@@ -39,6 +61,7 @@ start() {
   WAIT_MAX=10
   WAIT_TIMES=0
   READY=0
+  retry [ -n "$(docker ps -qa)" ]
   CONTAINERS=`docker ps -qa`
   until [ $WAIT_TIMES -ge $WAIT_MAX ] || [ $READY -eq 1 ]
   do
@@ -63,29 +86,10 @@ start() {
     exit 1
   fi
 
-  WAIT_TIMES=0
-  for C in "${CONTAINERS[@]}"
-  do
-    PORTS=`docker inspect -f '{{range $k,$v:=.HostConfig.PortBindings}}{{$k}}{{end}}' $C
-`
-    for P in "${PORTS[@]}"
-    do
-      RET_CODE=`nc -zv -w5 0.0.0.0 ${P%/*} | echo $?`
-      until [ $RET_CODE -eq 0 ] || [ $WAIT_TIMES -ge $WAIT_MAX ]
-      do
-        echo "check ${P}"
-        sleep 1
-        let WAIT_TIMES+=1
-        RET_CODE=`nc -zv -w5 0.0.0.0 ${P%/*} | echo $?`
-      done
-    done
+  PORTS=`docker inspect -f '{{range $k,$v:=.HostConfig.PortBindings}}{{$k}}{{end}}' $(docker ps -q)`
+  for P in ${PORTS[@]}; do
+    retry nc -zv -w5 0.0.0.0 ${P%/*}
   done
-
-  # exit 1 when start timeout
-  if [ $WAIT_TIMES -ge $WAIT_MAX ]
-  then
-    exit 1
-  fi
 }
 
 restart() {

From 5e666cf02a03678758b864dae3f9af048e4b4b52 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Mon, 29 Oct 2018 10:57:33 +0800
Subject: [PATCH 04/10] Replace Minio with NFS for Local Storage

---
 1.1.1/app/cluster.json.mustache               | 56 ++++++++-----
 1.1.1/app/config.json                         |  3 +-
 1.1.1/app/locale/en.json                      |  1 -
 1.1.1/app/locale/zh-cn.json                   |  2 +-
 1.1.1/vm-image/etc/confd/conf.d/env.sh.toml   |  2 +-
 1.1.1/vm-image/etc/confd/conf.d/exports.toml  |  8 ++
 1.1.1/vm-image/etc/confd/conf.d/hosts.toml    |  2 +-
 .../etc/confd/conf.d/registry.config.toml     |  2 +-
 .../vm-image/etc/confd/templates/env.sh.tmpl  | 11 ++-
 .../vm-image/etc/confd/templates/exports.tmpl |  1 +
 .../confd/templates/registry.config.yml.tmpl  | 13 +--
 1.1.1/vm-image/opt/harbor/bin/manager.sh      |  4 +-
 1.1.1/vm-image/opt/harborapp/bin/app.sh       | 79 +++++++++++++++++++
 13 files changed, 140 insertions(+), 44 deletions(-)
 create mode 100644 1.1.1/vm-image/etc/confd/conf.d/exports.toml
 create mode 100644 1.1.1/vm-image/etc/confd/templates/exports.tmpl
 create mode 100644 1.1.1/vm-image/opt/harborapp/bin/app.sh

diff --git a/1.1.1/app/cluster.json.mustache b/1.1.1/app/cluster.json.mustache
index f5120c9..3168ace 100644
--- a/1.1.1/app/cluster.json.mustache
+++ b/1.1.1/app/cluster.json.mustache
@@ -9,7 +9,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-p3ekx81u"
+      "image": "img-57d2wld3"
     },
     "instance_class": {{cluster.log_node.instance_class}},
     "count": 1,
@@ -41,7 +41,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-p3ekx81u"
+      "image": "img-57d2wld3"
     },
     "instance_class": {{cluster.storage_node.instance_class}},
     "count": {{cluster.storage_node.count}},
@@ -49,34 +49,44 @@
     "memory": {{cluster.storage_node.memory}},
     "volume": {
       "size": {{cluster.storage_node.volume_size}},
-      "mount_point": "/data/storage",
+      "mount_point": "/data/registry",
       "mount_options": "defaults,noatime",
       "filesystem": "ext4",
       "class": {{cluster.storage_node.volume_class}}
     },
     "user_access": false,
-    "custom_metadata": {
-      "cmd": "/opt/harbor/bin/app.sh storage generateToken"
-    },
     "services": {
+      "init": {
+        "order": 1,
+        "cmd": "/opt/harborapp/bin/app.sh init storage"
+      },
       "start": {
-        "order": 2,
-        "cmd": "/opt/harbor/bin/app.sh storage start"
+        "order": 1,
+        "cmd": "/opt/harborapp/bin/app.sh reset storage"
       },
       "stop": {
-        "order": 3,
-        "cmd": "/opt/harbor/bin/app.sh storage stop"
+        "order": 4,
+        "cmd": "/opt/harborapp/bin/app.sh stop storage"
       },
       "restart": {
-        "cmd": "/opt/harbor/bin/app.sh storage restart"
+        "cmd": "/opt/harborapp/bin/app.sh reset storage"
       }
+    },
+    "health_check": {
+      "enable": true,
+      "interval_sec": 60,
+      "timeout_sec": 10,
+      "action_timeout_sec": 30,
+      "healthy_threshold": 2,
+      "unhealthy_threshold": 2,
+      "check_cmd": "/opt/harborapp/bin/app.sh check storage"
     }
   }, {
     "role": "db_node",
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-p3ekx81u"
+      "image": "img-57d2wld3"
     },
     "instance_class": {{cluster.db_node.instance_class}},
     "count": 1,
@@ -109,7 +119,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-p3ekx81u"
+      "image": "img-57d2wld3"
     },
     "instance_class": {{cluster.cache_node.instance_class}},
     "count": 1,
@@ -134,7 +144,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-p3ekx81u"
+      "image": "img-57d2wld3"
     },
     "instance_class": {{cluster.job_node.instance_class}},
     "count": {{cluster.job_node.count}},
@@ -143,9 +153,13 @@
     "advanced_actions": ["scale_horizontal"],
     "user_access": false,
     "services": {
+      "init": {
+        "order": 4,
+        "cmd": "/opt/harborapp/bin/app.sh init job"
+      },
       "start": {
         "order": 4,
-        "cmd": "/opt/harbor/bin/manager.sh start /opt/harbor/bin/docker-compose.job.yml"
+        "cmd": "/opt/harborapp/bin/app.sh start job"
       },
       "stop": {
         "order": 2,
@@ -153,7 +167,7 @@
       },
       "restart": {
         "order": 2,
-        "cmd": "/opt/harbor/bin/manager.sh restart /opt/harbor/bin/docker-compose.job.yml"
+        "cmd": "/opt/harborapp/bin/app.sh restart job"
       }
     },
     "env": {
@@ -171,7 +185,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-p3ekx81u"
+      "image": "img-57d2wld3"
     },
     "instance_class": {{cluster.web_node.instance_class}},
     "count": {{cluster.web_node.count}},
@@ -181,16 +195,20 @@
     "advanced_actions": ["scale_horizontal"],
     "user_access": false,
     "services": {
+      "init": {
+        "order": 3,
+        "cmd": "/opt/harborapp/bin/app.sh init web"
+      },
       "start": {
         "order": 3,
-        "cmd": "/opt/harbor/bin/manager.sh start /opt/harbor/bin/docker-compose.web.yml"
+        "cmd": "/opt/harborapp/bin/app.sh start web"
       },
       "stop": {
         "order": 1,
         "cmd": "/opt/harbor/bin/manager.sh stop /opt/harbor/bin/docker-compose.web.yml"
       },
       "restart": {
-        "cmd": "/opt/harbor/bin/manager.sh restart /opt/harbor/bin/docker-compose.web.yml"
+        "cmd": "/opt/harborapp/bin/app.sh restart web"
       },
       "cleanImage": {
         "type": "custom",
diff --git a/1.1.1/app/config.json b/1.1.1/app/config.json
index 0e4519b..9318284 100644
--- a/1.1.1/app/config.json
+++ b/1.1.1/app/config.json
@@ -389,10 +389,11 @@
     "properties": [{
       "key": "HARBOR_HOST",
       "label": "HARBOR_HOST",
-      "description": "The address of harbor service",
+      "description": "The address of harbor service, e.g. https://harbor.example.com (notice not to end with slash '/')",
       "type": "string",
       "changeable": true,
       "default": "http://127.0.0.1",
+      "pattern": "^https?://[^:/]+(:[0-9]+)?$",
       "required": "yes"
     }, {
       "key": "QS_ACCESS",
diff --git a/1.1.1/app/locale/en.json b/1.1.1/app/locale/en.json
index 107b4f4..f9a899e 100644
--- a/1.1.1/app/locale/en.json
+++ b/1.1.1/app/locale/en.json
@@ -50,7 +50,6 @@
   "The password of Database MySQL":"Password of Harbor Database MySQL",
   "REGISTRY_READONLY":"Readonly",
   "Turn on readonly mode,only allow pull action":"Turn on readonly mode,only allow pull action,use it when GC",
-  "The address of harbor service":"The address of harbor service,",
   "QS_ACCESS":"Access_Key_ID",
   "QS_SECRET":"Secret_Access_Key",
   "QS_BUCKET":"QingStor Bucket",
diff --git a/1.1.1/app/locale/zh-cn.json b/1.1.1/app/locale/zh-cn.json
index 2e3d9ae..08a1ecd 100644
--- a/1.1.1/app/locale/zh-cn.json
+++ b/1.1.1/app/locale/zh-cn.json
@@ -55,7 +55,7 @@
   "The password of Database MySQL":"Harbor 数据库 MySQL 的密码",
   "REGISTRY_READONLY":"只读",
   "Turn on readonly mode,only allow pull action":"只允许 pull 操作，请在进行仓库清理 GC 时使用",
-  "The address of harbor service":"访问 Harbor 服务的地址，与负载均衡器前端保持一致",
+  "The address of harbor service, e.g. https://harbor.example.com (notice not to end with slash '/')":"访问 Harbor 服务的地址，与负载均衡器前端保持一致，示例：https://harbor.example.com （注意不要以斜线 '/' 结尾）",
   "QS_ACCESS":"Access_Key_ID",
   "QS_SECRET":"Secret_Access_Key",
   "QS_BUCKET":"QingStor Bucket 桶",
diff --git a/1.1.1/vm-image/etc/confd/conf.d/env.sh.toml b/1.1.1/vm-image/etc/confd/conf.d/env.sh.toml
index d9dc032..cd761ab 100644
--- a/1.1.1/vm-image/etc/confd/conf.d/env.sh.toml
+++ b/1.1.1/vm-image/etc/confd/conf.d/env.sh.toml
@@ -5,4 +5,4 @@ keys = [
   "/",
 ]
 
-reload_cmd = "/opt/harbor/bin/manager.sh restart docker-compose.web.yml"
\ No newline at end of file
+reload_cmd = "/opt/harborapp/bin/app.sh restart web"
diff --git a/1.1.1/vm-image/etc/confd/conf.d/exports.toml b/1.1.1/vm-image/etc/confd/conf.d/exports.toml
new file mode 100644
index 0000000..6cc3572
--- /dev/null
+++ b/1.1.1/vm-image/etc/confd/conf.d/exports.toml
@@ -0,0 +1,8 @@
+[template]
+src = "exports.tmpl"
+dest = "/opt/harborapp/conf/storage/nfs/exports"
+keys = [
+  "/",
+]
+
+reload_cmd = "/opt/harborapp/bin/app.sh reset storage"
diff --git a/1.1.1/vm-image/etc/confd/conf.d/hosts.toml b/1.1.1/vm-image/etc/confd/conf.d/hosts.toml
index 034c00d..f426ec1 100644
--- a/1.1.1/vm-image/etc/confd/conf.d/hosts.toml
+++ b/1.1.1/vm-image/etc/confd/conf.d/hosts.toml
@@ -4,4 +4,4 @@ dest = "/etc/hosts"
 keys = [
   "/",
 ]
-reload_cmd = "/opt/harbor/bin/manager.sh restart docker-compose.web.yml"
\ No newline at end of file
+reload_cmd = "/opt/harborapp/bin/app.sh restart web"
diff --git a/1.1.1/vm-image/etc/confd/conf.d/registry.config.toml b/1.1.1/vm-image/etc/confd/conf.d/registry.config.toml
index 377f5e0..4905e38 100644
--- a/1.1.1/vm-image/etc/confd/conf.d/registry.config.toml
+++ b/1.1.1/vm-image/etc/confd/conf.d/registry.config.toml
@@ -4,4 +4,4 @@ dest = "/opt/harbor/bin/harbor/common/config/registry/config.yml"
 keys = [
   "/",
 ]
-reload_cmd = "/opt/harbor/bin/manager.sh restart docker-compose.web.yml"
+reload_cmd = "/opt/harborapp/bin/app.sh restart web"
diff --git a/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl b/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
index c9d6a1e..b22a0b7 100644
--- a/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
+++ b/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
@@ -3,11 +3,10 @@
 EXT_ENDPOINT={{ range getvs "/env/HARBOR_HOST" }}{{ . }}{{ end }}
 QS_URL={{ range getvs "/env/QS_URL" }}{{ . }}{{ end }}
 
-{{- range getvs "/host/role" | filter "storage_node" }}
-NODE_ROLE={{ . }}
-{{- range getvs "/host/token" }}
-{{- $token := json . }}
-MINIO_ACCESS_KEY={{ $token.accessKey }}
-MINIO_SECRET_KEY={{ $token.secretKey }}
+NODE_ROLE={{ getv "/host/role" }}
+
+{{- range getvs "/host/role" | filter "(web|job)_node" }}
+{{- range getvs "/hosts/storage_node/*/ip" }}
+STORAGE_NODE_IP={{ . }}
 {{- end }}
 {{- end }}
diff --git a/1.1.1/vm-image/etc/confd/templates/exports.tmpl b/1.1.1/vm-image/etc/confd/templates/exports.tmpl
new file mode 100644
index 0000000..9085b3f
--- /dev/null
+++ b/1.1.1/vm-image/etc/confd/templates/exports.tmpl
@@ -0,0 +1 @@
+/registry	{{ range ls "/hosts" | filter "(web|job)_node" }}{{ range getvs (printf "/hosts/%s/*/ip" .) }}{{ . }}(rw,sync,no_subtree_check) {{ end }}{{ end }}
diff --git a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl b/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
index 1d6da74..e506945 100644
--- a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
+++ b/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
@@ -6,16 +6,8 @@ log:
 
 storage:
 {{- if gt (len (ls "/hosts/storage_node")) 0 }}
-{{- range getvs "/hosts/storage_node/*/token" }}
-{{- $token := json . }}
-    s3:
-        accesskey: {{ $token.accessKey }}
-        secretkey: {{ $token.secretKey }}
-        bucket: harbor
-        region: default
-        regionendpoint: http://{{ range getvs "/hosts/storage_node/*/ip" }}{{ . }}{{ end }}
-        rootdirectory: registry-root
-{{- end }}
+    filesystem:
+        rootdirectory: /storage
 {{- else }}
     qs:
         accesskey: {{getv "/env/QS_ACCESS"}}
@@ -62,4 +54,3 @@ redis:
     dialtimeout: 10ms
     readtimeout: 10ms
     writetimeout: 10ms
-
diff --git a/1.1.1/vm-image/opt/harbor/bin/manager.sh b/1.1.1/vm-image/opt/harbor/bin/manager.sh
index 9296e96..be47c0a 100644
--- a/1.1.1/vm-image/opt/harbor/bin/manager.sh
+++ b/1.1.1/vm-image/opt/harbor/bin/manager.sh
@@ -21,12 +21,12 @@ then
   exit 1
 fi
 
-function fail {
+fail() {
   echo $1 >&2
   exit 1
 }
 
-function retry {
+retry() {
   local n=0
   local max=10
   local delay=1
diff --git a/1.1.1/vm-image/opt/harborapp/bin/app.sh b/1.1.1/vm-image/opt/harborapp/bin/app.sh
new file mode 100644
index 0000000..53f82ce
--- /dev/null
+++ b/1.1.1/vm-image/opt/harborapp/bin/app.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+source /opt/harbor/bin/env.sh
+nodeRole=${NODE_ROLE%_node}
+[[ ",$2," =~ ",$nodeRole," ]] || exit 0
+
+# ======== Common ======== #
+clientMountPath=/data/registry
+
+ensureRegistryMounted() {
+  [[ -z "$STORAGE_NODE_IP" ]] && return
+
+  if ! grep -qs "$clientMountPath " /proc/mounts; then
+    mount $STORAGE_NODE_IP:/registry $clientMountPath
+    echo $(date '+%Y-%m-%d %H:%M:%S') Writting to file... >> $clientMountPath/$(hostname)
+    echo $(date '+%Y-%m-%d %H:%M:%S') Successfully! >> $clientMountPath/$(hostname)
+  fi
+}
+
+# ======== Storage Node ======== #
+nfsDockerVersion=1.2.0
+serverMountPath=/data/registry
+
+initStorage() {
+  chmod 777 $serverMountPath
+}
+
+resetStorage() {
+  stopStorage
+
+  docker run -dit --name nfs-server \
+    -v $serverMountPath:/registry:rw \
+    -v /opt/harborapp/conf/storage/nfs/exports:/etc/exports:ro \
+    --net=host \
+    --privileged \
+    erichough/nfs-server:$nfsDockerVersion
+}
+
+stopStorage() {
+  [[ "$(docker ps -qa -f name=nfs-server)" ]] && docker rm -f nfs-server
+}
+
+checkStorage() {
+  nc -zv -w5 0.0.0.0 2049 && return
+  resetStorage
+}
+
+# ======== Web Node ======== #
+initWeb() {
+  ensureRegistryMounted
+}
+
+startWeb() {
+  ensureRegistryMounted
+  /opt/harbor/bin/manager.sh start docker-compose.web.yml
+}
+
+restartWeb() {
+  ensureRegistryMounted
+  /opt/harbor/bin/manager.sh restart docker-compose.web.yml
+}
+
+# ======== Replication Job Node ======== #
+initJob() {
+  ensureRegistryMounted
+}
+
+startJob() {
+  ensureRegistryMounted
+  /opt/harbor/bin/manager.sh start docker-compose.job.yml
+}
+
+restartJob() {
+  ensureRegistryMounted
+  /opt/harbor/bin/manager.sh restart docker-compose.job.yml
+}
+
+# Start executing.
+"$1${2^}"

From 87c7f03ccc92227dee88fcdb91fc1ce50ea117b5 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Tue, 30 Oct 2018 13:09:02 +0800
Subject: [PATCH 05/10] Fix Concurrency Issue for App Starting

---
 1.1.1/app/cluster.json.mustache         | 12 +++----
 1.1.1/vm-image/opt/harbor/bin/app.sh    | 45 -------------------------
 1.1.1/vm-image/opt/harborapp/bin/app.sh | 28 ++++++++++++++-
 3 files changed, 33 insertions(+), 52 deletions(-)
 delete mode 100755 1.1.1/vm-image/opt/harbor/bin/app.sh

diff --git a/1.1.1/app/cluster.json.mustache b/1.1.1/app/cluster.json.mustache
index 3168ace..dcccc8b 100644
--- a/1.1.1/app/cluster.json.mustache
+++ b/1.1.1/app/cluster.json.mustache
@@ -9,7 +9,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-57d2wld3"
+      "image": "img-8fmo1vww"
     },
     "instance_class": {{cluster.log_node.instance_class}},
     "count": 1,
@@ -41,7 +41,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-57d2wld3"
+      "image": "img-8fmo1vww"
     },
     "instance_class": {{cluster.storage_node.instance_class}},
     "count": {{cluster.storage_node.count}},
@@ -86,7 +86,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-57d2wld3"
+      "image": "img-8fmo1vww"
     },
     "instance_class": {{cluster.db_node.instance_class}},
     "count": 1,
@@ -119,7 +119,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-57d2wld3"
+      "image": "img-8fmo1vww"
     },
     "instance_class": {{cluster.cache_node.instance_class}},
     "count": 1,
@@ -144,7 +144,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-57d2wld3"
+      "image": "img-8fmo1vww"
     },
     "instance_class": {{cluster.job_node.instance_class}},
     "count": {{cluster.job_node.count}},
@@ -185,7 +185,7 @@
     "container": {
       "type": "kvm",
       "zone": "pek3a",
-      "image": "img-57d2wld3"
+      "image": "img-8fmo1vww"
     },
     "instance_class": {{cluster.web_node.instance_class}},
     "count": {{cluster.web_node.count}},
diff --git a/1.1.1/vm-image/opt/harbor/bin/app.sh b/1.1.1/vm-image/opt/harbor/bin/app.sh
deleted file mode 100755
index 9db4074..0000000
--- a/1.1.1/vm-image/opt/harbor/bin/app.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-source /opt/harbor/bin/env.sh
-minioVersion="RELEASE.2018-10-06T00-15-16Z"
-
-[[ "$NODE_ROLE" == "$1_node" ]] || exit 0
-
-genTokenWithLength() {
-  < /dev/urandom tr -dc A-Za-z0-9 | head -c${1:-$1}
-}
-
-storageGenerateToken() {
-  accessKey=$(genTokenWithLength 21)
-  secretKey=$(genTokenWithLength 41)
-
-  cat << EOF
-{"accessKey":"$accessKey","secretKey":"$secretKey"}
-EOF
-}
-
-storageStart() {
-  mkdir -p /data/storage/harbor
-
-  docker run -d \
-    --name minio \
-    -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
-    -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
-    -v /data/storage:/data \
-    -p 80:9000 \
-    --restart always \
-    minio/minio:${minioVersion} server /data
-}
-
-storageStop() {
-  if [ "$(docker ps -q -a -f name=minio)" ]; then
-    docker rm -f minio
-  fi
-}
-
-storageRestart() {
-  storageStop
-  storageStart
-}
-
-"$1${2^}"
diff --git a/1.1.1/vm-image/opt/harborapp/bin/app.sh b/1.1.1/vm-image/opt/harborapp/bin/app.sh
index 53f82ce..989791e 100644
--- a/1.1.1/vm-image/opt/harborapp/bin/app.sh
+++ b/1.1.1/vm-image/opt/harborapp/bin/app.sh
@@ -17,6 +17,31 @@ ensureRegistryMounted() {
   fi
 }
 
+# Ensure the same command runs only once at a time.
+runSingleton() {
+  lockFile=/tmp/harborapp-$2-$3.lock
+  i=0
+  max=25
+  while [ $i -lt $max ]; do
+    mkdir $lockFile && break || {
+      sleep 1
+      ((i++))
+    }
+  done
+
+  if [ $i -ge $max ]; then
+    myPid=$$
+    stalePid=`ps -df | grep "$1 $2 $3" | grep -v $myPid | grep -v grep | awk '{print $2}'`
+    echo Killing stale process [pid=$stalePid].
+    [ -z "$stalePid" ] || kill -9 $stalePid
+    mv $lockFile $lockFile.deleteme && rm -rf $lockFile.deleteme
+    mkdir $lockFile || exit 1
+  fi
+
+  "$2${3^}"
+  mv $lockFile $lockFile.deleteme && rm -rf $lockFile.deleteme
+}
+
 # ======== Storage Node ======== #
 nfsDockerVersion=1.2.0
 serverMountPath=/data/registry
@@ -76,4 +101,5 @@ restartJob() {
 }
 
 # Start executing.
-"$1${2^}"
+me=`basename "$0"`
+runSingleton $me $1 $2

From a44bf57acae891a83703c2bbd445d8b1a87228a5 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Thu, 8 Nov 2018 11:18:18 +0800
Subject: [PATCH 06/10] Make QingStor Zone as Required Field to Avoid Confusion

---
 1.1.1/app/config.json | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/1.1.1/app/config.json b/1.1.1/app/config.json
index 9318284..3f9a923 100644
--- a/1.1.1/app/config.json
+++ b/1.1.1/app/config.json
@@ -260,7 +260,6 @@
         "type": "integer",
         "default": 50,
         "min": 10,
-        "max": 1000,
         "step": 10,
         "required": "yes"
       }]
@@ -411,6 +410,14 @@
       "changeable": true,
       "default": "secretkey",
       "required": "yes"
+    }, {
+      "key": "QS_ZONE",
+      "label": "QS_ZONE",
+      "description": "Provide the zone of bucket to use",
+      "type": "string",
+      "changeable": true,
+      "default": "pek3a",
+      "required": "yes"
     }, {
       "key": "QS_BUCKET",
       "label": "QS_BUCKET",
@@ -427,14 +434,6 @@
       "changeable": false,
       "default": "registry-root",
       "required": "yes"
-    }, {
-      "key": "QS_ZONE",
-      "label": "QS_ZONE",
-      "description": "Provide the zone of bucket to use",
-      "type": "string",
-      "changeable": true,
-      "default": "pek3a",
-      "required": "no"
     }, {
       "key": "QS_URL",
       "label": "QS_URL",

From be2e9b8989ca024e8ac823b438f8565feccdecc9 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Thu, 29 Nov 2018 14:24:07 +0800
Subject: [PATCH 07/10] Address Review Comments

---
 1.1.1/app/config.json | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/1.1.1/app/config.json b/1.1.1/app/config.json
index 3f9a923..f74700b 100644
--- a/1.1.1/app/config.json
+++ b/1.1.1/app/config.json
@@ -227,8 +227,10 @@
         "description": "If QingStor will be used as Harbor's backend, then DO NOT create this node; otherwise please create one; default to 0 meaning not to create",
         "type": "integer",
         "default": 0,
-        "max": 1,
-        "min": 0,
+        "range": [
+          0,
+          1
+        ],
         "changeable": false,
         "required": "yes"
       }, {
@@ -301,8 +303,10 @@
         "description": "Number of nodes for replica cluster to create",
         "type": "integer",
         "default": 0,
-        "max": 1,
-        "min": 0,
+        "range": [
+          0,
+          1
+        ],
         "required": "yes"
       }, {
         "key": "instance_class",

From 3e11a382aad14f357da924a4bb1afc7b5aea7059 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Thu, 29 Nov 2018 14:41:27 +0800
Subject: [PATCH 08/10] Refine Option Desc

---
 1.1.1/app/locale/zh-cn.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/1.1.1/app/locale/zh-cn.json b/1.1.1/app/locale/zh-cn.json
index 08a1ecd..6dd6198 100644
--- a/1.1.1/app/locale/zh-cn.json
+++ b/1.1.1/app/locale/zh-cn.json
@@ -48,7 +48,7 @@
   "web_node":"主服务节点",
   "log_node":"日志节点",
   "storage_node":"存储节点",
-  "If QingStor will be used as Harbor's backend, then DO NOT create this node; otherwise please create one; default to 0 meaning not to create": "如果使用 QingStor 做 Harbor 的后端存储则不要创建此节点，否则需要创建；默认为 0 表示不创建",
+  "If QingStor will be used as Harbor's backend, then DO NOT create this node; otherwise please create one; default to 0 meaning not to create": "使用本地存储请选择 1 ，使用 QingStor 请选择 0",
   "Configuration properties":"服务配置参数",
   "HARBOR_HOST":"Harbor 地址",
   "HARBOR_MYSQL_PW":"MySQL 密码",

From aed9299f1236f3a80e723307c459727fcae9de17 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Thu, 29 Nov 2018 18:08:18 +0800
Subject: [PATCH 09/10] Update Config for Bucket Root Directory

---
 1.1.1/app/config.json       |  4 ++--
 1.1.1/app/locale/zh-cn.json | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/1.1.1/app/config.json b/1.1.1/app/config.json
index f74700b..61b2ac9 100644
--- a/1.1.1/app/config.json
+++ b/1.1.1/app/config.json
@@ -436,8 +436,8 @@
       "description": "The root direcotry or prefix for Harbor image storage; keep untouched if not using QingStor",
       "type": "string",
       "changeable": false,
-      "default": "registry-root",
-      "required": "yes"
+      "default": "",
+      "required": "no"
     }, {
       "key": "QS_URL",
       "label": "QS_URL",
diff --git a/1.1.1/app/locale/zh-cn.json b/1.1.1/app/locale/zh-cn.json
index 6dd6198..6361b03 100644
--- a/1.1.1/app/locale/zh-cn.json
+++ b/1.1.1/app/locale/zh-cn.json
@@ -26,16 +26,16 @@
   "The instance type for the cluster to run, such as high performance, high performance plus": "节点实例类型，比如性能型与超高性能型",
   "The volume type for each instance,such as high performance,high performance plus":"磁盘类型，比如性能型与超高性能型",
   "access_key_id": "access_key_id",
-  "Provide the secretkey to connect QingStor for storing Image data; keep untouched if not using QingStor": "提供可用于访问 QingStor 的 Secret Key，用于 Harbor 的镜像存储；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
-  "Provide an access_key_id to use": "提供要使用的 access_key_id",
+  "Provide the secretkey to connect QingStor for storing Image data; keep untouched if not using QingStor": "（若使用本地存储请忽略此项）提供可用于访问 QingStor 的 Secret Key，用于 Harbor 的镜像存储",
+  "Provide an access_key_id to use": "（若使用本地存储请忽略此项）提供要使用的 access_key_id，用于 Harbor 的镜像存储",
   "secret_access_key": "secret_access_key",
-  "Provide the accesskey to connect QingStor for storing Image data; keep untouched if not using QingStor": "提供可用于访问 QingStor 的 Access Key，用于 Harbor 的镜像存储；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
-  "Provide the Address of QingStor": "提供QingStor的地址",
-  "zone": "区域",
-  "Provide the zone of bucket to use": "提供要使用的存储空间区域",
-  "The root direcotry or prefix for Harbor image storage; keep untouched if not using QingStor":"提供用于存储镜像的根目录名称；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
-  "bucket": "存储区域",
-  "Provide the bucket name to use; keep untouched if not using QingStor": "提供要使用的存储 Bucket 名称，需要提前创建，详情见《Harbor on QingCloud AppCenter 用户手册》；如果不使用 QingStor 则可以忽略此配置（保留默认值即可）",
+  "Provide the accesskey to connect QingStor for storing Image data; keep untouched if not using QingStor": "（若使用本地存储请忽略此项）提供可用于访问 QingStor 的 Access Key，用于 Harbor 的镜像存储",
+  "Provide the Address of QingStor": "（若使用本地存储请忽略此项）提供 QingStor 的地址",
+  "zone": "（若使用本地存储请忽略此项）QingStor 区域",
+  "Provide the zone of bucket to use": "（若使用本地存储请忽略此项）提供 QingStor 存储空间区域",
+  "The root direcotry or prefix for Harbor image storage; keep untouched if not using QingStor": "（若使用本地存储请忽略此项）提供存储桶里用于存储镜像的根目录名称，默认为空，表示使用整个桶；为了保证数据完整，创建集群后将无法更改此项",
+  "bucket": "QingStor 存储桶（ Bucket ）",
+  "Provide the bucket name to use; keep untouched if not using QingStor": "（若使用本地存储请忽略此项）存储桶名称，需要提前创建，详情见《Harbor on QingCloud AppCenter 用户手册》；如果存储桶已有其他数据，可考虑同时配置后面的 [存储根目录] 参数避免冲突；为了保证数据完整，创建集群后将无法更改此项",
   "volume size": "磁盘大小",
   "Volume Size": "磁盘大小",
   "Volume Class":"磁盘类型",

From b6121fcc7804714a2e90048c5c3edff643864394 Mon Sep 17 00:00:00 2001
From: Hongliang Wang <hongliangwang@yunify.com>
Date: Thu, 29 Nov 2018 20:37:09 +0800
Subject: [PATCH 10/10] Refactor Layout Structure

---
 {1.1.1/app => app}/cluster.json.mustache            |   0
 {1.1.1/app => app}/config.json                      |   0
 {1.1.1/app => app}/locale/en.json                   |   0
 {1.1.1/app => app}/locale/zh-cn.json                |   0
 .../etc/confd/conf.d/env.sh.toml                    |   0
 .../etc/confd/conf.d/exports.toml                   |   0
 .../etc/confd/conf.d/hosts.toml                     |   0
 .../etc/confd/conf.d/registry.config.toml           |   0
 .../etc/confd/templates/env.sh.tmpl                 |   0
 .../etc/confd/templates/exports.tmpl                |   0
 .../etc/confd/templates/hosts.tmpl                  |   0
 .../etc/confd/templates/registry.config.yml.tmpl    |   0
 .../vm-image => vm-image}/opt/harbor/bin/README.md  |   0
 .../opt/harbor/bin/cleanImage.sh                    |   0
 .../opt/harbor/bin/docker-compose.allinone.yml      |   0
 .../opt/harbor/bin/docker-compose.db.yml            |   0
 .../opt/harbor/bin/docker-compose.host.yml          |   0
 .../opt/harbor/bin/docker-compose.job.yml           |   0
 .../opt/harbor/bin/docker-compose.log.yml           |   0
 .../opt/harbor/bin/docker-compose.redis.yml         |   0
 .../opt/harbor/bin/docker-compose.web.yml           |   0
 {1.1.1/vm-image => vm-image}/opt/harbor/bin/env.sh  |   0
 .../harbor/bin/harbor-online-installer-v1.1.1.tgz   | Bin
 .../bin/harbor/common/config/nginx/nginx.conf       |   0
 .../opt/harbor/bin/install.ha.sh                    |   0
 .../vm-image => vm-image}/opt/harbor/bin/manager.sh |   0
 .../harbor/bin/snapshot/WX20170517-154334@2x.png    | Bin
 .../opt/harbor/bin/uninstall.sh                     |   0
 .../vm-image => vm-image}/opt/harborapp/bin/app.sh  |   0
 29 files changed, 0 insertions(+), 0 deletions(-)
 rename {1.1.1/app => app}/cluster.json.mustache (100%)
 rename {1.1.1/app => app}/config.json (100%)
 rename {1.1.1/app => app}/locale/en.json (100%)
 rename {1.1.1/app => app}/locale/zh-cn.json (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/conf.d/env.sh.toml (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/conf.d/exports.toml (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/conf.d/hosts.toml (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/conf.d/registry.config.toml (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/templates/env.sh.tmpl (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/templates/exports.tmpl (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/templates/hosts.tmpl (100%)
 rename {1.1.1/vm-image => vm-image}/etc/confd/templates/registry.config.yml.tmpl (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/README.md (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/cleanImage.sh (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.allinone.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.db.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.host.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.job.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.log.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.redis.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/docker-compose.web.yml (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/env.sh (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/harbor-online-installer-v1.1.1.tgz (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/harbor/common/config/nginx/nginx.conf (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/install.ha.sh (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/manager.sh (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/snapshot/WX20170517-154334@2x.png (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harbor/bin/uninstall.sh (100%)
 rename {1.1.1/vm-image => vm-image}/opt/harborapp/bin/app.sh (100%)

diff --git a/1.1.1/app/cluster.json.mustache b/app/cluster.json.mustache
similarity index 100%
rename from 1.1.1/app/cluster.json.mustache
rename to app/cluster.json.mustache
diff --git a/1.1.1/app/config.json b/app/config.json
similarity index 100%
rename from 1.1.1/app/config.json
rename to app/config.json
diff --git a/1.1.1/app/locale/en.json b/app/locale/en.json
similarity index 100%
rename from 1.1.1/app/locale/en.json
rename to app/locale/en.json
diff --git a/1.1.1/app/locale/zh-cn.json b/app/locale/zh-cn.json
similarity index 100%
rename from 1.1.1/app/locale/zh-cn.json
rename to app/locale/zh-cn.json
diff --git a/1.1.1/vm-image/etc/confd/conf.d/env.sh.toml b/vm-image/etc/confd/conf.d/env.sh.toml
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/conf.d/env.sh.toml
rename to vm-image/etc/confd/conf.d/env.sh.toml
diff --git a/1.1.1/vm-image/etc/confd/conf.d/exports.toml b/vm-image/etc/confd/conf.d/exports.toml
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/conf.d/exports.toml
rename to vm-image/etc/confd/conf.d/exports.toml
diff --git a/1.1.1/vm-image/etc/confd/conf.d/hosts.toml b/vm-image/etc/confd/conf.d/hosts.toml
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/conf.d/hosts.toml
rename to vm-image/etc/confd/conf.d/hosts.toml
diff --git a/1.1.1/vm-image/etc/confd/conf.d/registry.config.toml b/vm-image/etc/confd/conf.d/registry.config.toml
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/conf.d/registry.config.toml
rename to vm-image/etc/confd/conf.d/registry.config.toml
diff --git a/1.1.1/vm-image/etc/confd/templates/env.sh.tmpl b/vm-image/etc/confd/templates/env.sh.tmpl
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/templates/env.sh.tmpl
rename to vm-image/etc/confd/templates/env.sh.tmpl
diff --git a/1.1.1/vm-image/etc/confd/templates/exports.tmpl b/vm-image/etc/confd/templates/exports.tmpl
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/templates/exports.tmpl
rename to vm-image/etc/confd/templates/exports.tmpl
diff --git a/1.1.1/vm-image/etc/confd/templates/hosts.tmpl b/vm-image/etc/confd/templates/hosts.tmpl
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/templates/hosts.tmpl
rename to vm-image/etc/confd/templates/hosts.tmpl
diff --git a/1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl b/vm-image/etc/confd/templates/registry.config.yml.tmpl
similarity index 100%
rename from 1.1.1/vm-image/etc/confd/templates/registry.config.yml.tmpl
rename to vm-image/etc/confd/templates/registry.config.yml.tmpl
diff --git a/1.1.1/vm-image/opt/harbor/bin/README.md b/vm-image/opt/harbor/bin/README.md
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/README.md
rename to vm-image/opt/harbor/bin/README.md
diff --git a/1.1.1/vm-image/opt/harbor/bin/cleanImage.sh b/vm-image/opt/harbor/bin/cleanImage.sh
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/cleanImage.sh
rename to vm-image/opt/harbor/bin/cleanImage.sh
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.allinone.yml b/vm-image/opt/harbor/bin/docker-compose.allinone.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.allinone.yml
rename to vm-image/opt/harbor/bin/docker-compose.allinone.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.db.yml b/vm-image/opt/harbor/bin/docker-compose.db.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.db.yml
rename to vm-image/opt/harbor/bin/docker-compose.db.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.host.yml b/vm-image/opt/harbor/bin/docker-compose.host.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.host.yml
rename to vm-image/opt/harbor/bin/docker-compose.host.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.job.yml b/vm-image/opt/harbor/bin/docker-compose.job.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.job.yml
rename to vm-image/opt/harbor/bin/docker-compose.job.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.log.yml b/vm-image/opt/harbor/bin/docker-compose.log.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.log.yml
rename to vm-image/opt/harbor/bin/docker-compose.log.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.redis.yml b/vm-image/opt/harbor/bin/docker-compose.redis.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.redis.yml
rename to vm-image/opt/harbor/bin/docker-compose.redis.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/docker-compose.web.yml b/vm-image/opt/harbor/bin/docker-compose.web.yml
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/docker-compose.web.yml
rename to vm-image/opt/harbor/bin/docker-compose.web.yml
diff --git a/1.1.1/vm-image/opt/harbor/bin/env.sh b/vm-image/opt/harbor/bin/env.sh
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/env.sh
rename to vm-image/opt/harbor/bin/env.sh
diff --git a/1.1.1/vm-image/opt/harbor/bin/harbor-online-installer-v1.1.1.tgz b/vm-image/opt/harbor/bin/harbor-online-installer-v1.1.1.tgz
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/harbor-online-installer-v1.1.1.tgz
rename to vm-image/opt/harbor/bin/harbor-online-installer-v1.1.1.tgz
diff --git a/1.1.1/vm-image/opt/harbor/bin/harbor/common/config/nginx/nginx.conf b/vm-image/opt/harbor/bin/harbor/common/config/nginx/nginx.conf
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/harbor/common/config/nginx/nginx.conf
rename to vm-image/opt/harbor/bin/harbor/common/config/nginx/nginx.conf
diff --git a/1.1.1/vm-image/opt/harbor/bin/install.ha.sh b/vm-image/opt/harbor/bin/install.ha.sh
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/install.ha.sh
rename to vm-image/opt/harbor/bin/install.ha.sh
diff --git a/1.1.1/vm-image/opt/harbor/bin/manager.sh b/vm-image/opt/harbor/bin/manager.sh
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/manager.sh
rename to vm-image/opt/harbor/bin/manager.sh
diff --git a/1.1.1/vm-image/opt/harbor/bin/snapshot/WX20170517-154334@2x.png b/vm-image/opt/harbor/bin/snapshot/WX20170517-154334@2x.png
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/snapshot/WX20170517-154334@2x.png
rename to vm-image/opt/harbor/bin/snapshot/WX20170517-154334@2x.png
diff --git a/1.1.1/vm-image/opt/harbor/bin/uninstall.sh b/vm-image/opt/harbor/bin/uninstall.sh
similarity index 100%
rename from 1.1.1/vm-image/opt/harbor/bin/uninstall.sh
rename to vm-image/opt/harbor/bin/uninstall.sh
diff --git a/1.1.1/vm-image/opt/harborapp/bin/app.sh b/vm-image/opt/harborapp/bin/app.sh
similarity index 100%
rename from 1.1.1/vm-image/opt/harborapp/bin/app.sh
rename to vm-image/opt/harborapp/bin/app.sh