StorageOSCluster Resource Confi
images.initContainer |
StorageOS init container image |
-storageos/init:1.0.0 |
+storageos/init:2.1.0 |
images.kubeSchedulerContainer |
@@ -1349,6 +1354,11 @@ StorageOSCluster Resource Confi
/var/lib/kubelet/plugins/kubernetes.io~storageos |
+storageClassName |
+The name of the default StorageClass created for StorageOS volumes |
+fast |
+
+
tlsEtcdSecretRefName |
Secret containing etcd client certificates |
|
diff --git a/docs/reference/cluster-operator/examples/index.html b/docs/reference/cluster-operator/examples/index.html
index 21ba5a4fa..525375327 100644
--- a/docs/reference/cluster-operator/examples/index.html
+++ b/docs/reference/cluster-operator/examples/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/cluster-operator/index.html b/docs/reference/cluster-operator/index.html
index 34c1b2475..a5b71d40f 100644
--- a/docs/reference/cluster-operator/index.html
+++ b/docs/reference/cluster-operator/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/cluster-operator/index.xml b/docs/reference/cluster-operator/index.xml
index 94441f634..abb3c7fec 100644
--- a/docs/reference/cluster-operator/index.xml
+++ b/docs/reference/cluster-operator/index.xml
@@ -108,6 +108,11 @@
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -140,7 +145,7 @@
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -253,6 +258,11 @@
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/docs/reference/cluster-operator/install/index.html b/docs/reference/cluster-operator/install/index.html
index afe0cf7cc..7ec2fa11f 100644
--- a/docs/reference/cluster-operator/install/index.html
+++ b/docs/reference/cluster-operator/install/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/cluster-operator/upgrade/index.html b/docs/reference/cluster-operator/upgrade/index.html
index cb33e153a..faccd7e51 100644
--- a/docs/reference/cluster-operator/upgrade/index.html
+++ b/docs/reference/cluster-operator/upgrade/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/contributing/index.html b/docs/reference/contributing/index.html
index ba74e2a87..4ec16619e 100644
--- a/docs/reference/contributing/index.html
+++ b/docs/reference/contributing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/filesystems/index.html b/docs/reference/filesystems/index.html
index b3c421a1d..6556e1b45 100644
--- a/docs/reference/filesystems/index.html
+++ b/docs/reference/filesystems/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/gui/index.html b/docs/reference/gui/index.html
index 20552661c..b2cc777d1 100644
--- a/docs/reference/gui/index.html
+++ b/docs/reference/gui/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/index.html b/docs/reference/index.html
index 7e91dc78e..7cbdbfb3b 100644
--- a/docs/reference/index.html
+++ b/docs/reference/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/init-container/index.html b/docs/reference/init-container/index.html
index 58bbc5c5e..8aef3d56a 100644
--- a/docs/reference/init-container/index.html
+++ b/docs/reference/init-container/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/labels/index.html b/docs/reference/labels/index.html
index 2081ebe2b..bd62e0819 100644
--- a/docs/reference/labels/index.html
+++ b/docs/reference/labels/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/licence/index.html b/docs/reference/licence/index.html
index 3f9b3024c..4a2fa0d49 100644
--- a/docs/reference/licence/index.html
+++ b/docs/reference/licence/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/open_source_attribution/index.html b/docs/reference/open_source_attribution/index.html
index d9cc81d01..a983528a2 100644
--- a/docs/reference/open_source_attribution/index.html
+++ b/docs/reference/open_source_attribution/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/release_notes/index.html b/docs/reference/release_notes/index.html
index c16694de6..ec513142e 100644
--- a/docs/reference/release_notes/index.html
+++ b/docs/reference/release_notes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/resource_requests_and_limits/index.html b/docs/reference/resource_requests_and_limits/index.html
index 5e9bf810a..691f3c034 100644
--- a/docs/reference/resource_requests_and_limits/index.html
+++ b/docs/reference/resource_requests_and_limits/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/telemetry/index.html b/docs/reference/telemetry/index.html
index 2d7224851..32c5cb615 100644
--- a/docs/reference/telemetry/index.html
+++ b/docs/reference/telemetry/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/reference/tolerations/index.html b/docs/reference/tolerations/index.html
index 0b9204f98..1ff4293a9 100644
--- a/docs/reference/tolerations/index.html
+++ b/docs/reference/tolerations/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/self-eval/index.html b/docs/self-eval/index.html
index a583f9b95..f77932ea1 100644
--- a/docs/self-eval/index.html
+++ b/docs/self-eval/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/support/index.html b/docs/support/index.html
index 6075b80a5..907cdb3f4 100644
--- a/docs/support/index.html
+++ b/docs/support/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/cassandra/index.html b/docs/usecases/cassandra/index.html
index c1dcf9c01..ee1cbfe85 100644
--- a/docs/usecases/cassandra/index.html
+++ b/docs/usecases/cassandra/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/elasticsearch/index.html b/docs/usecases/elasticsearch/index.html
index ee617c509..e8be2b98a 100644
--- a/docs/usecases/elasticsearch/index.html
+++ b/docs/usecases/elasticsearch/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/index.html b/docs/usecases/index.html
index 7b114a72a..da19525fd 100644
--- a/docs/usecases/index.html
+++ b/docs/usecases/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/index.xml b/docs/usecases/index.xml
index c43653ca2..86237518d 100644
--- a/docs/usecases/index.xml
+++ b/docs/usecases/index.xml
@@ -1573,7 +1573,7 @@ files into blobs which are then sent to MinIO.
<h2 id="prerequisites">Prerequisites</h2>
<p>Here are the prerequisites for running Velero in your Kubernetes cluster:</p>
<ol>
-<li>Kubernetes cluster version 1.13+ with DNS</li>
+<li>Kubernetes cluster version 1.15 to 1.20 with DNS</li>
<li>Kubectl installed</li>
<li>Velero cli installed <a href="https://Velero.io/docs/main/basic-install/">https://Velero.io/docs/main/basic-install/</a></li>
</ol>
diff --git a/docs/usecases/influxdb/index.html b/docs/usecases/influxdb/index.html
index c71d5bb9a..005316bd2 100644
--- a/docs/usecases/influxdb/index.html
+++ b/docs/usecases/influxdb/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/jenkins/index.html b/docs/usecases/jenkins/index.html
index 22c6d2e57..a2441a594 100644
--- a/docs/usecases/jenkins/index.html
+++ b/docs/usecases/jenkins/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/kafka/index.html b/docs/usecases/kafka/index.html
index 01ce347a1..66bf76099 100644
--- a/docs/usecases/kafka/index.html
+++ b/docs/usecases/kafka/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/kubevirt/index.html b/docs/usecases/kubevirt/index.html
index 599a327b8..506f71896 100644
--- a/docs/usecases/kubevirt/index.html
+++ b/docs/usecases/kubevirt/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/mssql/index.html b/docs/usecases/mssql/index.html
index 5b3a09f5d..f02536168 100644
--- a/docs/usecases/mssql/index.html
+++ b/docs/usecases/mssql/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/mysql/index.html b/docs/usecases/mysql/index.html
index 079c8d783..0daae549d 100644
--- a/docs/usecases/mysql/index.html
+++ b/docs/usecases/mysql/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/nginx/index.html b/docs/usecases/nginx/index.html
index 4bc473422..ccd64bd39 100644
--- a/docs/usecases/nginx/index.html
+++ b/docs/usecases/nginx/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/postgres/index.html b/docs/usecases/postgres/index.html
index 6e202520b..25c343cba 100644
--- a/docs/usecases/postgres/index.html
+++ b/docs/usecases/postgres/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/prometheus/index.html b/docs/usecases/prometheus/index.html
index a412111d2..d6de293a2 100644
--- a/docs/usecases/prometheus/index.html
+++ b/docs/usecases/prometheus/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/redis/index.html b/docs/usecases/redis/index.html
index e687294fe..0bf74cc98 100644
--- a/docs/usecases/redis/index.html
+++ b/docs/usecases/redis/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/sidecar-backups/index.html b/docs/usecases/sidecar-backups/index.html
index c7497ad50..d77536330 100644
--- a/docs/usecases/sidecar-backups/index.html
+++ b/docs/usecases/sidecar-backups/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/docs/usecases/velero-backups/index.html b/docs/usecases/velero-backups/index.html
index 3ca739515..a42b7ac97 100644
--- a/docs/usecases/velero-backups/index.html
+++ b/docs/usecases/velero-backups/index.html
@@ -36,7 +36,7 @@
And here’s how Velero restores work:
In our case, we will use MinIO in the Kubernetes cluster as the “cloud provider” pictured in the backup diagram.">
-
+
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -1176,7 +1176,7 @@ Using Velero to backup yo
Prerequisites
Here are the prerequisites for running Velero in your Kubernetes cluster:
-- Kubernetes cluster version 1.13+ with DNS
+- Kubernetes cluster version 1.15 to 1.20 with DNS
- Kubectl installed
- Velero cli installed https://Velero.io/docs/main/basic-install/
diff --git a/docs/usecases/zookeeper/index.html b/docs/usecases/zookeeper/index.html
index b57f95298..307533460 100644
--- a/docs/usecases/zookeeper/index.html
+++ b/docs/usecases/zookeeper/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/index.html b/index.html
index 972fb4cca..703dfe705 100644
--- a/index.html
+++ b/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/index.json b/index.json
index 0db566038..01b3d75b2 100644
--- a/index.json
+++ b/index.json
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json b/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json
new file mode 100644
index 000000000..21b9b44e9
--- /dev/null
+++ b/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json
@@ -0,0 +1 @@
+[{"body":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher OpenShift ","excerpt":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher …","ref":"/docs/reference/cluster-operator/install/","title":"Install"},{"body":"Current Version The current StorageOS version is v2.3.2\nSections Our documentation is arranged into sections, accessible from the navigation bar on the left.\nIntroduction - Quickstart and support information.\nConcepts - Architectural and deep technical information.\nPrerequisites - We require certain prerequisites to be met for the product to function correctly. Please read these carefully and ensure they are implemented.\nPlatforms - Due to differences in the various orchestrators that StorageOS can run under, we list install guides and other platform specific operations here.\nOperations - Platform agnostic operations.\nUse Cases - A set of examples to get up and running with StorageOS quickly.\nReference - Information on our GUI, CLI, and other important information.\n","excerpt":"Current Version The current StorageOS version is v2.3.2\nSections Our documentation is arranged into …","ref":"/docs/introduction/","title":"Introduction"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed service such as EKS, AKS, GKE, DO or DockerEE platform can use the following Kubernetes guide to install StorageOS.\n StorageOS supports the five most recent Kubernetes releases, at minimum.\n Make sure to add a StorageOS licence after installing.\n 1.20 1.19 1.18 1.17 1.16 1.15 Install StorageOS on Kubernetes 1.20 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.19 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.18 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.17 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.16 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.15 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed …","ref":"/docs/install/kubernetes/","title":"Kubernetes"},{"body":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some significant enhancements over our v1 product. We\u0026rsquo;ve built V2 based on our observations of trends in the industry, as well as our own experience.\nMany of our customers want to run big clusters - in the tens or hundreds of nodes. In these sorts of big environments, the challenges multiply. Not only do we need to scale well, but we also need to be more failure tolerant. Bigger environments typically suffer higher failure rates (more nodes = greater chance of something failing), but are also subject to all sorts of transient conditions such as network partitions.\nThe second trend we\u0026rsquo;ve seen become increasingly common is the desire to run multiple clusters, and consume storage between them in some way - sometimes to implement novel topologies such as a centralised storage cluster with satellites consuming the storage, and sometimes to replicate data between those clusters for HA or DR purposes.\nWe\u0026rsquo;ve built V2 with these architectures and design patterns in mind. Not only does it scale well, but it contains the foundations we need to implement a rich set of multi-cluster functionality.\nUpgraded Control Plane At the heart of the V2 release is an upgraded control plane. We\u0026rsquo;ve changed a lot here. Firstly, our usage of etcd is vastly improved. We\u0026rsquo;ve learnt a lot about the subtleties of distributed consensus in the last year, particularly in noisy or unpredictable environments. Not only is StorageOS V2 much lighter on your etcd cluster, but it\u0026rsquo;s a lot more tolerant of transient failure conditions that are often found in cloud environments, or clusters under heavy load.\nWe spent some time describing and testing our internal state machine using the TLA+ formal verification language. This allows us to have a much higher degree of confidence that our algorithms will behave correctly, particularly under hard-to-test edge cases and failure conditions.\nAdditionally, we\u0026rsquo;ve changed the way volumes behave with respect to centralised scheduling. Each volume group (consisting of a master and 0 or more replicas) now behaves as a strongly consistent unit allowing it to take action independent of the activities of the rest of the cluster. Other state can be distributed around the cluster using eventually consistent mechanisms. This approach inherently scales better and allows StorageOS V2 to effectively manage many more nodes and volumes than before.\nWe\u0026rsquo;ve implemented TLS on all endpoints. Not only does this give you encrypted traffic between nodes in your storage cluster, it also protects all endpoints with strong, public key based authentication. Today\u0026rsquo;s IT environments can\u0026rsquo;t rely on firewalls to keep bad actors out - they must implement security at all layers within the stack - defense in depth. While we recognise that this brings a welcome relief to many security conscious administrators, we also know that managing certificate authorities (CAs) can be an unwelcome source of complexity. For this reason, StorageOS V2 implements an internal CA by default, to manage this complexity for you. If you\u0026rsquo;d prefer to integrate your own CA, we support that too - it\u0026rsquo;s up to you.\nFinally - our logging has undergone a complete transformation in this edition. We know that systems engineers and operators don\u0026rsquo;t just value headline features, but that observability and diagnostics are equally important. All logs are now decorated with rich context to help you understand what is happening within your cluster, and we\u0026rsquo;ll output in json by default, for easy ingestion into log aggregators such as Elasticsearch.\nUpgraded Data Plane Not to be outdone, our data plane contains some significant improvements.\nFirstly, we\u0026rsquo;ve completely re-written our sync algorithm (see Delta Sync, used when seeding or catching up replicas that have been offline or partitioned. Our new algorithm uses a Hash List to sync only changed sections of a volume (similar in some ways to what rsync does). StorageOS maintains these hashes during normal operation, meaning that when resyncing a failed replica, for example after a node reboot, we can very quickly and efficiently catch this replica up, rather than needing to promote and build a new one from scratch. This improves resiliency within your cluster, and prevents using excessive network bandwidth during failover conditions - at a time when it might be needed the most.\nSecondly, a new threading model, with dynamic pool sizing, means that StorageOS is faster, a lot faster. In our tests we observed improvements across the board, with improvements in throughput of up to 135% for some scenarios.\n","excerpt":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some …","ref":"/docs/introduction/overview/","title":"What is StorageOS V2"},{"body":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source implementation of the SCSI target.\nWe require the following modules to be loaded:\n target_core_mod tcm_loop target_core_file configfs target_core_user uio N.B. Other applications utilising TCMU cannot be run concurrently with StorageOS. Doing so may result in corruption of data. On startup, StorageOS will detect if other applications are using TCMU and fall back to FUSE. However if StorageOS is started first there is no mechanism for StorageOS to fallback to FUSE if another application begins to use TCMU. TCMU can be disabled using the DISABLE_TCMU StorageOSCluster spec parameter.\n Depending on the distribution, the modules are shipped as part of the base kernel package or as part of a kernel extras package which needs to be installed.\nDistribution Specifics The following distributions are supported by default:\n RHEL 7.5 CentOS 7 Debian 9 Ubuntu Azure RancherOS - Note CSI is not supported on RancherOS Ubuntu 16.04/18.04 requires the installation of additional packages.\n N.B. Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not provide the necessary linux-image-extra package - see below for more information\n Ubuntu Package Installation Ubuntu 16.04/18.04 Generic and Ubuntu 16.04 GCE require extra packages:\nUbuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Ubuntu With AWS Or GCE Kernels Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not yet provide the linux-image-extra package. As such you should either use Debian, CentOS or RHEL, or install the non-cloud-provider optimised Ubuntu kernel.\nInstalling the non-cloud-provider optimised Ubuntu kernel is something that should only be done with full understanding of potential ramifications.\nsudo apt -y update sudo apt install -y linux-virtual linux-image-extra-virtual sudo apt purge -y linux*aws # Reboot the machine sudo shutdown -r now Automatic Configuration Once required kernel modules are installed on the system, for convenience we provide a container which will ensure the appropriate modules are loaded and ready for use at runtime. On Docker installations, you will need to run the init container prior to starting StorageOS. Our installation guides for Kubernetes and OpenShift include this step.\n# Load the required kernel modules. The Kubernetes and OpenShift installations include this step. docker run --name enable_lio \\ --privileged \\ --rm \\ --cap-add=SYS_ADMIN \\ -v /lib/modules:/lib/modules \\ -v /sys:/sys:rshared \\ storageos/init:0.2 Manual Configuration For those wishing to manage their own kernel configuration, rather than using the init container, perform the following steps:\n Ensure kernel modules are all loaded per list above Ensure configfs is loaded and mounted at /sys/kernel/config ","excerpt":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source …","ref":"/docs/prerequisites/systemconfiguration/","title":"System Configuration"},{"body":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters of the StorageOSCluster custom resource and their default values.\n Parameter Description Default csi.deploymentStrategy CSI helper deployment strategy (statefulset or deployment) statefulset csi.enable Enable CSI setup false csi.enableControllerPublishCreds Enable CSI controller publish credentials false csi.enableNodePublishCreds Enable CSI node publish credentials false csi.enableProvisionCreds Enable CSI provision credentials false debug Enable debug mode for all the cluster nodes false disableFencing Disable Pod fencing false disableScheduler Disable StorageOS scheduler false disableTCMU Disable TCMU to allow co-existence with other TCMU users. Disabling TCMU degrades performance false disableTelemetry Disable telemetry reports false forceTCMU Forces TCMU to be enabled or causes StorageOS to abort startup false images.apiManagerContainer StorageOS API Manager container image storageos/api-manager:v1.0.0 images.csiClusterDriverRegistrarContainer CSI Cluster Driver Registrar Container image quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 images.csiExternalAttacherContainer CSI External Attacher Container image quay.io/k8scsi/csi-attacher:v1.0.1 images.csiExternalProvisionerContainer CSI External Provisioner Container image storageos/csi-provisioner:v1.0.1 ìmages.csiLivenessProbeContainer CSI Liveness Probe Container Image quay.io/k8scsi/livenessprobe:v1.0.1 images.csiNodeDriverRegistrarContainer CSI Node Driver Registrar Container image quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 images.hyperkubeContainer Deprecated field - HyperKube Container image Default dependent on Scheduler version images.initContainer StorageOS init container image storageos/init:2.1.0 images.kubeSchedulerContainer Kube scheduler container image Default dependent on Scheduler version images.nfsContainer StorageOS nfs container image storageos/nfs:1.0.0 images.nodeContainer StorageOS node container image storageos/node:v2.3.2 ingress.annotations Annotations of the ingress used by the cluster ingress.enable Enable ingress for the cluster false ingress.hostname Hostname to be used in cluster ingress storageos.local ingress.tls Enable TLS for the ingress false k8sDistro The name of the Kubernetes distribution is use, e.g. rancher or eks kvBackend.address Comma-separated list of addresses of external key-value store. (1.2.3.4:2379,2.3.4.5:2379) kvBackend.backend (v2 deprecated) Name of the key-value store to use. Set to etcd for external key-value store. embedded namespace Namespace where storageos cluster resources are created kube-system nodeSelectorTerms Set node selector for storageos pod placement pause Pause the operator for cluster maintenance false resources Set resource requirements for the containers secretRefName Reference name of storageos secret secretRefNamespace Namespace of storageos secret service.annotations Annotations of the Service used by the cluster service.externalPort External port of the Service used by the cluster 5705 service.internalPort Internal port of the Service used by the cluster 5705 service.name Name of the Service used by the cluster storageos service.type Type of the Service used by the cluster ClusterIP sharedDir Path to be shared with kubelet container when deployed as a pod /var/lib/kubelet/plugins/kubernetes.io~storageos storageClassName The name of the default StorageClass created for StorageOS volumes fast tlsEtcdSecretRefName Secret containing etcd client certificates tlsEtcdSecretRefNamespace Namespace of the tlsEtcdSecretRefName tolerations Set pod tolerations for storageos pod placement ","excerpt":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters …","ref":"/docs/reference/cluster-operator/configuration/","title":"Configuration"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment of an etcd cluster and configuration of CRI-O PID limits.\n For OpenShift upgrades, refer to the OpenShift platform page.\n If you have installed OpenShift in AWS ensure that the requisite ports are opened for the worker nodes\u0026rsquo; security group.\n Make sure to add a StorageOS licence after installing.\n StorageOS v2 supports OpenShift v4. For more information check the OpenShift platform page.\n OperatorHub Manual OperatorHub Select the OperatorHub from the Catalog sub menu and search for StorageOS\n Choose between using the RedHat Market Place or the Community Operators installation.\n Select StorageOS and click install\n Select the install options\n Make sure the Approval Strategy is set to Manual. So the StorageOS Operator doesn\u0026rsquo;t upgrade versions without explicit approval.\n Start the approval procedure\n Follow the approval link\n Approve the installation\n The StorageOS Cluster Operator is installed along the required CRDs\n Create a Secret in the openshift-operators project\n Use the YAML options to create a secret containing the apiUsername and an apiPassword key. The username and password defined in the secret will be used to authenticate when using the StorageOS CLI and GUI. For the communication between StorageOS and OpenShift, the CSI credentials csiProvisionUsername, csiProvisionPassword, csiControllerPublishUsername, csiControllerPublishPassword, csiNodePublishUsername, csiNodePublishPassword are needed. Take note of which project you created the secret in.\nInput the Secret as YAML for simplicity.\napiVersion:v1kind:Secretmetadata:name:storageos-apinamespace:openshift-operatorstype:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9z Go to the \u0026ldquo;Installed Operators\u0026rdquo;\n Verify that the StorageOS Cluster Operator is installed\n Go to the \u0026ldquo;StorageOS Cluster\u0026rdquo; section\n Create the StorageOS Cluster\n A StorageOS Cluster is defined using a Custom Resource Definition\n Create the Custom Resource\nThe StorageOS cluster resource describes the StorageOS cluster that will be created. Parameters such as the secretRefName, the secretRefNamespace and the kvBackend.address are mandatory.\nAdditional spec parameters are available on the Cluster Operator configuration page.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: storageos namespace: openshift-operators spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference the Secret created in the previous step secretRefNamespace: \u0026#34;openshift-operators\u0026#34; # Namespace of the Secret created in the previous step k8sDistro: \u0026#34;openshift\u0026#34; kvBackend: address: \u0026#39;storageos-etcd-client.etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Verify that the StorageOS Cluster Resource enters a running state.\n It can take up to a minute to report the StorageOS Pods ready\n Check the StorageOS Pods in the kube-system project\n A Status of 3/3 for the Daemonset Pods indicates that StorageOS is bootstrapped successfully.\n License cluster\n A newly installed StorageOS cluster does not include a licence. A cluster must be licensed within 24 hours of the installation. For more information, check the reference licence page.\n You can apply a Free Developer licence following the operations licensing page, or purchase a licence contacting sales@storageos.com.\n Manual install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment …","ref":"/docs/install/openshift/","title":"OpenShift"},{"body":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all nodes in the cluster:\n Port Number TCP/UDP Use 5701 TCP gRPC 5703 TCP DirectFS 5704 TCP Dataplane Supervisor 5705 TCP REST API 5711 TCP \u0026amp; UDP Gossip service 25705-25960 TCP RWX Volume Endpoints StorageOS also uses ephemeral ports to dial-out to these ports on other StorageOS nodes. For this reason, outgoing traffic should be enabled.\nFirewalls and VPS providers Some VPS providers (such as Digital Ocean) ship default firewall rulesets which must be updated to allow StorageOS to run. Some example rules are shown below - modify to taste.\nUFW For distributions using UFW, such as RHEL and derivatives:\nufw default allow outgoing ufw allow 5701:5711/tcp ufw allow 5711/udp ufw allow 25705:25960/tcp Firewalld For distributions that enable firewalld to control iptables such as some installations of OpenShift.\nfirewall-cmd --permanent --new-service=storageos firewall-cmd --permanent --service=storageos --add-port=5700-5800/tcp --add-port=25705-25960/tcp firewall-cmd --add-service=storageos --zone=public --permanent firewall-cmd --reload Iptables For those using plain iptables:\n# Inbound traffic iptables -I INPUT -i lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I INPUT -m state --state ESTABLISHED,RELATED -m comment --comment \u0026#39;Permit established traffic\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 5701:5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p udp --dport 5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 25705:25960 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT # Outbound traffic iptables -I OUTPUT -o lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I OUTPUT -d 0.0.0.0/0 -m comment --comment \u0026#39;Permit outbound traffic\u0026#39; -j ACCEPT Please ensure that the iptables rules you have added above come before any default DROP or REJECT rules.\n","excerpt":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all …","ref":"/docs/prerequisites/firewalls/","title":"Firewalls"},{"body":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, OpenShift or Rancher.\n Dive into our Use Case documentation for Workloads deployed in Kubernetes.\n ","excerpt":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, …","ref":"/docs/introduction/quickstart/","title":"Quickstart"},{"body":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and Password in base64 encoding.\nkubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;default\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z END This example contains a default password, for production installations, use a unique, strong password.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Create a cluster-config.yaml according to your needs from the examples below.\nkubectl create -f cluster-config.yaml Note that StorageOS will be deployed in spec.namespace (storageos by default), irrespective of what NameSpace the CR is defined in.\n Examples You can checkout all the parameters configurable in the configuration page.\n All examples must reference the storageos-api Secret.\nspec:secretRefName:\u0026#34;storageos-api\u0026#34;# Reference to the Secret created in the previous stepsecretRefNamespace:\u0026#34;storageos-operator\u0026#34;# Namespace of the SecretCheck out Cluster Definition examples for full CR files.\nInstalling with an external etcd spec:kvBackend:address:\u0026#39;10.43.93.95:2379\u0026#39;# IP of the SVC that exposes ETCD# address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can specify individual IPs of the etcd serversIf using Etcd with mTLS, you need to specify the secret that hold the certificates with the following parameters:\nspec:# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;storageos-etcd\u0026#34;# Namespace of the client certificates secretFollow the etcd operations page to setup the secret with the Etcd client certificate, client key and CA.\nInstalling to a subset of nodes In this case we select nodes that are workers. To make sure that StorageOS doesn\u0026rsquo;t start in Master nodes.\nYou can see the labels in the nodes by kubectl get node --show-labels.\nspec:nodeSelectorTerms:- matchExpressions:- key:\u0026#34;node-role.kubernetes.io/worker\u0026#34;operator:Invalues:- \u0026#34;true\u0026#34;# OpenShift uses \u0026#34;node-role.kubernetes.io/compute=true\u0026#34;# Rancher uses \u0026#34;node-role.kubernetes.io/worker=true\u0026#34;# Kops uses \u0026#34;node-role.kubernetes.io/node=\u0026#34; Different provisioners and Kubernetes distributions use node labels differently to specify master vs workers. Node Taints are not enough to make sure StorageOS doesn\u0026rsquo;t start in a node. The JOIN variable is defined by the operator by selecting all the nodes that match the nodeSelectorTerms.\n Enabling CSI spec:csi:enable:truedeploymentStrategy:deploymentenableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:trueenableControllerExpandCreds:trueThe credentials must be defined in the storageos-api Secret\napiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# Add base64 encoded creds below for CSI credentials.csiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9zSpecifying a shared directory for use with kubelet as a container spec:sharedDir:\u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39;Defining pod resource requests and reservations spec:resources:requests:memory:\u0026#34;512Mi\u0026#34;# cpu: \u0026#34;1\u0026#34;# limits:# memory: \u0026#34;4Gi\u0026#34;Limiting StorageOS can cause malfunction for IO to StorageOS volumes, therefore we do not currently recommend applying upper limits to resources for StorageOS pods.\nSpecifying custom Tolerations spec:tolerations:- key:\u0026#34;key1\u0026#34;operator:\u0026#34;Equal\u0026#34;value:\u0026#34;value1\u0026#34;effect:\u0026#34;EffectToTolerate\u0026#34;- key:\u0026#34;key2\u0026#34;operator:\u0026#34;Exists\u0026#34;Custom tolerations specified in the StorageOSCluster definition are added to all StorageOS components; the StorageOS daemonset, CSI helper and scheduler.\nIn the above example a toleration key1=value1:EffectToTolerate would be tolerated and key2 would be tolerated regardless of the value and effect. For more information about tolerations please see the Kubernetes documentation.\n","excerpt":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and …","ref":"/docs/reference/cluster-operator/examples/","title":"Cluster Operator examples"},{"body":" Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by default.\n StorageOS requires mount propagation enabled to present devices as volumes for containers (see linux kernel documentation here).\nCertain versions of docker ship with a systemd manifest with MountFlags set to \u0026lsquo;slave\u0026rsquo;, thus preventing StorageOS from working. This can be removed or set to \u0026lsquo;shared\u0026rsquo; with a systemd drop in:\nmkdir -p /etc/systemd/system/docker.service.d/ cat \u0026lt;\u0026lt;EOF \u0026gt; /etc/systemd/system/docker.service.d/mount_propagation_flags.conf [Service] MountFlags=shared EOF # systemctl daemon-reload # systemctl restart docker.service To confirm behaviour, the following command should run without error.\ndocker run -it --rm -v /mnt:/mnt:shared busybox sh -c /bin/date Orchestrators such as Kubernetes or OpenShift have their own ways of exposing this setting. Kubernetes 1.10 and OpenShift 3.10 have mount propagation enabled by default. Previous versions require that feature gates are enabled on the Kubernetes master\u0026rsquo;s controller-manager and apiserver services and in the kubelet service on each node.\nInstallations of orchestrators using Docker require that mount propagation is enabled for both.\nRefer to our installation pages for the orchestrators to see details on how to check and enable mount propagation where appropriate.\n","excerpt":"Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by …","ref":"/docs/prerequisites/mountpropagation/","title":"Mount Propagation"},{"body":"StorageOS is a certified Rancher application. We offer two installation methods:\n Rancher Catalogue - this is the easiest and requires just a few clicks Manual - allowing more control and visibility Before proceeding, ensure that you have followed our prerequisites. On Rancher, pay particular attention to the OS version and image used - some platforms require extra mainline kernel modules to be enabled.\n Catalog Manual Catalog Install StorageOS is a Certified application in the Rancher Catalog. You can install StorageOS using the Rancher application install.\nBefore completing the steps below, you will need an etcd cluster. For evaluations it is sufficient to use our simple test recipe. For production installations, please follow our production recipe. Make a note of the etcd endpoint URL in either case.\n Select the System project of your cluster\n Select the Apps tab and click Launch\n Search for StorageOS and click on the App\nThis will install the StorageOS operator, which manages the StorageOS DaemonSet.\n Check and ammend installation options\nA generic configuration for StorageOS is preset using the default values in the form. Be sure to check the etcd address and ensure it matches the value you noted at the beginning of this guide.\nThe catalog form exposes several useful parameters - documented below.\nFor further customization, you can opt to set the option to \u0026lsquo;Install StorageOS Cluster\u0026rsquo; to false and install a custom CR. See below for this.\n Launch the StorageOS cluster\n Verify the cluster bootstrap has successfully completed\n License the newly installed cluster\nNewly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nYou will need access to the StorageOS API on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 Now follow the instructions on our licensing operations page to obtain and apply a license.\nInstallation of StorageOS is now complete.\n Simple Customization - Modify Catalog Form The following options are exposed by the catalog form to allow some simple customization of the StorageOS installation.\n Cluster Operator namespace : The Kubernetes namespace where the StorageOS Cluster Operator and other resources will be created. Container Images : By default images are pulled from DockerHub, you can specify the image URLs when using private registries. Install StorageOS cluster : Controls the automatic deployment of StorageOS after installing the Cluster Operator. If set to false, the Operator will be created, but a Custom Resource will not be applied to the cluster. Launch the operator and proceed to the section Advanced Customization below. Namespace : The Kubernetes namespace where StorageOS will be installed. By default, StorageOS installs into the kube-system namespace, which will add a priority class to ensure high priority resource allocation. Installing StorageOS with the priority class prevents StorageOS from being evicted during periods of resource contention. It is inadvisable to modify this under normal circumstances. Username/Password : Default Username and Password for the admin account to be created at StorageOS bootstrap. A random password will be generated by leaving the field empty or clicking the Generate button. External etcd address(es) : Connection and configuration details for an external Etcd cluster.See our documentation here. Node Selectors and Tolerations : Control placement of StorageOS DaemonSet Pods. StorageOS will only be installed on the selected nodes. Tolerations : Define any tolerations you wish the DaemonSet to observe. Advanced Customization - Apply Custom CR If Install StorageOS Cluster was set to false, StorageOS will not be bootstrapped automatically. After the StorageOS Operator is installed, you can now create a Custom Resource that describes the StorageOS cluster.\n Select the System Workloads and Import YAML Create the Secret and CustomResource This is an example.\n--- apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z --- apiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container resources: requests: memory: \u0026#34;512Mi\u0026#34; nodeSelectorTerms: - matchExpressions: - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation operator: In values: - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Manual Installation The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.3.2\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"StorageOS is a certified Rancher application. We offer two installation methods:\n Rancher Catalogue …","ref":"/docs/install/rancher/","title":"Rancher"},{"body":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following yaml manifest.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n Upgrade StorageOS Operator using Helm If you have installed the StorageOS Operator using the Helm Chart, then you can upgrade the operator using the following commands.\n$ helm list NAME REVISION STATUS CHART APP VERSION NAMESPACE storageos-v1 4 DEPLOYED storageos-operator-0.2.11 1.3.0 storageos-operator $ helm repo update $ helm upgrade $NAME storageos/storageos-operator When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n ","excerpt":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following …","ref":"/docs/reference/cluster-operator/upgrade/","title":"Cluster Operator Upgrade"},{"body":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 active volumes per node 4.x kernels have a limitation of 4096 active volumes per node We are distribution agnostic as long as our prerequisites are met Orchestrators Kubernetes 1.15 to 1.20 OpenShift 4.0+ ","excerpt":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 …","ref":"/docs/introduction/platforms/","title":"Supported Platforms and Orchestrators"},{"body":"","excerpt":"","ref":"/docs/concepts/","title":"Concepts"},{"body":"StorageOS recommends that a PID cgroup limit of 32768 be used for StorageOS pods.\n Most environments fulfill this prerequisite by default. Check the StorageOS init container logs as shown below to ensure this is the case.\n StorageOS pods running in Kubernetes are part of a PID cgroup that may limit the maximum number of PIDs that all containers in the PID cgroup slice can spawn. As the Linux kernel assigns a PID to processes and Light Weight Processes (LWP) a low limit can be easily reached under certain circumstances. The PID limit can be set by the Kubernetes distribution or by the container runtime. Generally the limit is set to the machine wide default limit of 32768 but some environments can set this as low as 1024. A low PID limit may prevent StorageOS from spawning the required threads.\nThe StorageOS init container runs a script that checks for the PID limit of the PID cGroup slice that the StorageOS pod runs in. If the script finds that the limit is less than 32768 it will log a warning. This warning can be viewed using kubectl to check the init container logs.\n$ kubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init WARNING: Effective max.pids limit (1024) less than RECOMMENDED_MAX_PIDS_LIMIT (32768) Setting a Kubernetes PID limit Kubernetes defaults to an unlimited PodPidsLimit, which results in the usage of the machine wide limit; typically 32768.\nFor information on how to configure the Kubernetes PID limit see the Kubernetes documentation here.\nSetting a CRI-O PID limit Certain orchestrators or setups use CRI-O as the container runtime. Openshift 4.x currently has CRI-O set a PID limit of 1024 by default. To configure the default CRI-O limit in Openshift 4.x see the RedHat documentation here. To configure CRI-O more generally see the CRI-O documentation here.\n","excerpt":"StorageOS recommends that a PID cgroup limit of 32768 be used for StorageOS pods.\n Most environments …","ref":"/docs/prerequisites/pidlimits/","title":"PID Limits"},{"body":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a 64-bit architecture. Kubernetes 1.15 to 1.20 Container Runtime Engine: CRI-O, Containerd or Docker 1.10+ with mount propagation enabled. The necessary ports should be open. See the ports and firewall settings. Etcd cluster for StorageOS A mechanism for device presentation. Recommended: At least three nodes for replication and high availability. Kubernetes 1.15 to 1.20 Install the storageos CLI. If using Helm2, make sure the tiller ServiceAccount has enough privileges to create resources such as Namespaces, ClusterRoles, etc. For instance, following this installation procedure. System clocks synchronized using NTP or similar methods. While our distributed consensus algorithm does not require synchronised clocks, it does help to more easily correlate logs across multiple nodes. A PID cgroup limit of 32768. ","excerpt":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a …","ref":"/docs/prerequisites/","title":"Prerequisites"},{"body":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different distributions such as OpenShift, Rancher, EKS, AKS, GKE, etc. The user can provide standard PVC definitions and StorageOS will dynamically provision matching volumes. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nKubernetes and StorageOS communicate with each other to perform actions such as creation, deletion or mounting of volumes. The CSI (Container Storage Interface) driver is the standard method of communication. Using CSI, Kubernetes and StorageOS communicate over a Unix domain socket.\nCSI (Container Storage Interface) Note CSI is the standard method of communication that enables storage drivers for Kubernetes to release on their own schedule. The CSI standard allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI was released GA from Kubernetes 1.13. StorageOS v2 only supports the use of CSI as a storage driver. In addition, the StorageOS Cluster Operator handles the configuration of the CSI driver and its complexity by detecting the version of the Kubernetes installed.\nCheck out the status of the CSI release cycle in relation with Kubernetes in the CSI project page.\nCSI communication is fully supported by StorageOS if the cluster is deployed with any supported Linux Distribution.\nKubernetes Upgrades on Managed Services Managed services that support in place upgrades are fully supported. However, upgrading Kubernetes using green/blue deployments is not supported. This is because nodes are replaced rather than being upgraded. Any data stored on the nodes is lost when new nodes replace the previous ones.\nDocker Some managed Kubernetes platforms such as Azure AKS, enable the \u0026lsquo;Live-Restore\u0026rsquo; Docker feature, enabling containers to continue running while Docker is stopped or upgraded. This feature can cause nodes to hang while shutting-down or rebooting, as rather than going through an orderly shutdown, StorageOS (and other processes) are killed before the disks are synced and unmounted. Devices in this inaccessible state will log a warning similar to:\nTransport endpoint not connected To prevent this behaviour, we advise disabling this feature by setting\n{ ... \u0026quot;live-restore\u0026quot;: false } in /etc/docker/daemon.json.\nHere\u0026rsquo;s an example Ansible snippet that might be used to achieve this\n... - name: configure /etc/docker/daemon.json lineinfile: path: /etc/docker/daemon.json regexp: '^.*\u0026quot;live-restore\u0026quot;: true,$' line: ' \u0026quot;live-restore\u0026quot;: false,' backrefs: yes notify: restart docker ... Note: Use at own risk; you may need to adapt the example to work in your environment\n ","excerpt":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different …","ref":"/docs/install/","title":"Install"},{"body":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to a specific platform.\nStorageOS can be used in any Kubernetes distribution when the prerequisites are met. That includes Kubernetes manged services.\n","excerpt":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to …","ref":"/docs/platforms/","title":"Platforms"},{"body":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential cluster metadata and manage distributed configuration state. An embedded etcd instance is included in the StorageOS container, but for production environments and testing of production workloads, we recommend deploying an external etcd cluster. For more details about, and an example of, how to run etcd, see the External etcd Operations page.\nIt is highly recommended to use external etcd for cloud environments and place the etcd cluster on stable nodes. Placing the etcd on nodes that are recycled often might affect the normal operations of StorageOS.\nEtcd low latency IO It is recommended to run etcd on low-latency disks and keep other IO-intensive applications separate from the etcd nodes. Etcd is very sensitive to IO latency. Thus, the effect of disk contention can cause etcd downtime.\nBatch jobs such as backups, builds or application bundling can easily cause a high usage of disks making etcd unstable. It is recommended to run such workloads apart from the etcd servers.\nSetup of storage on the hosts We recommend creating a separate filesystem for StorageOS to mitigate the risk of filling the root filesystem on nodes. This has to be done for each node in the cluster.\nFollow the managing host storage best practices page for more details.\nResource reservations StorageOS resource consumption depends on the workloads and the StorageOS features in use.\nThe recommended minimum memory reservation for the StorageOS Pods is 512MB for non-production environments. However it is recommended to prepare nodes so StorageOS can operate with at least with 1-2GB of memory. StorageOS frees memory when possible.\nFor production environments, we recommend 4GB of Memory and 1 CPU as a minimum and to test StorageOS using realistic workloads and tune resources accordingly.\nStorageOS Pods resource allocation will impact directly on the availability of volumes in case of eviction or resource limit triggered restart. It is recommended to not limit StorageOS Pods.\nStorageOS implements a storage engine, therefore limiting CPU consumption might affect the I/O throughput of your volumes.\nSetting a Kubernetes PID limit StorageOS recommends that a PID cgroup limit of 32768 be set. StorageOS is a multi-threaded application and while most Kubernetes distributions set the PID cgroup limit to 32768, some environments can set a limit as low as 1024. The StorageOS init container will print a log message warning if the PID cgroup limit is too low. See our prerequisites for more information.\nMaintain a sufficient number of nodes for replicas to be created To ensure that a new replica can always be created, an additional node should be available. To guarantee high availability, clusters using Volumes with 1 replica must have at least 3 storage nodes. When using Volumes with 2 replicas, at least 4 storage nodes, 3 replicas, 5 nodes, etc.\nMinimum number of storage nodes = 1 (primary) + N (replicas) + 1\nFor more information, see the section on replication.\nStorageOS API username/password The API grants full access to StorageOS functionality, therefore we recommend that the default administrative password of \u0026lsquo;storageos\u0026rsquo; is reset to something unique and strong.\nYou can change the default parameters by encoding the apiUsername and apiPassword values (in base64) into the storageos-api secret.\nTo generate a unique password, a technique such as the following, which generates a pseudo-random 24 character string, may be used:\n# Generate strong password PASSWORD=$(cat -e /dev/urandom | tr -dc \u0026#39;a-zA-Z0-9-!@#$%^\u0026amp;*()_+~\u0026#39; | fold -w 24 | head -n 1) # Convert password to base64 representation for embedding in a K8S secret BASE64PASSWORD=$(echo -n $PASSWORD | base64) Note that the Kubernetes secret containing a strong password must be created before bootstrapping the cluster. Multiple installation procedures use this Secret to create a StorageOS account when the cluster first starts.\nStorageOS Pod placement StorageOS must run on all nodes that will contribute storage capacity to the cluster or that will host Pods which use StorageOS volumes. For production environments, it is recommended to avoid placing StorageOS Pods on Master nodes.\nStorageOS is deployed with a DaemonSet controller, and therefore tolerates the standard unschedulable (:NoSchedule) action. If that is the only taint placed on master or cordoned nodes StorageOS pods might start on them (see the Kubernetes docs for more details). To avoid scheduling StorageOS pods on master nodes, you can add an arbitrary taint to them for which the StorageOS DaemonSet won\u0026rsquo;t have a toleration.\nDedicated instance groups Cloud environments give users the ability to quickly scale the number of nodes in a cluster in response to their needs. Because of the ephemeral nature of the cloud, StorageOS recommends setting conservative downscaling policies.\nFor production clusters, it recommended to use dedicated instance groups for Stateful applications that allow the user to set different scaling policies and define StorageOS pools based on node selectors to collocate volumes.\nLosing a few nodes at the same time could cause the loss of data even when volume replicas are being used.\nPort blocking StorageOS exposes ports to operate. It is recommended that the ports are not accessible from outside the scope of your cluster.\nStorageOS in Docker EE StorageOS does not support running on Swarm nodes nor on mixed (Kubernetes and Swarm) nodes. StorageOS volumes have to be provisioned and used from Kubernetes nodes.\n","excerpt":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential …","ref":"/docs/best-practices/","title":"StorageOS Best Practices"},{"body":"","excerpt":"","ref":"/docs/operations/","title":"Operations"},{"body":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or other Orchestrators that are derived from Kubernetes such as OpenShift or Rancher. This is useful for running stateful applications, such as databases or CI/CD applications, under the control of Kubernetes as Kubernetes can make scheduling decisions without the application data being lost.\nWhat we have outlined in the cookbooks below are some quick deployments of stateful applications into a Kubernetes cluster. These examples are not production ready but have been provided to give you some insight into how to use StorageOS with stateful applications.\nStatefulSets The examples we have provided use StatefulSets as a way to deploy applications into Kubernetes. The reason for this is that the StatefulSet controller is designed to manage stateful applications and it \u0026ldquo;provides guarantees about the ordering and uniqueness\u0026rdquo; of sets of pods.\nPractically this means that when a StatefulSet scales, pods are created in order from 0-(N-1), if a StatefulSet scales down then pods are deleted in reverse order from (N-1)-0.\nSecondly, it means that the behaviour of the StatefulSet controller differs from that of Deployment and ReplicaSet controllers. For Deployment and ReplicaSet controllers \u0026ldquo;\u0026hellip; at many points in the lifetime of a replica set there will be 2 copies of a pod\u0026rsquo;s processes running concurrently\u0026rdquo;. Having two different pods mount a volume at the same time can cause corruption of data. Currently Kubernetes accessModes only apply restrictions to nodes mounting volumes rather than pods, so it is important that StatefulSets are used with StorageOS volumes so the necessary pod uniqueness guarantees are maintained.\nStatefulSet Manifests The StorageOS specific part of the Kubernetes manifests for these examples lies in the VolumeClaimTemplate that\u0026rsquo;s part of the statefulset definition.\nVolumeClaimTemplate\napiVersion:apps/v1kind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1template:metadata:labels:app:mssqlenv:prodspec:serviceAccountName:mssqlcontainers:- name:fooimage:barvolumeMounts:- name:bazmountPath:/var/opt/barenvFrom:- configMapRef:name:mssqlvolumeClaimTemplates:- metadata:name:bazlabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiIn the StatefulSet definition above the container has a volume mount defined called baz. The definition for this volume is found in the VolumeClaimTemplate where the fast storageClass will be used to dynamically provision storage if the persistent volume does not already exist.\n","excerpt":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or …","ref":"/docs/usecases/","title":"Use Cases: Install applications on Kubernetes with StorageOS"},{"body":"","excerpt":"","ref":"/docs/reference/bundles/","title":"Bundles"},{"body":"","excerpt":"","ref":"/docs/reference/","title":"Reference"},{"body":"When you need support, please raise a ticket via our Help Desk portal. It is important to select the priority of your ticket in accordance with the severity:\n Severity 1 - Urgent Severity 2 - High Severity 3 - Medium Severity 4 - Low This helps us route and prioritise the ticket accordingly. You can find definitions of our support severity levels on our website.\nResponses to tickets will be cc\u0026rsquo;d via email.\nFor community support and general enquiries, please join our public Slack channel. \nInformation to include in tickets To help us provide effective support, we request that you provide as much information as possible when contacting us. The list below is a suggested starting point. Additionally, please include anything specific, such as log entries, that may help us debug your issue.\nPlatform Cloud provider/Bare metal OS distribution and version Kernel version docker version and installation procedure (distro packages or docker install) StorageOS Version of StorageOS storageos get nodes storageos get volumes storageos describe volume VOL_ID # in case of issues with a specific volume Orchestrator related (Kubernetes, OpenShift, etc) Version and installation method Managed or self managed? kubectl -n kube-system get pod kubectl -n kube-system logs -lapp=storageos -c storageos kubectl -n kube-system get storageclass Specific for your namespaces: kubectl describe pvc PVC_NAME Specific for your namespaces: kubectl describe pod POD_NAME Environment Changes Details of any recent changes to your environment such as planned maintenance, node reboots, network failures, etcd outage, etc.. This can help speed up ticket triage and resolution considerably StorageOS Support Bundle StorageOS provides the ability to generate a support bundle that aggregates cluster information. See Support Bundle for a list of what is included.\nStorageOS engineers might ask for a support bundle to be generated during support cases.\nThe information in the bundle is used only for support purposes, and will be removed once it is no longer needed. If the information is sensitive and can\u0026rsquo;t be given to StorageOS, please make sure that the support engineers have as much information about your environment as possible.\nRefer to the Support Bundle documentation page for details of how to generate a bundle.\n","excerpt":"When you need support, please raise a ticket via our Help Desk portal. It is important to select the …","ref":"/docs/support/","title":"Support"},{"body":"Our self-evaluation guide is a step by step recipe for installing and testing StorageOS. This guide is divided into three sections:\n Installation - install StorageOS with a single command Feature Testing - short walkthrough of some of our features Benchmarking - a recipe to benchmark StorageOS on your infrastructure For more comprehensive documentation including installation advice for complex setups, operational guides, and use-cases, please see our main documentation site.\nSupport for Self Evaluations Should you have questions or require support, there are several ways to get in touch with us. The fastest way to get in touch is to join our public Slack channel. You can also get in touch via email to info@storageos.com.\nFurthermore you can fill out the form below and we will get in touch.\n hbspt.forms.create({ portalId: \"3402546\", formId: \"a07fecd3-ce5b-4835-b136-51a94a35632b\", sfdcCampaignId: \"70158000000BAZzAAO\" }); Installation In this document we detail a simple installation suitable for evaluation purposes. The etcd we install uses a 3 node cluster with ephemeral storage, and as such is not suitable for production workloads. However, for evaluation purposes it should be sufficient. For production deployments, please see our main documentation pages.\nA standard StorageOS installation uses the StorageOS operator, which performs most platform-specific configuration for you. The StorageOS operator has been certified by Red Hat and is open source.\nThe basic installation steps are:\n Check prerequisites Install an etcd cluster Install the StorageOS Operator Install StorageOS by applying a CR for the Operator to action Prerequisites While we do not require custom kernel modules or additional userspace tooling, StorageOS does have a few basic prerequisites that are met by default by most modern distributions:\n At least 1 CPU core, 2GB RAM free TCP ports 5701-5710 and TCP \u0026amp; UDP 5711 open between all nodes in the cluster A 64bit supported operating system - StorageOS can run without additional packages in Debian 9, RancherOS, RHEL7.5,8 and CentOS7,8 and need the package linux-image-extra for Ubuntu. Mainline kernel modules target_core_mod, tcp_loop, target_core_file, target_core_user, configfs, and ui. These are present by default on most modern linux distributions, and can be installed with standard package managers. See our system configuration page for instructions. Quick Installation For those feeling brave and/or lucky, we have encapsulated the installation instructions in this guide into a short bash script. As with similar instructions elsewhere on the internet, caution is advised here. The script is liberally commented and we recommend that you inspect it in your text editor of choice before running.\n Run the following command where kubectl is installed and with the context set for your Kubernetes cluster\n curl -sL https://storageos.run | bash Note that this script deploys a cluster with etcd on transient storage, suitable for evaluation purposes only. Do not use this cluster for production workloads.\nYour new cluster can be used straight away. You can now jump to the Provision a Volume section below to provision your first volume and license the cluster.\nInstalling StorageOS Step By Step This section describes the installation steps in some more detail.\nInstall etcd Here we install a 3 node etcd cluster using the CoreOS operator to install on ephemeral storage within your Kubernetes cluster. This is suitable for evaluation purposes. For production deployment instructions, please see our production etcd page.\n Download repo\n$ git clone https://github.com/coreos/etcd-operator.git Configure NS, Role and RoleBinding\n$ export ROLE_NAME=etcd-operator $ export ROLE_BINDING_NAME=etcd-operator $ export NAMESPACE=etcd Create Namespace\n$ kubectl create namespace $NAMESPACE Deploy Operator\n$ ./etcd-operator/example/rbac/create_role.sh $ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END Create the EtcdCluster resource\n$ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;YAML apiVersion: \u0026#34;etcd.database.coreos.com/v1beta2\u0026#34; kind: \u0026#34;EtcdCluster\u0026#34; metadata: name: \u0026#34;storageos-etcd\u0026#34; spec: size: 3 # Set to 1 if insufficient nodes version: \u0026#34;3.4.7\u0026#34; pod: etcdEnv: - name: ETCD_QUOTA_BACKEND_BYTES value: \u0026#34;2147483648\u0026#34; # 2 GB - name: ETCD_AUTO_COMPACTION_RETENTION value: \u0026#34;1000\u0026#34; # Keep 1000 revisions - name: ETCD_AUTO_COMPACTION_MODE value: \u0026#34;revision\u0026#34; # Set the revision mode resources: requests: cpu: 200m memory: 300Mi securityContext: runAsNonRoot: true runAsUser: 9000 fsGroup: 9000 tolerations: - operator: \u0026#34;Exists\u0026#34; # add tolerations to match all taints on the master nodes where etcd will be deployed affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster operator: In values: - storageos-etcd topologyKey: kubernetes.io/hostname YAML This creates a 3 node etcd cluster and associated service in Kubernetes. We can use the service endpoint when creating the StorageOS cluster.\nInstall StorageOS Operator In order to install the StorageOS operator download the requisite yaml manifests or apply them with kubectl.\n$ kubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml You can verify the operator is running using the following command\n$ kubectl get pods -n storageos-operator Install StorageOS Once the StorageOS operator has been installed a StorageOS cluster can be generated by creating a StorageOSCluster resource.\nA StorageOSCluster resource describes the state of the StorageOS cluster that is desired and the StorageOS operator will create the desired StorageOS cluster. For examples of StorageOSCluster resources please see our examples page here. For a full list of the configurable spec parameters of the StorageOSCluster resource please see here.\n Create a secret defining the API username and password\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9zEND Create a StorageOSCluster resource\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:example-storageosclusternamespace:\u0026#34;storageos-operator\u0026#34;spec:# StorageOS Pods are in kube-system by defaultsecretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;storageos-operator\u0026#34;k8sDistro:\u0026#34;upstream\u0026#34;# Set the Kubernetes distribution for your cluster (upstream, eks, aks, gke, rancher, dockeree)# storageClassName: fast # The storage class creates by the StorageOS operator is configurablekvBackend:address:\u0026#34;storageos-etcd-client.etcd.svc.cluster.local:2379\u0026#34;END Confirm that the cluster has been created and that StorageOS pods are running\n$ kubectl -n kube-system get pods StorageOS pods enter a ready state after a minimum of 45s has passed.\n Deploy the StorageOS CLI as a container\n$ kubectl -n kube-system run \\ --image storageos/cli:v2.3.0 \\ --restart=Never \\ --env STORAGEOS_ENDPOINTS=storageos:5705 \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sh -c \u0026#34;while true; do sleep 999999; done\u0026#34; Provision a StorageOS Volume Now that we have a working StorageOS cluster, we can provision a volume to test everything is working as expected.\n Create a PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Verify that the volume was created with the CLI\n pvc-1 should be listed in the CLI output\n $ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that consumes the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Check that the pod starts successfully. If the pod starts successfully then the StorageOS cluster is working correctly\n$ kubectl get pod d1 -w The pod mounts a StorageOS volume under /mnt so any files written there will persist beyond the lifetime of the pod. This can be demonstrated using the following commands.\n Execute a shell inside the pod and write some data to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo Hello World! \u0026gt; /mnt/hello root@d1:/# cat /mnt/hello Hello World! should be printed out.\n Delete the pod\n$ kubectl delete pod d1 Recreate the pod\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Open a shell inside the pod and check the contents of /mnt/hello\n$ kubectl exec -it d1 -- cat /mnt/hello Hello World! should be printed out.\n License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nYou will need access to the StorageOS API on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 Now follow the instructions on our operations licensing page to obtain and apply a license.\nInstallation of StorageOS is now complete.\nStorageOS Features Now that you have a fully functional StorageOS cluster we will explain some of our features that may be of use to you as you complete application and synthetic benchmarks.\nStorageOS features are all enabled/disabled by applying labels to volumes. These labels can be passed to StorageOS via persistent volume claims (PVCs) or can be applied to volumes using the StorageOS CLI or GUI.\nThe following is not an exhaustive feature list but outlines features which are commonly of use during a self-evaluation.\nVolume Replication StorageOS enables synchronous replication of volumes using the storageos.com/replicas label.\nThe volume that is active is referred to as the master volume. The master volume and its replicas are always placed on separate nodes. In fact if a replica cannot be placed on a node without a replica of the same volume, the volume will fail to be created. For example, in a three node StorageOS cluster a volume with 3 replicas cannot be created as the third replica cannot be placed on a node that doesn\u0026rsquo;t already contain a replica of the same volume.\nSee our replication documentation for more information on volume replication.\n To test volume replication create the following PersistentVolumeClaim\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-replicated labels: storageos.com/replicas: \u0026#34;1\u0026#34; spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Note that volume replication is enabled by setting the storageos.com/replicas label on the volume.\n Confirm that a replicated volume has been created by using the StorageOS CLI or UI\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that uses the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: replicated-pod spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-replicated END Write data to the volume\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# echo Hello World! \u0026gt; /mnt/hello root@replicated-pod:/# cat /mnt/hello Hello World! should be printed out.\n Find the location of the master volume and shutdown the node\nShutting down a node causes all volumes, with online replicas, on the node to be evicted. For replicated volumes this immediately promotes a replica to become the new master.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-replicated Bound pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 5Gi RWO fast 1m $ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker (online) kind-worker2 1/1 26 seconds ago Check the location of the master volume and notice that it is on a new node. If the pod that mounted the volume was located on the same node that was shutdown then the pod will need to be recreated.\n$ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker2 (online) kind-worker2 1/1 46 seconds ago Check that the data is still accessible to the pod\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# cat /mnt/hello Hello World! should be printed out.\n Benchmarking Benchmarking storage is a complex topic. Considering the many books and papers that have been written about benchmarking, we could write many paragraphs here and only begin to scratch the surface.\nTaking this complexity into account we present recipes for both synthetic benchmarks using FIO, and a sample application benchmark to test PostgreSQL using pgbench. The approaches are complementary - synthetic benchmarks allow us to strictly control the parameters of the IO we put through the system in order to stress various aspects of it. Application benchmarks allow us to get a sense of the performance of the system when running an actual representative workload - which of course is the ultimate arbiter of performance.\nDespite the inherent complexity of benchmarking storage there are a few general considerations to keep in mind.\nConsiderations Local vs. Remote Volumes When a workload is placed on the same node as a volume, there is no network round trip for IO, and performance is consequently improved. When considering the performance of StorageOS it is important to evaluate both local and remote volumes; since for certain workloads we want the high performance of a local attachment, but we also desire the flexibility of knowing that our less performance-sensitive workloads can run from any node in the cluster.\nThe Effect of Synchronous Replication Synchronous replication does not impact the read performance of a volume, but it can have a significant impact on the write performance of the volume, since all writes must be propagated to replicas before being acked to the application. The impact varies in proportion to the inter-node latency. For an inter-node latency of 1ms, we have a max ceiling of 1000 write IOPS, and in reality a little less than that to allow for processing time on the nodes. This is less concerning then it may first appear, since many applications will issue multiple writes in parallel (known as increasing the queue depth).\nSynthetic Benchmarks Prerequisites Kubernetes cluster with a minimum of 3 nodes and a minimum of 30 Gb available capacity StorageOS CLI running as a pod in the cluster Synthetic benchmarks using tools such as FIO are a useful way to begin measuring StorageOS performance. While not fully representative of application performance, they allow us to reason about the performance of storage devices without the added complexity of simulating real world workloads, and provide results easily comparable across platforms.\nFIO has a number of parameters that can be adjusted to simulate a variety of workloads and configurations. Parameters that are particularly important are:\n Block Size - the size of the IO units performed. Queue Depth - the amount of concurrent IOs in flight IO Pattern - access can be random across the disk, or sequentially. IO subsystems typically perform better with sequential IO, because of the effect of read ahead caches, and other factors For this self-evaluation we will run a set of tests based on the excellent DBench, which distills the numerous FIO options into a series of well-crafted scenarios:\n Random Read/Write IOPS and BW - these tests measure the IOPS ceiling (with a 4k block size) and bandwidth ceiling (with a 128K block size) of the volume using a random IO pattern and high queue depth Average Latency - these tests measure the IO latency of the volume under favourable conditions using a random access pattern, low queue depth and low block size Sequential Read/Write - these tests measure the sequential read/write throughput of the volume with a high queue depth and high block size Mixed Random Read/Write IOPS - these tests measure the performance of the volume under a 60/40 read/write workload using a random access pattern and low blocksize For convenience we present a single script to run the scenarios using local and remote volumes both with and without a replica. Deploy the FIO tests for the four scenarios using the following command:\ncurl -sL https://docs.storageos.com/sh/deploy-synthetic-benchmarks.sh | bash The script will take approximately 20 minutes to complete, and will print the results to STDOUT.\nThe exact results observed will depend on the particular platform and environment, but the following trends should be observable:\n local volumes perform faster than remote volumes read performance is independent of the number of replicas write performance is dependent on the number of replicas Application Benchmarks While synthetic benchmarks are useful for examining the behaviour of StorageOS with very specific workloads, in order to get a realistic picture of StorageOS performance actual applications should be tested.\nMany applications come with test suites which provide standard workloads. For best results, test using your application of choice with a representative configuration and real world data.\nAs an example of benchmarking an application the following steps lay out how to benchmark a Postgres database backed by a StorageOS volume.\n Start by cloning the StorageOS use cases repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/pgbench.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the Postgres examples folder\n$ cd storageos-usecases/pgbench Decide which node you want the pgbench pod and volume to be located on. The node needs to be labelled app=postgres\n$ kubectl label node \u0026lt;NODE\u0026gt; app=postgres Then set the storageos.com/hint.master label in 20-postgres-statefulset.yaml file to match the StorageOS nodeID for the node you have chosen before creating all the files. The StorageOS nodeID can be obtained using the cli and doing a describe node\n$ kubectl create -f . Confirm that Postgres is up and running\n$ kubectl get pods -w -l app=postgres Use the StorageOS CLI or the GUI to check the master volume location and the mount location. They should match\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Exec into the pgbench container and run pgbench\n$ kubectl exec -it pgbench -- bash -c \u0026#39;/opt/cpm/bin/start.sh\u0026#39; Conclusion After completing these steps you will have benchmark scores for StorageOS. Please keep in mind that benchmarks are only part of the story and that there is no replacement for testing actual production or production like workloads.\nStorageOS invites you to provide feedback on your self-evaluation to the slack channel or by directly emailing us at info@storageos.com]\n","excerpt":"Our self-evaluation guide is a step by step recipe for installing and testing StorageOS. This guide …","ref":"/docs/self-eval/","title":"Self Evaluation Guide"},{"body":"","excerpt":"","ref":"/index.json","title":""},{"body":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: '#ohpen_swagger_ui', presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ] }) window.ui = ui } \t","excerpt":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: …","ref":"/docs/reference/api/","title":"API Reference"},{"body":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply licence --from-file \u0026lt;path-to-licence-file\u0026gt; $ echo \u0026#34;\u0026lt;licence file contents\u0026gt;\u0026#34; | storageos apply licence --from-stdin Flags: --from-file string reads a StorageOS product licence from a specified file path --from-stdin reads a StorageOS product licence from the standard input -h, --help help for licence Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply …","ref":"/docs/reference/cli/apply/","title":"Apply licence"},{"body":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes.\nFundamentally, StorageOS uses the storage attached to the nodes in the StorageOS cluster to create and present virtual volumes into containers. Space on the host is consumed from the mount point /var/lib/storageos/data, so it is therefore recommended that disk devices are used exclusively for StorageOS, as described in Managing Host Storage \nStorageOS is agnostic to the underlying storage and runs equally well on bare metal, in virtual machines or on cloud providers.\nRead about the cloud native storage principles behind StorageOS.\nStorageOS on Kubernetes StorageOS is architected as a series of containers that fulfil separate, discrete functions.\n StorageOS Controlplane\nResponsible for monitoring and maintaining the state of volumes and nodes in the cluster\n StorageOS Dataplane\nResponsible for all I/O path related tasks; reading, writing, compression and caching\n StorageOS Scheduler\nResponsible for scheduling applications on the same node as applications volume\n CSI helper\nResponsible for registering StorageOS with Kubernetes as a CSI driver\n StorageOS Operator\nResponsible for the creation and maintenance of the StorageOS cluster\n StorageOS is deployed by the StorageOS operator. In Kubernetes, the StorageOS Controlplane and Dataplane are deployed in a single pod managed by a daemonset. This daemonset runs on every node in the cluster that will consume or present storage. The scheduler, CSI helper and Operator run as separate pods and are controlled as deployments.\nStorageOS is designed to feel familiar to Kubernetes and Docker users. Storage is managed through standard StorageClasses and PersistentVolumeClaims, and features are controlled by Kubernetes-style labels and selectors, prefixed with storageos.com/. By default, volumes are cached to improve read performance and compressed to reduce network traffic.\nAny pod may mount a StorageOS virtual volume from any node that is also running StorageOS, regardless of whether the pod and volume are collocated on the same node. Therefore, applications may be started or restarted on any node and access volumes transparently.\n","excerpt":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes. …","ref":"/docs/concepts/architecture/","title":"Architecture"},{"body":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos attach --namespace my-namespace-name my-volume my-node Flags: -h, --help help for attach Global Flags: --cache-dir string set the directory used by the StorageOS CLI to cache data that can be used for future commands (default \u0026#34;/root/.cache/storageos\u0026#34;) -c, --config string specifies the config file path (default \u0026#34;/root/.config/storageos/config.yaml\u0026#34;) --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) --no-auth-cache disable the CLI\u0026#39;s caching of authentication sessions -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 15s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos …","ref":"/docs/reference/cli/attach/","title":"Attach"},{"body":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our Kubernetes installation instructions page.\nAn EKS deployment of Kubernetes uses AWS Linux by default with an optimized kernel. As the requisite kernel modules are not available for StorageOS to use TCMU, FUSE will be used as a fallback. Using FUSE instead of TCMU has performance implications.\nFor more details about the OS Distributions check the System Configuration page.\n","excerpt":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our …","ref":"/docs/platforms/aws-eks/","title":"AWS EKS"},{"body":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our Kubernetes installation instructions page.\nAn AKS deployment of Kubernetes uses Ubuntu by default with an optimized kernel. All versions of Ubuntu with a kernel version later than4.15.0-1029-azure meet the StorageOS prerequisites.\n","excerpt":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our …","ref":"/docs/platforms/azure-aks/","title":"Azure AKS"},{"body":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing StorageOS persistent volumes with Cassandra means that if a Cassandra pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up the pod data is immediately available. Should Kubernetes schedule the Cassandra pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not a StorageOS master is located on the same node.\nAs Cassandra has features to allow it to handle replication careful consideration of whether to allow StorageOS or Cassandra to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Cassandra on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiVersion:apps/v1kind:StatefulSetmetadata:name:cassandraspec:selector:matchLabels:app:cassandraserviceName:cassandrareplicas:3...spec:...volumeMounts:- name:cassandra-datamountPath:/var/lib/cassandra...volumeClaimTemplates:- metadata:name:cassandra-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Move into the Cassandra examples folder and create the objects\ncd storageos-usecases kubectl create -f ./cassandra Confirm Cassandra is up and running.\n$ kubectl get pods -w -l app=cassandra NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 8m32s cassandra-1 1/1 Running 0 7m51s cassandra-2 1/1 Running 0 6m36s Connect to the Cassandra client pod and connect to the Cassandra server through the service\n$ kubectl exec -it cassandra-0 -- cqlsh cassandra-0.cassandra Connected to K8Demo at cassandra-0.cassandra:9042. [cqlsh 5.0.1 | Cassandra 3.11.3 | CQL spec 3.4.4 | Native protocol v4] Use HELP for help. cqlsh\u0026gt; SELECT cluster_name, listen_address FROM system.local; cluster_name | listen_address --------------+---------------- K8Demo | 100.96.7.124 (1 rows) ","excerpt":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing …","ref":"/docs/usecases/cassandra/","title":""},{"body":"The storageos command line interface (CLI) is used to manage cluster-wide configuration.\nInstallation # linux/amd64 curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.3.0/storageos_linux_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ # MacOS curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.3.0/storageos_darwin_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ You will need to provide the correct credentials to connect to the API. The default installation uses the storageos-api Secret to generate the first admin user. By default, it creates a single user with username storageos and password storageos:\nexport STORAGEOS_USERNAME=storageos export STORAGEOS_PASSWORD=storageos export STORAGEOS_ENDPOINTS=10.1.5.249:5705 Run CLI as a container You can run the cli as a container in your Kubernetes cluster. Then exec into it to run commands.\nkubectl -n kube-system run \\ --image storageos/cli:v2.3.0 \\ --restart=Never \\ --env STORAGEOS_ENDPOINTS=storageos:5705 \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sh -c \u0026#34;while true; do sleep 100000; done\u0026#34; Usage $ storageos Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Formatting CLI Output StorageOS CLI output can be formatted using the --output option. The strings that are passed to --output are \u0026lsquo;json\u0026rsquo;, \u0026lsquo;yaml\u0026rsquo; or \u0026lsquo;text\u0026rsquo;.\nCheatsheet Command Subcommand Description apply licence Apply licence to cluster. attach Attach Volume to a node. create user, volume Create resources such as users or volumes. delete volume Delete resources. describe node, volume Show detailed view of resources. detach volume Detach volume from a node. get cluster, diagnostics, namespace, node, user, volume List resources. help Help version Show CLI version. Source is available on Github.\n","excerpt":"The storageos command line interface (CLI) is used to manage cluster-wide configuration. …","ref":"/docs/reference/cli/","title":"Command line interface"},{"body":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator acts as a Kubernetes controller that watches the StorageOSCluster CR (Custom Resource). Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\nYou can find the source code in the cluster-operator repository.\nInstall the operator following orchestrator specific procedure.\nTo deploy a StorageOS cluster you will need to fulfil the following steps.\n Install Etcd Install operator Create storageos-api Secret Create a CR (Custom Resource) to deploy StorageOS ","excerpt":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS …","ref":"/docs/reference/cluster-operator/","title":"Cluster Operator"},{"body":"Every StorageOS cluster has a unique ClusterID generated at bootstrap. A StorageOS Licence is specific for a ClusterID.\nHow to obtain the ClusterID You can obtain the ClusterID using either the CLI or GUI.\nGUI You will need access to the StorageOS GUI on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 As an alternative, an Ingress controller may be preferred.\nOnce you have obtained access to the GUI, login using whatever credentials you used to create the cluster and go to the \u0026ldquo;Licence\u0026rdquo; section on the left navigation menu.\nCLI This CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence.\n","excerpt":"Every StorageOS cluster has a unique ClusterID generated at bootstrap. A StorageOS Licence is …","ref":"/docs/operations/cluster-id/","title":"Find ClusterID"},{"body":"StorageOS clusters represent groups of nodes which run a common distributed control plane.\nTypically, a StorageOS cluster maps one-to-one to a Kubernetes (or similar orchestrator) cluster, and we expect our daemonset to run on all worker nodes within the cluster that will consume or present storage.\nClusters use etcd to maintain state and manage distributed consensus between nodes.\n","excerpt":"StorageOS clusters represent groups of nodes which run a common distributed control plane. …","ref":"/docs/concepts/clusters/","title":"Clusters"},{"body":"StorageOS compression is handled on a per volume basis and is disabled by default in v2.2+, as performance is generally increased when compression is disabled due to block alignment. This means that there is a trade off between volume performance and the space the volume occupies on the backend device.\nCompression can be enabled by setting the label storageos.com/nocompress=false on a volume at volume creation time.\nStorageOS utilises the lz4 compression algorithm when writing to the backend store and when compressing replication traffic before it is sent across the network.\nStorageOS detects whether a block can be compressed or not by creating a heuristic that predicts the size of a compressed block. If the heuristic indicates that the compressed block is likely to be larger than the original block then the uncompressed block is stored. Block size increases post compression if the compression dictionary is added to a block that cannot be compressed. By verifying whether blocks can be compressed, disk efficiency is increased and CPU resources are not wasted on attempts to compress uncompressible blocks. StorageOS\u0026rsquo; patented on disk format is used to tell whether individual blocks are compressed without overhead. As such volume compression can be dynamically enabled/disabled even while a volume is in use.\nWhen compression and encryption are both enabled for a volume, blocks are compressed then encrypted.\n","excerpt":"StorageOS compression is handled on a per volume basis and is disabled by default in v2.2+, as …","ref":"/docs/concepts/compression/","title":"Compression"},{"body":"We are always looking to improve our documentation. If you like to help people and can write, read on for the process for submitting your contributions. If your guide is published, you\u0026rsquo;ll receive $250 per article by PayPal.\nContent guidelines A guide contains step by step instructions for how to accomplish a specific task using StorageOS. To be accepted, guides must be:\n Written in English. Relevant, accurate and complete. Technically correct and thoroughly tested. Follow the writing style guide. Guides should avoid:\n Duplicating an existing guide or other sources, such as blogs or forum posts. Including irrelevant material. Submission and review You should submit your guide as a pull request to the GitHub repo.\nYour submission will be left open for community review for two weeks. After this, your submission will be reviewed internally for about another week.\nIf accepted, your pull request will be approved and you will have 36 hours to send your submission title and PayPal account information. Non-response will be taken as a go-ahead to publish.\nLegal COPYRIGHT OWNERSHIP. The StorageOS Guides \u0026amp; Tutorials repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license.\nCREDIT. Nothing contained in this Agreement shall be deeded to require StorageOS to use the Work, or any part thereof, in connection with StorageOS Guides \u0026amp; Tutorials or otherwise. Credit for the Work shall read, “Contributed by writer’s name.”\nPAYMENT. Upon publication of a submission to the StorageOS Guides \u0026amp; Tutorials Repository, the writer will be paid the sum of USD $250.00 as an electronic payment.\n","excerpt":"We are always looking to improve our documentation. If you like to help people and can write, read …","ref":"/docs/reference/contributing/","title":"Contributing to the docs"},{"body":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: user Create a new user account volume Provision a new volume Flags: -h, --help help for create Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos create [command] --help\u0026#34; for more information about a command. create volume $ storageos create volume --help Provision a new volume Usage: storageos create volume [flags] Examples: $ storageos create volume --description \u0026#34;This volume contains the data for my app\u0026#34; --fs-type \u0026#34;ext4\u0026#34; --labels env=prod,rack=db-1 --size 10GiB --namespace my-namespace-name my-app $ storageos create volume --replicas 1 --namespace my-namespace-name my-replicated-app Flags: --async perform the operation asynchronously, using the configured timeout duration --cache caches volume data (default true) --compress compress data stored by the volume at rest and during transit (default true) -d, --description string a human-friendly description to give the volume -f, --fs-type string the filesystem to format the new volume with once provisioned (default \u0026#34;ext4\u0026#34;) -h, --help help for volume -l, --labels strings an optional set of labels to assign to the new volume, provided as a comma-separated list of key=value pairs -r, --replicas uint the number of replicated copies of the volume to maintain -s, --size string the capacity to provision the volume with (default \u0026#34;5GiB\u0026#34;) --throttle deprioritises the volume\u0026#39;s traffic by reducing the rate of disk I/O Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) create user $ storageos create user --help Create a new user account Usage: storageos create user [flags] Examples: $ storageos create user --with-username=alice --with-admin=true Flags: -h, --help help for user --with-admin control whether the user is given administrative priviledges --with-groups stringArray the list of policy groups to assign to the user --with-password string the password to assign to the user. If not specified, this will be prompted for. --with-username string the username to assign Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: …","ref":"/docs/reference/cli/create/","title":"Create"},{"body":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available Commands: volume Delete a volume Flags: -h, --help help for delete Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos delete [command] --help\u0026#34; for more information about a command. delete volume $ storageos delete volume --help Delete a volume Usage: storageos delete volume [volume name] [flags] Examples: $ storageos delete volume my-test-volume my-unneeded-volume $ storagoes delete volume --namespace my-namespace my-old-volume Flags: --async perform the operation asynchronously, using the configured timeout duration --cas string make changes to a resource conditional upon matching a provided version -h, --help help for volume Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available …","ref":"/docs/reference/cli/delete/","title":"Delete"},{"body":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] Available Commands: node Retrieve detailed information for nodes in the cluster volume Show detailed information for volumes Flags: -h, --help help for describe Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos describe [command] --help\u0026#34; for more information about a command. describe node $ storageos describe node --help Retrieve detailed information for nodes in the cluster Usage: storageos describe node [node names...] [flags] Aliases: node, nodes Examples: $ storageos describe node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) describe volume $ storageos describe volume --help Show detailed information for volumes Usage: storageos describe volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos describe volumes $ storageos describe volume --namespace my-namespace-name my-volume-name Flags: -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] …","ref":"/docs/reference/cli/describe/","title":"Describe"},{"body":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] Examples: $ storageos detach volume --namespace my-namespace-name my-volume Flags: --cas string make changes to a resource conditional upon matching a provided version -h, --help help for detach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] …","ref":"/docs/reference/cli/detach/","title":"Detach Volume"},{"body":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The bundle packages the information needed for the engineering team to understand the context of your cluster and begin troubleshooting any issues you may be experiencing.\nGenerate Bundle CLI The StorageOS CLI can generate the bundle.\n$ storageos get diagnostics The file generated is in the form of diagnostics-${TIME_STAMP}.gz\n GUI Or you can use the StoregeOS GUI.\n Go to section \u0026ldquo;Cluster\u0026rdquo; Press the button \u0026ldquo;DOWNLOAD DIAGNOSTICS\u0026rdquo;. Data collected in the bundle Most of the data collected in the bundle is regarding the state of the StorageOS cluster, however some other information regarding the infrastructure is also gathered. The information is used to have a clear view of the cluster where StorageOS is running.\nThe bundle incorporates for each node:\n StorageOS Daemonset Pod logs lshw dmesg (kernel logs) StorageOS metadata for the ControlPlane and DataPlane StorageOS metadata collected cluster metadata namespaces metadata nodes metadata volumes metadata capacity stats environment variables health Privacy StorageOS can only obtain the bundle if it is downloaded by the user and given to our engineering team, or uploaded for analysis. The data received by StorageOS is private and never leaves nor will leave StorageOS Inc.\nThe data contained in the cluster diagnostic bundle has the sole purpose of helping customers troubleshoot their issues.\n","excerpt":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The …","ref":"/docs/reference/bundles/diagnostic-bundle/","title":"Diagnostic bundle"},{"body":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page.\nDocker EE and the Universal Control Plane can run on different Linux distributions. StorageOS supports RHEL, CentOS, Debian, and selected Ubuntu images. For more details, check out the supported OSs in the prerequisites page.\nStorageOS only supports Kubernetes nodes managed by Docker Enterprise Edition, not those nodes running Swarm. Mixed nodes (those running Kubernetes and Swarm workloads) are not supported. As a consequence, StorageOS volumes can only be provisioned on Kubernetes nodes, and only these nodes should be used for stateful workloads.\n","excerpt":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page. …","ref":"/docs/platforms/dockeree/","title":"Docker Enterprise Edition"},{"body":"","excerpt":"","ref":"/docs/","title":"Documentation"},{"body":"ElasticSearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, most popularly used to aggregate logs, but also to serve as a search backend to a number of different applications.\nUsing StorageOS persistent volumes with ElasticSearch (ES) means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Elasticsearch pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nElasticsearch has features to allow it to handle data replication, and as such careful consideration of whether to allow StorageOS or Elasticsearch to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Elasticsearch on Kubernetes Prerequisites Some OS tuning is required, which is done automatically when using our example from the use cases repository.\nElasticsearch requires vm.max_map_count to be increased to a minimum of 262144, which is a system wide setting. One way to achieve this is to run sysctl -w vm.max_map_count=262144 and update /etc/sysctl.conf to ensure it persists over a reboot. See ElasicSearch reference here.\nAdministrators should be aware that this impacts the behaviour of nodes and that there may be collisions with other application settings. Administrators are advised to centrally collate sysctl settings using the tooling of their choice.\nDeployment of the application StatefulSet defintion ---apiVersion:apps/v1kind:StatefulSetmetadata:name:esdata[...]spec:serviceAccountName:elasticsearchcontainers:- name:dataimage:elasticsearch:6.7.0imagePullPolicy:IfNotPresent[...]volumeMounts:- name:datamountPath:/usr/share/elasticsearch/data/data[...]volumeClaimTemplates:- metadata:name:\u0026#34;data\u0026#34;spec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- default StorageOS storage class nameresources:requests:storage:10Gi# \u0026lt;--- change this to the appropriate valueThis excerpt is from the StatefulSet definition (/elasticsearch/10-es-data.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nInstallation Clone the use cases repo You can find the latest files in the StorageOS use cases repostiory in /elasticsearch/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases Create the kubernetes objects\nThis will install an ES cluster with 3 master, 3 data and 3 coordinator nodes. Combined they will require ~ 14 GiB of available memory in your cluster, however, more may be used as the application is being used\nkubectl apply -f ./elasticsearch/ Once completed, an internal service object will have been created making the cluster available as http://elasticsearch:9200/ which is the default Kibana (when installed via Helm) will be using.\n Confirm Elasticsearch is up and running\nkubectl get pods -l component=elasticsearch NAME READY STATUS RESTARTS AGE elasticsearch-exporter-d86ffd94-zw45l 1/1 Running 0 5m44s es-coordinator-b7b984dd4-7wlz5 1/1 Running 0 5m44s es-coordinator-b7b984dd4-89w26 1/1 Running 0 5m44s es-coordinator-b7b984dd4-b4t6j 1/1 Running 0 5m44s es-master-78dfd5b49f-9gf5c 1/1 Running 0 5m44s es-master-78dfd5b49f-smsbw 1/1 Running 0 5m44s es-master-78dfd5b49f-z4qpj 1/1 Running 0 5m44s esdata-0 1/1 Running 0 5m44s esdata-1 1/1 Running 0 4m34s esdata-2 1/1 Running 0 3m22s Connect to ElasticSearch\nTo connect to ES directly, you can use the following port-forward command\nkubectl port-forward svc/elasticsearch 9200 and then access it via http://localhost:9200\n Kibana (optional) One of the most popular uses of ES is to use it for log aggregation and indexing, Kibana helps us visualize the data in these indices and can be easily used when installed via its Helm chart\n Install the helm chart.\nhelm install stable/kibana Once installed, use a port-foward to Kibana instead of directly to ES\nkubectl port-forward --namespace default $(kubectl get pods --namespace default -l \u0026#34;app=kibana\u0026#34; -o jsonpath=\u0026#34;{.items[0].metadata.name}\u0026#34;) 5601 and then access it via http://localhost:5601\n Monitoring (optional) As part of the example deployment, ES metrics are exposed and can be scraped by Prometheus on port 9108 (see 77-es-exporter.yaml). This is enabled by default, and should work with the default Prometheus install via Helm. If you\u0026rsquo;re using the Prometheus service monitors, you can monitor this installation by creating a monitor for the es-exporter service. For an example of how this is done to monitor StorageOS, please see prometheus-setup.\n","excerpt":"ElasticSearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, …","ref":"/docs/usecases/elasticsearch/","title":""},{"body":"Etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to durably persist the StorageOS cluster state. As the backing store for Kubernetes, StorageOS uses etcd for many of the same reasons.\nStorageOS uses etcd as the single source of truth for all StorageOS objects. Whenever a request is made to create, update or delete an object the result is written to etcd before the request is completed. Using etcd as a configuration store allows nodes to retrieve the current cluster state after being offlined, allowing offlined nodes to rejoin the cluster.\n N.B. StorageOS v2.0 does not provide an embedded etcd server as previous versions did. You will need to setup an etcd server for StorageOS to use prior to installation of StorageOS. Please see our etcd prerequisites page for more information on how to install and configure etcd.\n ","excerpt":"Etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to …","ref":"/docs/concepts/etcd/","title":"Etcd"},{"body":"Check the etcd prerequisites page for a step by step installation of etcd.\nBest practices StorageOS uses etcd as a service, whether it is deployed following the step by step instructions or as a custom installation. It is expected that the user maintains the availability and integrity of the etcd cluster.\nIt is highly recommended to keep the cluster backed up and ensure high availability of its data.\nNetwork low latency It is important to keep the latency between StorageOS nodes and the etcd replicas low. Deploying an etcd cluster in a different data center or region can make StorageOS detect etcd nodes as unavailable due to latency. 10ms latency between StorageOS and etcd would be the maximum threshold for proper functioning of the system.\nDisk low latency Etcd is very sensitive to disk latency. Because of that, it is recommended to run etcd away from other IO-intensive workloads. Operations such as backups, builds or application bundling cause a heavy usage of disks. If such operations run alongside the etcd nodes, they will cause etcd to become unstable and suffer downtime. It is best to run etcd nodes isolated from other IO workloads.\nIOPS requirements As a general rule, for etcd to operate normally on production clusters, we recommend using the size of machine offered by your cloud provider that guarantees a minimum of 500 IOPS. For example, 750 baseline IOPS are guaranteed on a 250GB AWS gp2 EBS instance at time of writing, and block instances on other cloud providers will also specify baseline IOPS figures.\nCloud providers usually provide \u0026ldquo;bursts\u0026rdquo; of IOPS - temporarily higher rates, limited by credits - with larger volumes providing higher burst capacity. If you are relying on burst capacity for etcd, which requires sustained high performance, careful assessment is necessary to ensure sufficient capacity.\nThe rate of etcd operations is affected by the number of nodes, volumes and replicas in the cluster, therefore the figure of 500 is provided as a guideline only. A development cluster with 5 nodes will not have the same etcd traffic as a production cluster with 100 nodes. Adding monitoring to etcd will help to characterise the traffic, and therefore to assess the individual requirements of a cluster and adjust its resources accordingly.\nEtcd advertise urls The etcd startup parameters advertise-client-urls and initial-advertise-peer-urls specify the addresses etcd clients or other etcd members should use to contact the etcd server. The advertised addresses must be reachable from the remote machines - i.e. where StorageOS is running - so it can connect successfully. Do not advertise addresses like localhost or 0.0.0.0 for a production setup since these addresses are unreachable from remote machines.\nMonitoring It is highly recommended to add monitoring to the etcd cluster. Etcd serves Prometheus metrics on the client port http://etcd-url:2379/metrics.\nYou can use StorageOS developed Grafana Dashboards for etcd. When using etcd for production, you can use the etcd-cluster-as-service, while the etcd-cluster-as-pod can be used when using etcd from the operator.\nDefragmentation Etcd uses revisions to store multiple versions of keys. Compaction removes all key revision prior to a certain revision from etcd. Typically the etcd configuration enables the automatic compaction of keys to prevent performance degradation and limit the storage required. Compaction of revisions can create fragmentation that means space on disk is available for use by etcd but is unavailable for use by the file system. In order to reclaim this space, etcd can be defragmented.\nReclaiming space is important because when the etcd database file grows over the \u0026ldquo;DB_BACKEND_BYTES\u0026rdquo; parameter, the cluster triggers an alarm and sets itself read only and only allows reads and deletes. To avoid hitting the db backend bytes limit, compaction and defragmentation are required. How often defragmentation is required depends on the churn of key revisions in etcd.\nThe Grafana Dashboards mentioned above indicate when nodes require defragmentation. Be aware that defragmentation is a blocking operation that is performed per node, hence the etcd node will be locked for the duration of the defragmentation. Defragmentation usually takes a few milliseconds to complete.\nYou can also set cronjobs that execute the following defragmentation script. It will run a defrag when the DB is at 80% full. A defragmentation operation has to be executed per etcd node and it is a blocking operation. It is recommended to not execute the defragmentation on all etcd members at the same time. If using a cronjob, set them up for different times.\nhttps://raw.githubusercontent.com/storageos/deploy/master/k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd/roles/install_etcd/templates/defrag-etcd.sh.j2 chmod +x defrag-etcd.sh Known CoreOS Etcd Operator issues This topology is only recommended for deployments where isolated nodes cannot be used.\nEtcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd in pods it is therefore important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster causes the loss of all the etcd keystore and make StorageOS unable to perform metadata changes.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. A restore of the etcd keyspace from a backup might cause issues due to the disparity between the cluster state and its metadata in a different point in time. If you need to restore from a backup after a failure of etcd, contact the StorageOS support team.\n","excerpt":"Check the etcd prerequisites page for a step by step installation of etcd.\nBest practices StorageOS …","ref":"/docs/operations/etcd/","title":"Etcd"},{"body":"StorageOS requires an etcd cluster in order to function. For more information on why etcd is required please see our etcd concepts page.\nWe do not support using the Kubernetes etcd for StorageOS installations.\nWe provide two methods for installing etcd. For those looking for a quick route to evaluating StorageOS, our first method installs etcd into your Kubernetes cluster using the CoreOS operator. Due to limitations in the CoreOS operator, this installation is not backed by any persistent storage, and therefore is unsuitable for production installations.\nFor production installations, there is currently no satisfactory way of installing a production-grade etcd cluster inside Kubernetes (although the landscape is changing rapidly, watch this space), and our production guidance remains to install etcd on separate machines outside of your Kubernetes cluster. This method is the best way to ensure a stable StorageOS cluster. Please see our etcd operations page for additional information on deployment best practices and concerns.\n Ephemeral pods within Kubernetes (Testing) External Virtual Machines (Production) Click the tabs below to select the installation method of your choice.\n Testing Production Testing - Installing Etcd Into Your Kubernetes Cluster This fast and convenient method is useful for quickly creating an etcd cluster in order to evaluate StorageOS. Do not use it for production installations.\nThis method uses the CoreOS etcd-operator to install a 3 node etcd cluster within your Kubernetes cluster, in the storageos-etcd namespace. We then install a Kubernetes service in that same namespace.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. A restore of the etcd keyspace from a backup might cause issues due to the disparity between the cluster state and its metadata in a different point in time. If you need to restore from a backup after a failure of etcd, contact the StorageOS support team.\nQuick Install For a one command install, the following script uses kubectl to create an etcd cluster in the storageos-etcd namespace. It requires kubectl in the system path, and the context set to the appropriate cluster.\ncurl -s https://docs.storageos.com/sh/deploy-etcd.sh | bash Installation Step by Step For those who would prefer to execute the steps by themselves, they are as follows:\n Configure Namespace\nexport NAMESPACE=storageos-etcd Create Namespace\nkubectl create namespace $NAMESPACE If running in Openshift, an SCC is needed to start Pods\noc adm policy add-scc-to-user anyuid system:serviceaccount:$NAMESPACE:default Create ClusterRole and ClusterRoleBinding\n$ kubectl -n $NAMESPACE create -f-\u0026lt;\u0026lt;END apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: etcd-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: etcd-operator subjects: - kind: ServiceAccount name: default namespace: $NAMESPACE END $ kubectl -n $NAMESPACE create -f-\u0026lt;\u0026lt;END apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: etcd-operator rules: - apiGroups: - etcd.database.coreos.com resources: - etcdclusters - etcdbackups - etcdrestores verbs: - \u0026#34;*\u0026#34; - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - \u0026#34;*\u0026#34; - apiGroups: - \u0026#34;\u0026#34; resources: - pods - services - endpoints - persistentvolumeclaims - events verbs: - \u0026#34;*\u0026#34; - apiGroups: - apps resources: - deployments verbs: - \u0026#34;*\u0026#34; # The following permissions can be removed if not using S3 backup and TLS - apiGroups: - \u0026#34;\u0026#34; resources: - secrets verbs: - get END Deploy Etcd Operator\n$ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END Wait for the Etcd Operator Pod to start\nkubectl -n $NAMESPACE get pod -lapp=etcd-operator Create the EtcdCluster resource\n$kubectl-n$NAMESPACEcreate-f- \u0026lt;\u0026lt;ENDapiVersion:\u0026#34;etcd.database.coreos.com/v1beta2\u0026#34;kind:\u0026#34;EtcdCluster\u0026#34;metadata:name:\u0026#34;storageos-etcd\u0026#34;spec:size:3version:\u0026#34;3.4.7\u0026#34;pod:etcdEnv:- name:ETCD_QUOTA_BACKEND_BYTESvalue:\u0026#34;2147483648\u0026#34;# 2 GB- name:ETCD_AUTO_COMPACTION_RETENTIONvalue:\u0026#34;1000\u0026#34;# Keep 1000 revisions (default)- name:ETCD_AUTO_COMPACTION_MODEvalue:\u0026#34;revision\u0026#34;# Set the revision moderesources:requests:cpu:200mmemory:300MisecurityContext:runAsNonRoot:truerunAsUser:9000fsGroup:9000tolerations:- operator:\u0026#34;Exists\u0026#34;affinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight:100podAffinityTerm:labelSelector:matchExpressions:- key:etcd_clusteroperator:Invalues:- storageos-etcdtopologyKey:kubernetes.io/hostnameEND Installation Verification $ kubectl -n storageos-etcd get pod,svc NAME READY STATUS RESTARTS AGE pod/etcd-operator-55978c4587-8kx7b 1/1 Running 0 2h pod/storageos-etcd-qm9tmrpnlm 1/1 Running 0 2h pod/storageos-etcd-rzhjdz74hp 1/1 Running 0 2h pod/storageos-etcd-wvvv2d9g98 1/1 Running 0 2h NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/storageos-etcd ClusterIP None \u0026lt;none\u0026gt; 2379/TCP,2380/TCP 22h service/storageos-etcd-client ClusterIP 172.30.132.255 \u0026lt;none\u0026gt; 2379/TCP 22h The URL from the Service storageos-etcd-client.storageos-etcd.svc:2379 will be used later in the StorageOS Cluster CustomResource the kvBackend.address.\n Known etcd-operator issues Etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd using the CoreOS Operator, it is important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster causes the loss of all the etcd keystore and make StorageOS unable to perform metadata changes.\n Production - Etcd on External Virtual Machines For production installations, StorageOS strongly recommends running etcd outside of Kubernetes on a minimum of 3 dedicated virtual machines. This topology offers strong guarantees of resilience and uptime. We recommend this architecture in all environments, including those where Kubernetes is being deployed as a managed service.\nStorageOS doesn\u0026rsquo;t require a high performance etcd cluster, as the throughput of metadata to the cluster is low. However, we recommend a careful assessment of IOPS capacity best practices to ensure that etcd operates normally.\nDepending on the level of redundancy you feel comfortable with you can install etcd on the Kubernetes Master nodes. Take extreme care to avoid collisions of the StorageOS etcd installation with the Kubernetes etcd when using the Kubernetes Master nodes. Precautions such as changing the default configuration for the client and peer ports, and ensuring the etcd data directory is modified. The ansible playbook below will default the etcd installation directory to /var/lib/storageos-etcd.\nYou can choose between two installation options.\n Manual Installation Ansible Installation Installation - Manual This section documents the steps required for manual installation of etcd using standard package management commands and systemd manifests.\n Repeat the following steps on all the nodes that will run etcd as a systemd service.\n Configure Etcd version and ports\nexport ETCD_VERSION=\u0026#34;3.4.9\u0026#34; export CLIENT_PORT=\u0026#34;2379\u0026#34; export PEERS_PORT=\u0026#34;2380\u0026#34; If targeting Kubernetes Master nodes, you must change CLIENT_PORT, PEERS_PORT\n Download Etcd from CoreOS official site\ncurl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz mkdir -p /tmp/etcd-v${ETCD_VERSION}-linux-amd64 tar -xzvf /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -C /tmp/etcd-v${ETCD_VERSION}-linux-amd64 --strip-components=1 rm /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz Install Etcd binaries\ncd /tmp/etcd-v${ETCD_VERSION}-linux-amd64 mv etcd /usr/local/sbin/etcd3 mv etcdctl /usr/local/sbin/etcdctl chmod 0755 /usr/local/sbin/etcd3 /usr/local/sbin/etcdctl Set up persistent Etcd data directory\nmkdir /var/lib/storageos-etcd Create the systemd environment file\nOn all nodes that will run etcd create a systemd environemnt file /etc/etcd.conf which has the IPs of all the nodes. The NODE_IP will need to change to correspond to the node IP where the environment file resides. NODE1_IP, NODE2_IP and NODE3_IP will remain the same across all three files.\n$ cat \u0026lt;\u0026lt;END \u0026gt; /etc/etcd.conf # NODE_IP is the IP of the node where this file resides. NODE_IP=10.64.10.228 # Node 1 IP NODE1_IP=10.64.10.228 # Node 2 IP NODE2_IP=10.64.14.233 # Node 3 IP NODE3_IP=10.64.12.111 CLIENT_PORT=${CLIENT_PORT} PEERS_PORT=${PEERS_PORT} END # Verify that variables are expanded in the file $ cat /etc/etcd.conf Create the systemd unit file for etcd3 service\nCreate a systemd unit file /etc/systemd/system/etcd3.service with the following information:\n[Unit] Description=etcd3 Documentation=https://github.com/coreos/etcd Conflicts=etcd2.service [Service] Type=notify Restart=always RestartSec=5s LimitNOFILE=40000 TimeoutStartSec=0 EnvironmentFile=/etc/etcd.conf ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ --data-dir /var/lib/storageos-etcd \\ --quota-backend-bytes 8589934592 \\ --auto-compaction-retention 1000 \\ --auto-compaction-mode revision \\ --initial-cluster-state new \\ --initial-cluster-token etcd-token \\ --listen-client-urls http://${NODE_IP}:${CLIENT_PORT},http://127.0.0.1:${CLIENT_PORT} \\ --advertise-client-urls http://${NODE_IP}:${CLIENT_PORT} \\ --listen-peer-urls http://${NODE_IP}:${PEERS_PORT} \\ --initial-advertise-peer-urls http://${NODE_IP}:${PEERS_PORT} \\ --initial-cluster etcd-${NODE1_IP}=http://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=http://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=http://${NODE3_IP}:${PEERS_PORT} [Install] WantedBy=multi-user.target $NODE_IP is the IP address of the machine you are installing etcd on.`\n Note that setting the advertise-client-urls incorrectly will cause any client connection to fail. StorageOS will fail to communicate to Etcd.\n If enabling TLS, it is recomended to generate your own CA certificate and key. You will need to distribute the keys and certificates for the client auth on all etcd nodes. Moreover, the ExecStart value should look as below:\n ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ --data-dir /var/lib/storageos-etcd \\ --quota-backend-bytes 8589934592 \\ --auto-compaction-retention 1000 \\ --auto-compaction-mode revision \\ --peer-auto-tls \\ --client-cert-auth --trusted-ca-file=/path/to/client-cert.pem \\ --cert-file=/path/to/ca.pem \\ --key-file=/path/to/client-key.pem \\ --initial-cluster-state new \\ --initial-cluster-token etcd-token \\ --listen-client-urls https://${NODE_IP}:${CLIENT_PORT} \\ --advertise-client-urls https://${NODE_IP}:${CLIENT_PORT} \\ --listen-peer-urls https://${NODE_IP}:${PEERS_PORT} \\ --initial-advertise-peer-urls https://${NODE_IP}:${PEERS_PORT} \\ --initial-cluster etcd-${NODE1_IP}=https://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=https://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=https://${NODE3_IP}:${PEERS_PORT} Reload and start the etc3 systemd service\n$ systemctl daemon-reload $ systemctl enable etcd3.service $ systemctl start etcd3.service Installation Verification\n The etcdctl binary is installed at /usr/local/bin on the nodes.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:${CLIENT_PORT} member list # $NODE_IP - the IP of the node 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Read the etcd operations page for our etcd recommendations.\n Installation - Ansible For a repeatable and automated installation, use of a configuration management tool such as ansible is recommended. StorageOS provides an ansible playbook to help you deploy etcd on standalone virtual machines.\n Clone StorageOS deployment repository\ngit clone https://github.com/storageos/deploy.git cd k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd Edit the inventory file\n The inventory file targets the nodes that will run etcd. The file hosts is an example of such an inventory file.\n $ cat hosts [nodes] centos-1 ip=\u0026#34;10.64.10.228\u0026#34; fqdn=\u0026#34;ip-10-64-10-228.eu-west-2.compute.internal\u0026#34; centos-2 ip=\u0026#34;10.64.14.233\u0026#34; fqdn=\u0026#34;ip-10-64-14-233.eu-west-2.compute.internal\u0026#34; centos-3 ip=\u0026#34;10.64.12.111\u0026#34; fqdn=\u0026#34;ip-10-64-12-111.eu-west-2.compute.internal\u0026#34; # Edit the inventory file $ vi hosts # Or your own inventory file The ip or fqdn are used to expose the advertise-client-urls of Etcd. Failing to provide valid ip/fqdn will cause any client connection to fail. StorageOS will fail to communicate to Etcd.\n Edit the etcd configuration\n If targeting Kubernetes Master nodes, you must change etcd_port_client, etcd_port_peers\n $ cat group_vars/all etcd_version: \u0026#34;3.4.9\u0026#34; etcd_port_client: \u0026#34;2379\u0026#34; etcd_port_peers: \u0026#34;2380\u0026#34; etcd_quota_bytes: 8589934592 # 8 GB etcd_auto_compaction_mode: \u0026#34;revision\u0026#34; etcd_auto_compaction_retention: \u0026#34;1000\u0026#34; members: \u0026#34;{{ groups[\u0026#39;nodes\u0026#39;] }}\u0026#34; installation_dir: \u0026#34;/var/lib/storageos-etcd\u0026#34; advertise_format: \u0026#39;fqdn\u0026#39; # fqdn || ip backup_file: \u0026#34;/tmp/backup.db\u0026#34; tls: enabled: false ca_common_name: \u0026#34;eu-west-2.compute.internal\u0026#34; etcd_common_name: \u0026#34;*.eu-west-2.compute.internal\u0026#34; cert_dir: \u0026#34;/etc/etcdtls\u0026#34; ca_cert_file: \u0026#34;etcd-ca.pem\u0026#34; etcd_server_cert_file: \u0026#34;server.pem\u0026#34; etcd_server_key_file: \u0026#34;server-key.pem\u0026#34; etcd_client_cert_file: \u0026#34;etcd-client.crt\u0026#34; etcd_client_key_file: \u0026#34;etcd-client.key\u0026#34; $ vi group_vars/all Choose between using IP addressing or FQDN in the advertise_format parameter. It allows you to decide how Etcd advertises its address to clients. This is particularly relevant when using TLS.\n If enabling TLS, it is recomended to generate your own CA certificate and key. You can do it by generating the CA from the machine running Ansible by: ansible-playbook create_ca.yaml.\n Install\nansible-playbook -i hosts install.yaml Installation Verification\n The playbook installs the etcdctl binary on the nodes, at /usr/local/bin.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 member list 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Benefits of Running External to Kubernetes Etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. In the case that quorum is lost, an etcd node stops and marks its contents as read-only. Another peer might have a newer version that has not been committed to the database. Quorum is fundamental for etcd operations.\nIn a Kubernetes environment, applications are scheduled across and in some scenarios such as \u0026ldquo;DiskPressure\u0026rdquo; they may need to be evicted from a node, and be scheduled onto a different node. With an application such as etcd, the scenario described can result in quorum being lost, making the cluster unable to recover automatically. Usually a 3 node etcd cluster can survive losing one node and recover. However, losing a second node at the same time or even having a network partition between them will result in quorum lost.\nBind Etcd IPs to Kubernetes Service Kubernetes external services use a DNS name to reference external endpoints, making them easy to reference from inside the cluster. You can use the example from the helper github repository to deploy the external Service. Using an external service can make monitoring of etcd from Prometheus easier.\n Using Etcd with StorageOS During installation of StorageOS the kvBackend.address parameter of the StorageOS operator is used to specify the address of the etcd cluster. See the StorageOS cluster operator configuration examples for more information.\n","excerpt":"StorageOS requires an etcd cluster in order to function. For more information on why etcd is …","ref":"/docs/prerequisites/etcd/","title":"Etcd"},{"body":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. StorageOS implements dynamic provisioning, so the creation of a PVC will automatically provision a PV (PersistentVolume) that can be used to persist data written by a Pod.\nCreate the PersistentVolumeClaim You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic PVC definition\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;fast\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC will dynamically provision a 5GB volume using the fast StorageClass. This StorageClass was created during the StorageOS install and triggers creation of a PersistentVolume by StorageOS.\nFor installations with CSI, you can create multiple StorageClasses in order to specify default labels.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels. For simplicity\u0026rsquo;s sake this example will use unreplicated volumes.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# Reference to the StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1labels:storageos.com/replicas:\u0026#34;1\u0026#34;spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC has the storageos.com/replicas label set. This label tells StorageOS to add a replica for the volume that is created. For the sake of keeping this example simple an unreplicated volume will be used.\n Move into the examples folder and create a PVC using the PVC definition above.\n$ # from storageos-usecases/00-basic $ kubectl create -f ./pvc-basic.yaml You can view the PVC that you have created with the command below\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-1 Bound pvc-f8ffa027-e821-11e8-bc0b-0ac77ccc61fa 5Gi RWO fast 1m Create a pod that mounts the PVC created in step 2.\n$ kubectl create -f ./pod.yaml The command above creates a Pod that uses the PVC that was created in step 1.\napiVersion:v1kind:Podmetadata:name:d1spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mntname:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-1In the Pod definition above volume v1 references the PVC created in step 2, and is mounted in the pod at /mnt. In this example a debian image is used for the container but any container image with a shell would work for this example.\n Confirm that the pod is up and running\n$ kubectl get pods NAME READY STATUS RESTARTS AGE d1 1/1 Running 0 1m Execute a shell inside the container and write some contents to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo \u0026#34;Hello World!\u0026#34; \u0026gt; /mnt/helloworld root@d1:/# cat /mnt/helloworld Hello World! By writing to /mnt inside the container, the StorageOS volume created by the PVC is being written to. If you were to kill the pod and start it again on a new node, the helloworld file would still be avaliable.\nIf you wish to see more use cases with actual applications please see our Use Cases documentation.\n ","excerpt":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. …","ref":"/docs/operations/firstpvc/","title":"StorageOS Volume Guide"},{"body":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available Commands: cluster Fetch cluster-wide configuration details diagnostics Fetch a cluster diagnostic bundle namespace Retrieve basic details of cluster namespaces node Retrieve basic details of nodes in the cluster user Fetch user details volume Retrieve basic details of volumes Flags: -h, --help help for get Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos get [command] --help\u0026#34; for more information about a command. get cluster $ storageos get cluster --help Fetch cluster-wide configuration details Usage: storageos get cluster [flags] Examples: $ storageos get cluster Flags: -h, --help help for cluster get diagnostics $ storageos get diagnostics --help Fetch a cluster diagnostic bundle Usage: storageos get diagnostics [flags] Examples: $ storageos get diagnostics $ storageos get diagnostics --output-file ~/my-diagnostics Flags: -h, --help help for diagnostics --output-file string writes the generated diagnostic bundle to a specified file path get namespace $ storageos get namespace -h Retrieve basic details of cluster namespaces Usage: storageos get namespace [namespace names...] [flags] Aliases: namespace, namespaces Examples: $ storageos get namespaces $ storageos get namespace my-namespace-name Flags: -h, --help help for namespace -l, --selector stringArray filter returned results by a set of comma-separated label selectors get node $ storageos get node -h Retrieve basic details of nodes in the cluster Usage: storageos get node [node names...] [flags] Aliases: node, nodes Examples: $ storageos get node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors get user $ storageos get user -h Fetch user details Usage: storageos get user [user names...] [flags] Aliases: user, users Examples: $ storageos get user my-username $ storageos get user my-username-1 my-username-2 $ storageos get user --use-ids my-userid $ storageos get user --use-ids my-userid-1 my-userid-2 Flags: -h, --help help for user get volume $ storageos get volume -h Retrieve basic details of volumes Usage: storageos get volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos get volumes --all-namespaces $ storageos get volume --namespace my-namespace-name my-volume-name Flags: --all-namespaces retrieves volumes from all accessible namespaces. This option overrides the namespace configuration -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors ","excerpt":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available …","ref":"/docs/reference/cli/get/","title":"Get"},{"body":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any of the nodes in the cluster. Initially you can log in as the default administrator, with the username and password from the storageos-api Secret. By default storageos, storageos.\n You can access the GUI by either port-forwarding with kubectl or using an Ingress rule. i.e kubectl -n kube-system port-forward svc/storageos 5705.\n Nodes Volumes You can create volumes, including replicated volumes, and view volume details:\nLicensing Cluster info Namespaces Volumes can be namespaced across different projects or teams, and you can switch namespace using the left hand panel:\n","excerpt":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any …","ref":"/docs/reference/gui/","title":"Graphical user interface (GUI)"},{"body":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the status of nodes in the cluster.\n$ storageos get nodes NAME HEALTH AGE LABELS node1 online 44 minutes ago node2 online 44 minutes ago node3 online 44 minutes ago node4 online 44 minutes ago node5 online 44 minutes ago ","excerpt":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the …","ref":"/docs/operations/health/","title":"Cluster health"},{"body":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Use \u0026#34;storageos [command] --help\u0026#34; for more information about a command. ","excerpt":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to …","ref":"/docs/reference/cli/help/","title":"Help"},{"body":"StorageOS uses the storage available on the nodes where it is installed to present as available for volumes.\nIn order to mitigate against problems caused by filling the host root disk, we recommend mounting a separate device into the /var/lib/storageos directory. StorageOS is agnostic to the type of filesystem mounted in /var/lib/storageos.\nExtending Available Storage StorageOS uses subdirectories of /var/lib/storageos/data to hold user data. By default, the directory /var/lib/storageos/data/dev1 will be created when a node is bootstrapped, and used for pool data. It is possible to shard the data by creating more directories into this structure. StorageOS will save data in any directory that conforms to the pattern /var/lib/storageos/data/dev[0-9]+, such as /var/lib/storageos/data/dev2 or /var/lib/storageos/data/dev5. This functionality enables operators to mount different devices into devX directories and StorageOS will recognise them as available storage automatically.\nThere are two possible options to expand the available disk space for StorageOS to allocate:\n Mount filesystem in /var/lib/storageos/data/devX Use LVM to expand the logical volume available to StorageOS Option 1: Mount Additional Devices This option enables operators to expand the cluster\u0026rsquo;s available space at any time without having to stop applications or forcing operational downtime. The expansion of disk is transparent for applications and StorageOS Volumes. StorageOS will use the new available space to create new data files.\n Context\nWe assume that there is a disk available in our Linux system without formatting in addition to the root filesystem. StorageOS data dir dev1 (/var/lib/storageos/data/dev1) is using /dev/xvda1. We will use the device /dev/xvdf to expand StorageOS available space.\nList available block devices in the host.\nroot@node0:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Format device\nroot@node0:/var/lib/storageos/data# mkfs -t ext4 /dev/xvdf mke2fs 1.42.12 (29-Aug-2014) Creating filesystem with 26214400 4k blocks and 6553600 inodes Filesystem UUID: 380712fa-6f82-477a-81a5-d7466d4c6b7f Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 4096000, 7962624, 11239424, 20480000, 23887872 Allocating group tables: done Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done Mount filesystem\nroot@node0:~# mkdir -p /var/lib/storageos/data/dev2 root@node0:~# mount /dev/xvdf /var/lib/storageos/data/dev2 Verify available storage\nIn less than 30 seconds, StorageOS will see the new available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 244,491,013,324 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Note that the node node0 has increased the TOTAL capacity in 100Gi.\n Persist the mount at boot by adding the mount endpoint to /etc/fstab\n Option 2: Expand Existing Devices Backed by LVM This option enables operators to take advantage of LVM to manage disks.\n Context\nWe assume that /var/lib/storageos is mounted onto an LVM volume. We are using a volumegroup named storageos and logical volume called data. There is a second physical disk /dev/xvdg unused.\nList available block devices in the host.\nroot@node2:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk `-storageos-data 254:0 0 99G 0 lvm /var/lib/storageos xvdg 202:96 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 107,696,304,947 # --\u0026gt; LVM storageos/data volume Add physical disk to LVM\nroot@node2:~# vgextend storageos /dev/xvdg Volume group \u0026quot;storageos\u0026quot; successfully extended The volume group storageos must have 2 physical volumes (#PV)\nroot@node2:~# vgs VG #PV #LV #SN Attr VSize VFree storageos 2 1 0 wz--n- 199.99g 104.99g Extend logical volume data\nroot@node2:~# lvextend -L+100G /dev/storageos/data Size of logical volume storageos/data changed from 95.00 GiB (24320 extents) to 195.00 GiB (49920 extents). Logical volume data successfully resized Resize the FileSystem\n Your filesystem must support the option to be expanded, and to do so while in use. Otherwise, you need to unmount first.\n root@node2:~# resize2fs /dev/storageos/data resize2fs 1.42.12 (29-Aug-2014) Filesystem at /dev/storageos/data is mounted on /var/lib/storageos; on-line resizing required old_desc_blocks = 6, new_desc_blocks = 13 The filesystem on /dev/storageos/data is now 51118080 (4k) blocks long. Check new available space\nThe mounted file system to /var/lib/storageos has increased its size.\nroot@node2:~# df -h /dev/mapper/storageos-data Filesystem Size Used Avail Use% Mounted on /dev/mapper/storageos-data 192G 60M 183G 1% /var/lib/storageos StorageOS available storage has increased too.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 206,158,430,208 # --\u0026gt; 100G more available Persist the mount at boot by adding the mount point to /etc/fstab\n ","excerpt":"StorageOS uses the storage available on the nodes where it is installed to present as available for …","ref":"/docs/operations/managing-host-storage/","title":"Managing Host Storage"},{"body":"InfluxDB with StorageOS InfluxDB is a popular open source time series database application optimised for managing datasets consisting of many small measurements. Its advantages include the ability to handle very high write and query loads. Its uses include monitoring, analytics and the recording and analysis of data from sensors.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying InfluxDB on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition:\napiVersion:apps/v1kind:StatefulSetmetadata:name:influxdbspec:replicas:1selector:matchLabels:app:influxdbserviceName:influxdb...spec:serviceAccountName:influxdb...volumeMounts:- mountPath:/var/lib/influxdbname:data...volumeClaimTemplates:- metadata:name:dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:20GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaimTemplate that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Create the InfluxDB objects\nkubectl create -f ./influxdb Confirm InfluxDB is up and running.\n$ kubectl get pods NAME READY STATUS RESTARTS AGE influxdb-client 1/1 Running 0 1m influxdb-0 1/1 Running 0 1m Connect to the InfluxDB client pod, then to the InfluxDB server through the service (this reflects the common kubernetes pattern of maintaining a client pod to conveniently inspect a resource interactively). The default user (admin) and password (admin) are defined in the StatefulSet.\n$ kubectl exec -it influxdb-client -- bash root@influxdb-client:/# influx -host influxdb-0.influxdb Connected to http://influxdb-0.influxdb:8086 version 1.8.2 InfluxDB shell version: 1.8.2 \u0026gt; auth username: admin password: \u0026gt; show databases name: databases name _internal \u0026gt; CREATE DATABASE weather; \u0026gt; USE weather Using database weather \u0026gt; INSERT temperature,location=London value=26.4 \u0026gt; INSERT temperature,location=London value=24.9 \u0026gt; INSERT temperature,location=London value=22.2 \u0026gt; INSERT temperature,location=London value=14.7 \u0026gt; INSERT temperature,location=London value=19.5 \u0026gt; INSERT temperature,location=Paris value=27.1 \u0026gt; INSERT temperature,location=Paris value=27.5 \u0026gt; INSERT temperature,location=Paris value=21.3 \u0026gt; INSERT temperature,location=Paris value=26.7 \u0026gt; INSERT temperature,location=Paris value=30.0 \u0026gt; SELECT MEAN(*) FROM \u0026#34;temperature\u0026#34; GROUP BY \u0026#34;location\u0026#34; name: temperature tags: location=London time mean_value 0 25.65 name: temperature tags: location=Paris time mean_value 0 26.90 In the above steps we have inserted some time series data on the temperature at two locations, and calculated the mean for both. InfluxDB offers a variety of such aggregations (see the docs here), allowing convenient analysis of time series data.\nConfiguration If you need custom startup options, you can edit or add to the environment variables within the 20-statefulset.yaml file.\nBackups In this example of how to perform backups of an InfluxDB database on a Kubernetes cluster, we write the output backup file to an Amazon Web Services (AWS) S3 bucket. Other approaches, such as backing up to internal servers or other StorageOS volumes, are possible. For this example to run successfully, Base64-encoded AWS credentials and an S3 bucket name should be inserted into the data field of the backup/50-secret-config.yaml file.\n$ echo -n \u0026#39;\u0026lt;your-aws-access-key-id\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX $ echo -n \u0026#39;\u0026lt;your-aws-secret-access-key\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX $ echo -n \u0026#39;\u0026lt;your-aws-default-region\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX $ echo -n \u0026#39;\u0026lt;your-S3-bucket-name\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX Secret definition:\napiVersion:v1kind:Secretmetadata:name:backup-pod-environmenttype:Opaquedata:AWS_ACCESS_KEY_ID:XXXXXXXXXXXXAWS_SECRET_ACCESS_KEY:XXXXXXXXXXXXAWS_DEFAULT_REGION:XXXXXXXXXXXXBUCKET_NAME:XXXXXXXXXXXXDB_NAME:d2VhdGhlcg==DB_HOST:aW5mbHV4ZGItMC5pbmZsdXhkYjo4MDg4To perform the backup, create the secret and job from the manifest files in the backup directory.\nkubectl create -f ./influxdb/backup/ Confirm that the backup pod has been created, and the backup performed successfully.\nNAME READY STATUS RESTARTS AGE backup-ks976 0/1 Completed 0 1m client 1/1 Running 0 10m influxdb-0 1/1 Running 0 10m mysql-0 1/1 Running 0 10m The files generated by the backup operation should now be available in you S3 bucket. Re-perform the operation at any time by deleting and re-creating the backup job.\n$ kubectl delete -f \u0026#34;influxdb/40-backup-job.yaml\u0026#34; job.batch \u0026#34;backup\u0026#34; deleted $ kubectl create -f \u0026#34;influxdb/40-backup-job.yaml\u0026#34; job.batch/backup created ","excerpt":"InfluxDB with StorageOS InfluxDB is a popular open source time series database application optimised …","ref":"/docs/usecases/influxdb/","title":"InfluxDB"},{"body":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init container that sets the system configuration for StorageOS. The container also manages configuration changes required when upgrading StorageOS versions.\nThe container belongs to the DaemonSet that the StorageOS Cluster Operator starts when a StorageOSCluster resource is created. The storageos-init container is executed as an initContainer as part of a Kubernetes Pod. Therefore, only successful execution of the storageos-init container processes will result in the main container starting.\nScript Framework The code responsible for fulfilling the requirements is based on a Script Framework.\nThe script framework executes a set of scripts, performing checks, verifications and other procedures needed for StorageOS to be able to start. The scripts stdout and stderr are written to the stdout and stderr of the init app. The container logs contain all the logs of the individual scripts that run. The exit statuses of the scripts are used to determine initialization failure or success. Any non-zero exit status is logged as an event in the Kubernetes Pod events.\nIf any of the scripts fail, the storageos-init container will propagate the failure to Kubernetes, showing the status of the Pod as Init:Err.\nTo view the output of all storageos-init containers the following command can be used:\nkubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init For more details, check the StorageOS init container project.\nScripts executed The storageos-init container executes the following scripts.\n enable-lio dbupgrade ","excerpt":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init …","ref":"/docs/reference/init-container/","title":"Init container"},{"body":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a StorageOS persistent volume being mounted on /var/jenkins_home. Deploying Jenkins using StorageOS offers multiple benefits. Firstly Jenkins can spin up multiple build pods at once to allow concurrent builds of different projects. Secondly Jenkins configuration is on a PersistentVolume so even if the Jenkins pod is rescheduled the configuration will persist.\nUsing StorageOS volume replicas allows for failure of nodes holding the PersistentVolume without interrupting Jenkins. Lastly by enabling StorageOS fencing Jenkins time to recover, in case of node failures, is greatly reduced.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information\nDeploying Jenkins on Kubernetes You can find the latest files in the StorageOS example deployment repository.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./jenkins Confirm that Jenkins is up and running\n$ kubectl get pods -w -l app=jenkins NAME READY STATUS RESTARTS AGE jenkins-0 1/1 Running 0 1m Connect to the Jenkins UI through the Jenkins service.\nYou can do this by port forwarding the Jenkins Kubernetes service to your localhost and accessing the UI via your browser. Alternatively if you have network access to your Kubernetes nodes then you can create a NodePort service and access Jenkins like that. A NodePort service has been left in 10-service.yaml commented out.\nTo port-foward the Jenkins service use the following command.\n$ kubectl port-foward svc/jenkins 8080 To login to the Jenkins UI use the credentials specified in 07-config.yaml, unless these have been changed from the defaults the username/password is admin/password.\n Create a Jenkins job.\nOnce you are logged into the UI you can create a job that will be farmed out to a Kubernetes plugin build agent. Click New Item, enter a name for the project and select Freestyle project. Next add an Execute shell build step. As a proof of concept you can use the bash below to have the pod execute a sleep.\n#!/bin/bash sleep 1000 Save the project and select Schedule a build of your project. You can watch for the appearance of a build pod using kubectl get pods -l jenkins=agent -w. Once the pod is created you should see the Build Executor status in the Jenkins UI display the pod.\nTo see multiple projects being built at once create another project and try scheduling a build of both projects at the same time.\n ","excerpt":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a …","ref":"/docs/usecases/jenkins/","title":""},{"body":"Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub and traditional queues.\nUsing StorageOS persistent volumes with Apache Kafka means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the kafka pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nKafka has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Kafka to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Apache Zookeeper is required by Kafka to function; we assume it to already exist and be accessible within the Kubernetes cluster as zookeeper, see how to run Zookeeper with StorageOS here StorageOS is assumed to have been installed; please check for the latest available version here Kafka pods require 1536 MB of memory for successful scheduling Helm To simplify the deployment of kafka, we\u0026rsquo;ve used this Kafka helm chart (incubator) (version 0.13.8, app version 5.0.1) and rendered it into the example deployment files you can find in our GitHub repo.\nClone the use cases repo You can find the latest files in the StorageOS use cases repository in /kafka/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition\n---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:kafkalabels:app:kafka...spec:serviceName:kafka-headlesspodManagementPolicy:OrderedReadyupdateStrategy:type:OnDeletereplicas:3# \u0026lt;--- number of kafa pods to runtemplate:...spec:serviceAccountName:kafkacontainers:...- name:kafka-brokerimage:\u0026#34;confluentinc/cp-kafka:5.0.1\u0026#34;imagePullPolicy:\u0026#34;IfNotPresent\u0026#34;...volumeMounts:- name:datadirmountPath:\u0026#34;/var/data\u0026#34;volumes:- name:jmx-configconfigMap:name:kafka-metricsterminationGracePeriodSeconds:60volumeClaimTemplates:- metadata:name:datadirspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]resources:requests:storage:10Gi# \u0026lt;--- storage requested for each podstorageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Create the kubernetes objects\nkubectl apply -f ./kafka/ Confirm kafka is up and running\n$ kubectl get pods -l app=kafka NAME READY STATUS RESTARTS AGE kafka-0 2/2 Running 0 10m kafka-1 2/2 Running 0 9m26s kafka-2 2/2 Running 0 7m59s Connect to kafka\nConnect to the kafka test client pod and send some test data to kafka through its service endpoint\n Connect to the pod\nkubectl exec -it kafka-test-client /bin/bash Create a topic\n/usr/bin/kafka-topics --zookeeper zookeeper:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1 Send some test data\n/usr/bin/kafka-run-class org.apache.kafka.tools.ProducerPerformance --topic test-rep-one --num-records 5000 --record-size 100 --throughput -1 --print-metrics --producer-props acks=1 bootstrap.servers=kafka:9092 buffer.memory=67108864 batch.size=8196 ","excerpt":"Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub …","ref":"/docs/usecases/kafka/","title":""},{"body":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual machines (VMs) in Kubernetes pods.\nDeploying Kubevirt using StorageOS offers multiple benefits. Kubevirt can spin up VMs as Kubernetes pods, using images on StorageOS persistent volumes. Doing this allows the VM data to persist through restarts and rescheduling. Using StorageOS volume replicas also allows for failure of nodes holding the PersistentVolume without interrupting the VM running off the PersistentVolume. Containerized Data Importer (CDI) can also be used to prepare StorageOS volumes with disk images in an automated fashion. Simply by declaring that a VirtualMachine will use a DataVolume and providing the disk image URL, a StorageOS volume can be dynamically provisioned and automatically prepared with the disk image.\nThis usecase will guide you through installing KubeVirt and CDI on your Kubernetes cluster, and create a VM. By the end of the guide you\u0026rsquo;ll be able to launch a shell inside the KubeVirt VM that\u0026rsquo;s running as a Kubernetes pod.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Please ensure you have met the Kubevirt prerequisites, please see the Kubevirt installation instructions for more information.\nAs part of this installation it is assumed that you are running a Kubernetes cluster on VMs. As such nested virtualization or hardware emulation need to be enabled.\nDeploying KubeVirt on Kubernetes For ease of installation we have enabled hardware emulation. If your VMs support nested virtualization then edit the Kubevirt ConfigMap ./kubevirt-install/10-cm.yaml, removing the line debug.useEmulation: \u0026quot;true\u0026quot;.\n In order to deploy Kubevirt you just need to clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/kubevirt $ kubectl create -f ./kubevirt-install Check that the Kubevirt pods are running.\n$ kubectl get pods -w -n kubevirt NAME READY STATUS RESTARTS AGE virt-api-57546d479b-p26d4 1/1 Running 0 1m virt-api-57546d479b-zs5dw 1/1 Running 0 1m virt-controller-56b5498854-7xsfz 1/1 Running 1 1m virt-controller-56b5498854-bz559 1/1 Running 1 1m virt-handler-6z4kq 1/1 Running 0 1m virt-handler-7szhl 1/1 Running 0 1m virt-handler-jmm6w 1/1 Running 0 1m virt-operator-79c9bdd859-8xq98 1/1 Running 0 1m virt-operator-79c9bdd859-kfjz6 1/1 Running 0 1m Once Kubevirt is running install CDI.\n$ kubectl create -f ./cdi Check that the CDI pods are running correctly.\n$ kubectl get pods -n cdi NAME READY STATUS RESTARTS AGE cdi-apiserver-8668f888df-s6pp4 1/1 Running 0 1m cdi-deployment-5cf794896b-whh4j 1/1 Running 0 1m cdi-operator-5887f96c-dz2hg 1/1 Running 0 1m cdi-uploadproxy-97fbbfcbf-6f9xs 1/1 Running 0 1m Now that CDI and Kubevirt are running, VMs can be created. In this example VMs running Cirros, a small and lightweight OS, will be created. The vm-cirros.yaml manifest creates a VirtualMachine that uses a DataVolume. This means that CDI will create a StorageOS backed PVC and download the image that the VirtualMachineInstance (VMI) will boot from onto the PVC.\n$ kubectl create -f ./vm-cirros.yaml Check that the VMI is running. Note that the VMI will only be created after CDI has downloaded the Cirros disk image onto a StorageOS persistent volume so depending on your connection speed this may take some time.\n$ kubectl get vmi NAME AGE PHASE IP NODENAME cirros 1m Running 10.244.2.12 ip-10-1-10-154.storageos.net $ kubectl get pods NAME READY STATUS RESTARTS AGE virt-launcher-cirros-drqhr 1/1 Running 0 1m Connect to the VM console.\nThis example uses the virtctl kubectl plugin in order to connect to the VMs console. The escape sequence ^] is ctrl + ]\n$ kubectl virt console cirros Successfully connected to cirros console. The escape sequence is ^] login as \u0026#39;cirros\u0026#39; user. default password: \u0026#39;gocubsgo\u0026#39;. use \u0026#39;sudo\u0026#39; for root. cirros login: cirros Password: $ Cloning Volumes CDI allows for images to be cloned using a DataVolume manifest. Verify that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n N.B. Ensure that the VMI is stopped before continuing!\n Verify that the VMI is stopped before continuing, and that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE cirros Bound pvc-f4833060-5a77-420c-927e-6bc518d9df3c 12Gi RWO fast 1m Once the PVC\u0026rsquo;s existence is confirmed then create a new DataVolume that uses the cirros PVC as its source.\n$ kubectl create -f ./cloned.yaml Watch as the CDI pods are created.\n$ kubectl get pods -w You\u0026rsquo;ll see that a cdi-upload-cloned-datavolume pod is created and then a cdi-clone-source pod is created. The cdi-source pod mounts the original cirros volume and sends the contents of the volume to the cdi-upload pod. The cdi-upload pod creates and mounts a new PVC and writes the contents of the original volume to it.\n ","excerpt":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual …","ref":"/docs/usecases/kubevirt/","title":""},{"body":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to various StorageOS artefacts. Applying specific feature labels triggers compression, replication and other storage features. No feature labels are present by default.\nStorageOS Node labels Nodes do not have any feature labels present by default. When StorageOS is run within Kubernetes, the Cluster Operator syncs any node labels set on Kubernetes nodes within StorageOS. Node labels may also be set with the CLI or UI.\n Feature Label Values Description Compute only storageos.com/computeonly true / false Specifies whether a node should be computeonly where it only acts as a client and does not host volume data locally, otherwise the node is hyperconverged (the default), where the node can operate in both client and server modes. You can set the computeonly label on the Kubernetes node and the label will be sync\u0026rsquo;d to the StorageOS node (labels take an eventual consistency reconciliation time of ~1min).\nkubectl label node $NODE storageos.com/computeonly=true StorageOS Volume labels Volumes do not have any feature labels present by default. WARNING: The caching and compression labels can only apply during provision time, they can\u0026rsquo;t be changed during execution.\n Feature Label Values Description Caching storageos.com/nocache true / false Switches off caching. Compression storageos.com/nocompress true / false Switches off compression of data at rest and in transit. Replication storageos.com/replicas integers [0, 5] Replicates entire volume across nodes. Typically 1 replica is sufficient (2 copies of the data); more than 2 replicas is not recommended. To create a volume with a feature label:\n Option 1: PVC Label\nAdd the label in the PVC definition, for instance:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc-3labels:storageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;-----spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:1G Option 2: Set label in the StorageClass\nAny PVC using the StorageClass inherits the label. The PVC label takes precedence over the StorageClass parameters.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;--------provisioner:storageos# CSI driver (recommended) Option 3:\nOnce a PVC is created, you can update the Labels in StorageOS both in the UI or CLI. Those labels are going to be visible only for StorageOS and will not be synced to the Kubernetes resource.\n ","excerpt":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to …","ref":"/docs/reference/labels/","title":"StorageOS Feature labels"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying a Free Developer licence.\nTo learn how to apply a licence to your cluster, check the operations licensing page.\nFree Developer licence The developer licence only requires the user to register and issue a licence through StorageOS.\nThe developer licence is free and grants a licence for a 5 node StorageOS cluster with 5TB of provisioned capacity. It is designed to enable basic cloud native workflows in Kubernetes that require the persistence of stateful application data. Dynamic provisioning, distributed access to data and high availability of volumes through synchronous replication and automatic failover are some of the features that are available under the developer licence.\nThe free developer licence includes community support through our public Slack channel..\nCommercial licences For information on our commercial offerings, including support, please contact sales@storageos.com.\nNote about capacity limits Some StorageOS licences have limits on capacity. StorageOS allows provisioning volumes until the limit of the licence is reached. Only the size of the volume requested by the Persistent Volume Claim counts for the licence limit, regardless whether of whether the volume has replication enabled.\nOnce the licence limit is reached, new volumes are not able to provision unless provisioned capacity is released, i.e deleting volumes. That behaviour is not tight to the capacity of the backend disks on your nodes.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/docs/reference/licence/","title":"Licence"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying for a Free Developer licence.\nObtaining a Developer licence via the GUI You will need access to the StorageOS GUI on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 As an alternative, an Ingress controller may be preferred.\nOnce you have obtained access to the GUI, login using whatever credentials you used to create the cluster.\nYou can obtain and apply a free Developer licence in the StorageOS web GUI by creating or logging in with a StorageOS account on the StorageOS portal via the licence page of the GUI:\nWait a few seconds for the licence generation process to complete, at which point your licence will be visible.\nApplying a previously obtained licence via the GUI Commercial licences are delivered through contact with the StorageOS team. To apply such keys, via the web GUI, visit the licence section of the GUI and click on the tab \u0026ldquo;Upgrade\u0026rdquo;, for the specific licence level you purchased. Then paste the licence key and click on \u0026ldquo;UPLOAD KEY TO CLUSTER\u0026rdquo;. Note that you can also view your cluster ID on the same page.\nApplying a licence via the CLI Before getting a licence, you need to know the ID of your StorageOS cluster.\nThis CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence. Once, given the key, you can apply the licence by using the following command.\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos apply licence --from-stdin Read the licence CLI command reference for further information.\nObtaining an Enterprise licence Please contact sales@storageos.com to discuss pricing for commercial licences.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/docs/operations/licensing/","title":"Licensing"},{"body":"This procedure aims to replace the current nodes of your Etcd cluster with new ones. Migrating Etcd is a delicate operation, so we recommend a careful assessment of the steps required before starting.\nThe following procedure is scoped for Etcd clusters that are provisioned outside Kubernetes on their own nodes.\n Please note that this procedure involves full downtime of StorageOS and access to its data\n The following procedure is designed for Etcd clusters that are provisioned externally to Kubernetes, with members that are situated on their own nodes.\n Migration To perform Etcd migrations safely, we recommend adding new Etcd nodes to the existing Etcd cluster, before editing the StorageOS custom resource definition and finally deleting the old Etcd nodes one by one. The full procedure is as follows:\n Prepare all the nodes that will host etcd. The Etcd provisioning steps can be found in the etcd section of the StorageOS prerequisites page.\nOn each new node, edit the /etc/etcd.conf file to contain the IP addresses of all old and new nodes. In the /etc/systemd/system/etcd3.service file, ensure that the --initial-cluster argument includes all old and new nodes. Also ensure that the --initial-cluster-state argument is set to existing.\nFor example, adding 3 further nodes to an initially 3 node cluster would involve populating /etc/etcd.conf on each of the fourth, fifth and sixth nodes with the IP addresses of all 6 nodes intended for the cluster, as well as environment variables for the client port and peers port:\n# NODE_IP is the IP of the node where this file resides. NODE_IP=192.168.152.142 # Node 1 IP NODE1_IP=192.168.195.168 # Node 2 IP NODE2_IP=192.168.202.40 # Node 3 IP NODE3_IP=192.168.174.117 # Node 4 IP NODE4_IP=192.168.152.142 # Node 5 IP NODE5_IP=192.168.198.252 # Node 6 IP NODE6_IP=192.168.150.122 CLIENT_PORT=2379 PEERS_PORT=2380 The executable section of /etc/systemd/system/etcd3.service on each of the new nodes should then be edited to reflect these environment variables. For example, the service file for the second new node to be added to an initially 3 node cluster would refer to the three old nodes and two new nodes. Note that in the --initial-cluster variable we do not include members that have not yet been added to the cluster, apart from the member we are currently adding.\n... ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ ... --initial-cluster-state existing \\ ... --initial-cluster etcd-${NODE1_IP}=http://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=http://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=http://${NODE3_IP}:${PEERS_PORT},etcd-${NODE4_IP}=http://${NODE4_IP}:${PEERS_PORT},etcd-${NODE5_IP}=http://${NODE5_IP}:${PEERS_PORT} ... At runtime, the value of initial-cluster will resolve to the following:\n\u0026quot;etcd-192.168.195.168=http://192.168.195.168:2380,etcd-192.168.202.40=http://192.168.202.40:2380,etcd-192.168.174.117=http://192.168.174.117:2380,etcd-192.168.152.1\u0026quot; Connect to an Etcd node that already belongs to the cluster, and add one of the new nodes to the cluster, using the etcdctl member add --learner command per the etcd documentation. The output includes environment variables with the name of the new member and the state and constituents of the cluster now that the new node has been added:\n# Name of the new member of etcd ETCD_NEW_MEMBER=\u0026#34;etcd-192.168.152.142\u0026#34; # Peer url for the new member of etcd, including the port ETCD_NEW_MEMBER_PEER=\u0026#34;http://192.168.152.142:2380\u0026#34; # Client url for the new member of etcd, including the port ETCD_NEW_MEMBER_CLIENT=\u0026#34;http://192.168.195.168:2379\u0026#34; # Add the new member to the cluster ETCDCTL_API=3 etcdctl member add $ETCD_NEW_MEMBER \\ --peer-urls=\u0026#34;$ETCD_NEW_MEMBER_PEER\u0026#34; \\ --endpoints=\u0026#34;$ETCD_NEW_MEMBER_CLIENT\u0026#34; Member 4a5820690b65300c added to cluster 97f788d9ca9b3357 ETCD_NAME=\u0026#34;etcd-192.168.152.142\u0026#34; ETCD_INITIAL_CLUSTER=\u0026#34;etcd-192.168.202.40=http://192.168.202.40:2380,etcd-192.168.174.117=http://192.168.174.117:2380,etcd-192.168.152.142=http://192.168.152.142:2380,etcd-192.168.195.168=http://192.168.195.168:2380\u0026#34; ETCD_INITIAL_ADVERTISE_PEER_URLS=\u0026#34;http://192.168.152.142:2380\u0026#34; ETCD_INITIAL_CLUSTER_STATE=\u0026#34;existing\u0026#34; Start the Etcd service on the new node:\nsystemctl daemon-reload systemctl enable etcd3.service systemctl start etcd3.service List the members in the cluster and confirm that the new node is present and has started, and has therefore successfully been added:\n$ ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:2379 member list aa9dec86cc9c2d1, started, etcd-192.168.202.40, http://192.168.202.40:2380, http://192.168.202.40:2379, false 48823f29326a24f7, started, etcd-192.168.174.117, http://192.168.174.117:2380, http://192.168.174.117:2379, false 4a5820690b65300c, started, etcd-192.168.152.142, http://192.168.152.142:2380, http://192.168.152.142:2379, true gwk09ksjs5a6n862, started, etcd-192.168.152.142, http://192.168.152.142:2380, http://192.168.195.168:2379, false Promote the learner to be a full voting member of the cluster with the etcdctl member promote \u0026lt;member id\u0026gt; command.\n As per the etcd documentation, the learner will fail to be promoted if it is not ready.\n Repeat the previous two steps for all the remaining nodes onto which you wish to migrate etcd. Per the Etcd documentation, when adding more than one member to a cluster, it best practice to configure each member one at a time, verifying that each starts successfully before moving on to the next.\n Delete the StorageOS Custom Resource per the instructions in the uninstall operations page.\n Although deleting the StorageOSCluster Custom Resource will stop StorageOS, the data will be safe. When StorageOS starts again, the Volumes will be available.\n Connect to one of the newly-added nodes, and use etcdctl to remove the old Etcd nodes 1 by 1:\n$ ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:2379 member remove 48823f29326a24f7 Member 48823f29326a24f7 removed from cluster 97f788d9ca9b3357 After removing a node, check that Etcd is healthy and quorum has been maintained with the following commands:\nexport ETCD_ENDPOINTS=\u0026quot;http://192.168.2020.40:2379,http://192.168.174.117.2379,...\u0026quot; etcdctl --endpoints $ETCD_ENDPOINTS endpoint health -w table etcdctl --endpoints $ETCD_ENDPOINTS endpoint status -w table Edit the kvBackend.address parameter of the StorageOS Custom Resource Definition to reflect the locations of the new Etcd nodes, removing the references to the old nodes\n Recreate the StorageOS Custom Resource with the newly-edited kvBackend.address parameter\n Perform a member list again, to confirm that only the new machines are still present in the cluster:\n$ ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:2379 member list 4a5820690b65300c, started, etcd-192.168.152.142, http://192.168.152.142:2380, http://192.168.152.142:2379, false b1780933f495adb3, started, etcd-192.168.150.122, http://192.168.150.122:2380, http://192.168.150.122:2379, false ee47ff85984afcc0, started, etcd-192.168.198.252, http://192.168.198.252:2380, http://192.168.198.252:2379, false At this stage Etcd has been successfully migrated.\n Etcd Learners As of v3.4, Etcd includes the functionality to add a new member as a \u0026ldquo;learner\u0026rdquo; (a non-voting member of the cluster). This makes the process of adding new members safer, as it allows the new member to complete the process of synchronisation before participating in quorum, minimising cluster downtime.\nIf you are using an earlier version of etcd, make the following substitutions to the above process:\n In step 2, replace the etcdctl member add --learner command with etcdctl member add Skip theetcdctl member promote \u0026lt;member id\u0026gt; command Using Etcd learners provides added resilience, so consider upgrading to a version of Etcd that supports this as soon as possible.\n","excerpt":"This procedure aims to replace the current nodes of your Etcd cluster with new ones. Migrating Etcd …","ref":"/docs/operations/etcd/etcd-migration/","title":"Migrate Etcd"},{"body":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on linux.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MS SQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1...spec:serviceAccountName:mssql...volumeMounts:- name:datamountPath:/var/opt/mssql...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MS SQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mssql Confirm MS SQL is up and running.\n$ kubectl get pods -w -l app=mssql NAME READY STATUS RESTARTS AGE mssql-0 1/1 Running 0 1m Connect to the MS SQL client pod and connect to the MS SQL server through the service\n$ kubectl exec -it mssql-0 -- /opt/mssql-tools/bin/sqlcmd -S mssql-0.mssql -U SA -P \u0026#39;Password15\u0026#39; 1\u0026gt; USE master; 2\u0026gt; GO Changed database context to \u0026#39;master\u0026#39;. 1\u0026gt; SELECT name, database_id, create_date FROM sys.databases ; 2\u0026gt; GO name database_id create_date --------------------------- ----------- ----------------------- master 1 2003-04-08 09:13:36.390 tempdb 2 2018-11-02 16:30:37.907 model 3 2003-04-08 09:13:36.390 msdb 4 2018-10-19 01:18:57.300 (4 rows affected) Configuration If you need custom startup options, you can edit the ConfigMap file 15-mssql-configmap.yaml with your desired MS SQL configuration settings.\n","excerpt":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on …","ref":"/docs/usecases/mssql/","title":""},{"body":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With mTLS both StorageOS and etcd authenticate each other ensuring that communication only happens between mutually authenticated end points, and that all communication is encrypted.\nStorageOS uses the certificates and keys from a Secret to cypher and authenticate Etcd traffic.\nHow to create the certificates Secret The client auth certificates need the following file names, in the Secret.\n etcd-client-ca.crt - containing the etcd Certificate Authority certificate etcd-client.crt - containing the etcd Client certificate etcd-client.key - cotaining the etcd Client key kubectl create secret -n storageos-etcd generic \\ etcd-client-tls \\ --from-file=\u0026#34;etcd-client-ca.crt\u0026#34; \\ --from-file=\u0026#34;etcd-client.crt\u0026#34; \\ --from-file=\u0026#34;etcd-client.key\u0026#34; How to use the mTLS certificates Secret with StorageOS Below is an example StorageOSCluster resource that can be used to setup StorageOS with etcd using mTLS.\napiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:storageos-clusternamespace:\u0026#34;storageos-operator\u0026#34;spec:# StorageOS Pods are in kube-system by defaultsecretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;default\u0026#34;images:nodeContainer:\u0026#34;storageos/node:v2.3.2\u0026#34;namespace:\u0026#34;storageos\u0026#34;# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;etcd\u0026#34;# Namespace of the client certificates secretkvBackend:address:\u0026#34;https://storageos-etcd-cluster-client.storagos-etcd.svc:2379\u0026#34;# Etcd client service address.backend:\u0026#34;etcd\u0026#34;# Backend typetlsEtcdSecretRefName and tlsEtcdSecretRefNamespace are used to pass a reference to the Secret.\nThe StorageOS operator uses the etcd secret that contains the client certificates, to build a secret in the StorageOS installation namespace. This secret contains the certificate filenames and certificate file contents. The StorageOS daemonset that is created by the operator mounts the secret as a volume so that the certificate files are available inside the pod. Environment variables containing the file paths are passed to the StorageOS process in order to use the files from the mounted path.\nA worked example of setting up StorageOS with external etcd using mTLS is avaliable here. For ease of use the example uses the CoreOS etcd operator and the CoreOS guide The example uses the CoreOS etcd operator and follows the CoreOS guide for Cluster TLS.\n","excerpt":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With …","ref":"/docs/operations/etcd/storageos-secret-info/","title":"Encrypting communication with Etcd"},{"body":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular web-based applications including WordPress.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MySQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiversion:apps/v1kind:statefulsetmetadata:name:mysqlspec:selector:matchlabels:app:mysqlenv:prodservicename:mysqlreplicas:1...spec:serviceaccountname:mysql...volumemounts:- name:datamountpath:/var/lib/mysqlsubpath:mysql- name:confmountpath:/etc/mysql/mysql.conf.d...volumeclaimtemplates:- metadata:name:datalabels:env:prodspec:accessmodes:[\u0026#34;readwriteonce\u0026#34;]storageclassname:\u0026#34;fast\u0026#34;# storageos storageclass resources:requests:storage:5giThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MySQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mysql Confirm MySQL is up and running.\n$ kubectl get pods -w -l app=mysql NAME READY STATUS RESTARTS AGE mysql-0 1/1 Running 0 1m Connect to the MySQL client pod and connect to the MySQL server through the service\n$ kubectl exec client -- mysql -h mysql-0.mysql -e \u0026#34;show databases;\u0026#34; Database information_schema mysql performance_schema Configuration If you need custom startup options, you can edit the ConfigMap file 15-mysqld-configmap.yaml with your desired MySQL configuration settings.\n","excerpt":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular …","ref":"/docs/usecases/mysql/","title":""},{"body":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a StorageOS cluster to be used by multiple teams across multiple projects.\nIt is not necessary to create StorageOS namespaces manually, as StorageOS maps Kubernetes namespaces on a one-to-one basis when PersistentVolumeClaims using the StorageOS StorageClass are created.\nAccess to Namespaces is controlled through user or group level policies.\n","excerpt":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a …","ref":"/docs/concepts/namespaces/","title":"Namespaces"},{"body":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is created by default.\nNamespaces apply to volumes.\nManaging Namespaces In order to create a new namespace navigate to \u0026ldquo;Namespaces\u0026rdquo; in the GUI, and select \u0026ldquo;Create Namespace\u0026rdquo;.\nWhen a Kubernetes PVC is created in a namespace, StorageOS automatically maps the Volume in the same namespace. Namespaces are created by StorageOS to fulfil the RBAC rules enforced by Kubernetes roles.\nIn order to delete a namespace, all volumes must be deleted from the namespace before the namespace can be deleted.\n","excerpt":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is …","ref":"/docs/operations/namespaces/","title":"Namespaces"},{"body":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load balancer or even as a Kubernetes ingress controller.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Nginx on Kubernetes You can find the latest files in the StorageOS use cases repostiory\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet definition\napiVersion:apps/v1kind:StatefulSetmetadata:name:nginxspec:serviceName:nginxspec:serviceAccountName:nginxcontainers:- name:nginximage:nginxports:- containerPort:80volumeMounts:- name:nginx-datamountPath:/usr/share/nginx/htmlsubPath:html- name:nginx-configmountPath:/etc/nginx/conf.dvolumes:- name:nginx-configconfigMap:name:nginxvolumeClaimTemplates:- metadata:name:nginx-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Nginx examples folder and create the objects\ncd storageos-usecases kubectl create -f ./nginx Confirm Nginx is up and running.\n$ kubectl get pods -w -l app=nginx NAME READY STATUS RESTARTS AGE nginx-0 1/1 Running 0 1m Connect to the nginx pod and write a file to /usr/share/nginx/html that Nginx will serve.\n$ kubectl exec nginx-0 -it -- bash root@nginx-0:/# echo Hello world! \u0026gt; /usr/share/nginx/html/greetings.txt Connect to the Busybox pod and connect to the Nginx server through the service and retrieve the directory index from Nginx.\n$ kubectl exec -it busybox -- /bin/sh / # wget -q -O- nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;greetings.txt\u0026#34;\u0026gt;greetings.txt\u0026lt;/a\u0026gt; 27-Feb-2019 12:04 13 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; Retrieve and display the contents of the greetings.txt file\n/ # wget -q -O- nginx/greetings.txt Hello world! Configuration If you need custom startup options, you can edit the ConfigMap file 15-nginx-configmap.yaml with your desired Nginx configuration settings.\n","excerpt":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load …","ref":"/docs/usecases/nginx/","title":""},{"body":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A node must be running a daemonset pod in order to consume and/or present storage.\nBy default StorageOS nodes run in hyperconverged mode. This means that the node hosts data from StorageOS volumes and can present volumes to applications.\nAlternatively, a node can run in computeonly mode, which means no storage is consumed on the node itself and the node only presents volumes hosted by other nodes. Volumes presented to applications running on compute only nodes are therefore all remote. Compute only nodes can be very useful for topologies where nodes are ephemeral and should not host data, but the ephemeral nodes host applications that require StorageOS volumes. The nodes that are not intended to hold data, but just to present StorageOS volumes, can be set as computeonly.\nA node can be marked as compute only at any point in time by adding the label storageos.com/computeonly=true, following the labels reference.\n","excerpt":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A …","ref":"/docs/concepts/nodes/","title":"Nodes"},{"body":"The following document lists the open source software attributions in the StorageOS Control Plane, Data Plane and CLI.\nCNats The MIT License (MIT) Copyright (c) 2015 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CppUTest Copyright (c) 2007, Michael Feathers, James Grenning and Bas Vodde All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the \u0026lt;organization\u0026gt; nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE EARLIER MENTIONED AUTHORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL \u0026lt;copyright holder\u0026gt; BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Hayai Copyright (c) 2011 - Nick Bruun. This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. If you meet (any of) the author(s), you're encouraged to buy them a beer, a drink or whatever is suited to the situation, given that you like the software. This notice may not be removed or altered from any source distribution. LZ4 LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LibreSSL LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- ==================================================================== Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit. (http://www.openssl.org/)\u0026quot; 4. The names \u0026quot;OpenSSL Toolkit\u0026quot; and \u0026quot;OpenSSL Project\u0026quot; must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact openssl-core@openssl.org. 5. Products derived from this software may not be called \u0026quot;OpenSSL\u0026quot; nor may \u0026quot;OpenSSL\u0026quot; appear in their names without prior written permission of the OpenSSL Project. 6. Redistributions of any form whatsoever must retain the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)\u0026quot; THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==================================================================== This product includes cryptographic software written by Eric Young (eay@cryptsoft.com). This product includes software written by Tim Hudson (tjh@cryptsoft.com). Original SSLeay License ----------------------- Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) All rights reserved. This package is an SSL implementation written by Eric Young (eay@cryptsoft.com). The implementation was written so as to conform with Netscapes SSL. This library is free for commercial and non-commercial use as long as the following conditions are aheared to. The following conditions apply to all code found in this distribution, be it the RC4, RSA, lhash, DES, etc., code; not just the SSL code. The SSL documentation included with this distribution is covered by the same copyright terms except that the holder is Tim Hudson (tjh@cryptsoft.com). Copyright remains Eric Young's, and as such any Copyright notices in the code are not to be removed. If this package is used in a product, Eric Young should be given attribution as the author of the parts of the library used. This can be in the form of a textual message at program startup or in documentation (online or textual) provided with the package. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: \u0026quot;This product includes cryptographic software written by Eric Young (eay@cryptsoft.com)\u0026quot; The word 'cryptographic' can be left out if the rouines from the library being used are not cryptographic related :-). 4. If you include any Windows specific code (or a derivative thereof) from the apps directory (application code) you must include an acknowledgement: \u0026quot;This product includes software written by Tim Hudson (tjh@cryptsoft.com)\u0026quot; THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The licence and distribution terms for any publically available version or derivative of this code cannot be changed. i.e. this code cannot simply be copied and put under another distribution licence [including the GNU Public Licence.] protobuf This license applies to all parts of Protocol Buffers except the following: - Atomicops support for generic gcc, located in src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Code generated by the Protocol Buffer compiler is owned by the owner of the input file used when generating it. This code is not standalone and requires a support library to be linked with it. This support library is itself covered by the above license. rocksdb BSD License For rocksdb software Copyright (c) 2011-present, Facebook, Inc. All rights reserved. --------------------------------------------------------------------- Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/armon/go-metrics The MIT License (MIT) Copyright (c) 2013 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/armon/go-radix The MIT License (MIT) Copyright (c) 2014 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/bgentry/speakeasy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \u0026quot;[]\u0026quot; replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \u0026quot;printed page\u0026quot; as the copyright notice for easier identification within third-party archives. Copyright [2013] [the CloudFoundry Authors] Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/dgrijalva/jwt-go Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgrijalva/jwt-go/request Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgryski/go-metro MetroHash This package is a mechanical translation of the reference C++ code for MetroHash, available at https://github.com/jandrewrogers/MetroHash I claim no additional copyright over the original implementation. The MIT License (MIT) Copyright (c) 2015 J. Andrew Rogers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/docker/docker/pkg/mount Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv/store Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/golang/protobuf/proto Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/context Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/handlers Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/mux Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/websocket Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/consul/api Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/errwrap Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-cleanhttp Mozilla Public License, version 2.0 1. Definitions 1.1. \u0026quot;Contributor\u0026quot; means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. \u0026quot;Contributor Version\u0026quot; means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. \u0026quot;Contribution\u0026quot; means Covered Software of a particular Contributor. 1.4. \u0026quot;Covered Software\u0026quot; means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. \u0026quot;Incompatible With Secondary Licenses\u0026quot; means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. \u0026quot;Executable Form\u0026quot; means any form of the work other than Source Code Form. 1.7. \u0026quot;Larger Work\u0026quot; means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. \u0026quot;License\u0026quot; means this document. 1.9. \u0026quot;Licensable\u0026quot; means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. \u0026quot;Modifications\u0026quot; means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. \u0026quot;Patent Claims\u0026quot; of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. \u0026quot;Secondary License\u0026quot; means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. \u0026quot;Source Code Form\u0026quot; means the form of the work preferred for making modifications. 1.14. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) means an individual or a legal entity exercising rights under this License. For legal entities, \u0026quot;You\u0026quot; includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, \u0026quot;control\u0026quot; means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an \u0026quot;as is\u0026quot; basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - \u0026quot;Incompatible With Secondary Licenses\u0026quot; Notice This Source Code Form is \u0026quot;Incompatible With Secondary Licenses\u0026quot;, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-msgpack/codec Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/go-multierror Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/memberlist Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/coordinate Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/serf Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/kelseyhightower/envconfig Copyright (c) 2013 Kelsey Hightower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/mattn/go-isatty Copyright (c) Yasuhiro MATSUMOTO \u0026lt;mattn.jp@gmail.com\u0026gt; MIT License (Expat) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/miekg/dns Extensions of the original work are copyright (c) 2011 Miek Gieben As this is fork of the official Go code the same license applies: Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/mitchellh/cli Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/conf The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/logger The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server/pse The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/util The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats/encoders/builtin The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nuid The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/pborman/uuid Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/Sirupsen/logrus The MIT License (MIT) Copyright (c) 2014 Simon Eskildsen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/thejerf/suture Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/urfave/negroni The MIT License (MIT) Copyright (c) 2014 Jeremy Saenz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang.org/x/crypto/bcrypt Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/crypto/blowfish Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/net/context Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/unix Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/registry Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/svc/eventlog Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/time/rate Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. gopkg.in/yaml.v2 Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ","excerpt":"The following document lists the open source software attributions in the StorageOS Control Plane, …","ref":"/docs/reference/open_source_attribution/","title":"StorageOS Open Source Software Attribution Notice"},{"body":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform actions such as creation, deletion and mounting of volumes through CSI. The CSI container running in the StorageOS Daemonset creates a Linux socket that allows the communication between OpenShift and StorageOS\nInstallation StorageOS v2 supports OpenShift 4.0, 4.1, 4.2, 4.3, 4.4 and 4.5.\nTo install StorageOS on OpenShift, please follow our installation instructions page.\n N.B. Openshift 4 uses the CRI-O container runtime that sets a default PID limit of 1024. StorageOS recommends that the limit be raised to 32768. Please see our prerequisites for more details.\n OpenShift Upgrades OpenShift provides an upgrade operator that automates the process of orchestrator version changes.\nThis procedure can cause StorageOS to malfunction due to sequential node restarts not taking the presence of stateful application data into consideration. To avoid this issue, make sure all stateful workloads using StorageOS Volumes are stopped - usually by scaling StatefulSets to 0. Please contact StorageOS support for further advice if required.\n OpenShift requires the internal registry to be available during the upgrade, however StorageOS volumes may not be available. Therefore using StorageOS for the internal registry is not recommended.\n CSI (Container Storage Interface) Note CSI is the standard that enables storage drivers to release on their own schedule. This allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles. StorageOS v2 uses CSI to implement communication with the OpenShift controlplane.\nStorageOS PersistentVolumeClaims The user can provide standard PVC definitions and StorageOS will dynamically provision them. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\n","excerpt":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform …","ref":"/docs/platforms/openshift/","title":"OpenShift"},{"body":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a user or group access to a namespace, a policy needs to be created mapping the user or group to the namespace.\n Note: Users always have access to the default namespace\n For more information on how to use policies, see the Policies operations page.\n","excerpt":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a …","ref":"/docs/concepts/policies/","title":"Policies"},{"body":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user level so access can be controlled granularly.\nUsers can belong to one or more groups to control their namespace permissions. Additionally user specific policies can be created to grant a user access to a namespace. Users can belong to any number of groups and have any number of user level policies configured.\n Note: Users are created with access to the default namespace. Policies cannot be applied to the default namespace.\n Managing Policies To start creating policies, at least one custom namespace and user are required. For more information on how to create namespaces see our Namespace guide, for users see our Users CLI reference.\nIn order to create a policy navigate to \u0026ldquo;Policies\u0026rdquo; in the GUI and select \u0026ldquo;Create Policy\u0026rdquo;. A policy controls access to a variety of StorageOS resources and is applied to a user, by placing the user in the policies group.\nIn order to delete a policy, all users must be removed from the policy group before deletion of the policy can be completed.\n","excerpt":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user …","ref":"/docs/operations/policies/","title":"Policies"},{"body":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational database management system (ORDBMS).\nPostgres is deployed across a wide variety of platforms with a mix of workloads ranging from small, single-node use cases to large Internet-facing clusters with many concurrent users.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. If you need to setup StorageOS on Kubernetes then please see our guide to [installing StorageOS on Kubernetes]({% link \u0026ldquo;docs/install/kubernetes.md\u0026rdquo; \u0026gt;}}).\nDeploying PostgreSQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases PersistentVolumeClaim and Pod definition excerpts\nkind:PersistentVolumeClaimmetadata:name:pg-dataannotations:volume.beta.kubernetes.io/storage-class:fast...kind:Podmetadata:name:postgressspec:securityContext:fsGroup:26containers:- name:pgimage:crunchydata/crunchy-postgres:centos7-10.4-1.8.3volumeMounts:- mountPath:/pgdataname:data...volumes:- name:datapersistentVolumeClaim:claimName:pg-dataThis excerpt is from the PersistentVolumeClaim and Pod definition. The pod definition references the pg-data VolumeClaim so storage is dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the PostgreSQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./postgres Confirm PostgreSQL is up and running.\n$ kubectl get pod postgres-0 -w NAME READY STATUS RESTARTS AGE postgres-0 1/1 Running 0 1m Connect to the PostgreSQL client pod and connect to the PostgreSQL server through the service.\n$ kubectl exec -it postgres-0 -- psql -h postgres-0.postgres -U primaryuser postgres -c \u0026#34;\\l\u0026#34; Password for user primaryuser: password List of Databases Name | Owner | Encoding | Collate | Ctype | Access privileges +=========================================================================+ postgres | postgres | SQL_ASCII | C | C | template0 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres template1 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres userdb | postgres | SQL_ASCII | C | C | =Tc/postgres + | | | | | postgres=CTc/postgres+ | | | | | testuser=CTc/postgres (4 rows) The password for the primary user is password. You can see this is set in the ConfigMap file.\n Configuration If you need custom startup options, you can edit the ConfigMap file 15-postgresd-configmap.yaml with your desired PostgreSQL configuration settings.\n","excerpt":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational …","ref":"/docs/usecases/postgres/","title":""},{"body":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting in Kubernetes.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Prometheus on Kubernetes This is the Prometheus use case for StorageOS. Following are the steps for creating a Prometheus StatefulSet and using StorageOS to provide persistent storage.\n You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases Prometheus Custom Resource definition\napiVersion:monitoring.coreos.com/v1kind:Prometheusmetadata:name:prometheus-storageoslabels:app:prometheus-operatorspec:...storage:volumeClaimTemplate:metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:storageos-replicatedresources:requests:storage:1GiThis excerpt is from the Prometheus Custom Resource definition. This file contains the VolumeClaimTemplate that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs due to the VolumeClaimTemplate in the Prometheus StatefulSet. The Prometheus StatefulSet is created by the Prometheus Operator, triggered by the creation of the Prometheus resource.\n Move into the Prometheus examples folder and install the Prometheus Operator and create a Prometheus resource.\n$ cd storageos-usecases/prometheus $ ./install-prometheus.sh Confirm Prometheus is up and running.\n$ kubectl get pods -w -l app=prometheus NAME READY STATUS RESTARTS AGE prometheus-prometheus-storageos-0 3/3 READY 0 1m You can see the created PVC using.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-prometheus-prometheus-storageos-0 Bound pvc-b6c17c0a-e76b-4a0b-8fc6-46c0e1629210 1Gi RWO storageos-replicated 65m In the Prometheus deployment script, a service monitor is created. A Service Monitor is a special object that the Prometheus operator uses to create configuration for Endpoints for Prometheus to scrape. Although the name implies that a Service Monitor defines Kubernetes services that will be scraped, Prometheus actually targets the Endpoints that the services point to. The new Prometheus instance will use the storageos-etcd service monitor to start scraping metrics from the storageos-etcd pods. Assuming the storageos cluster was setup using ETCD as pods. For more information about service monitors, have a look at the upstream documentation.\n$ kubectl get servicemonitor NAME AGE storageos-etcd 5d1h The Prometheus web ui can be accessed by port-forwarding the Prometheus pods port to localhost.\n$ kubectl port-forward prometheus-prometheus-storageos-0 9090 Then launch a web browser and go to localhost:9090 to access the Prometheus web ui. You can confirm that the storageos-etcd target is configured there.\n Configuration In the storageos-usecases/prometheus/manifests/prometheus directory there are other example Service Monitors. For more information about Prometheus, check out the prometheus documentation.\n","excerpt":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and …","ref":"/docs/usecases/prometheus/","title":""},{"body":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS transparently supports Rancher deployments using CentOS, RHEL or Debian and can support other Linux distributions detailed in the systems supported page if the appropriate kernel modules are present.\n","excerpt":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS …","ref":"/docs/platforms/rancher/","title":"Rancher"},{"body":" Please note: StorageOS Project edition is required to create RWX Volumes.\n StorageOS supports ReadWriteMany (RWX) access mode Persistent Volumes. A RWX PVC can be used simultaneously by many Pods in the same Kubernetes namespace for read and write operations.\nStorageOS RWX Volumes are based on a shared filesystem - in the case of our implementation, this is NFS.\nArchitecture For each RWX Volume, the following components are involved:\nStorageOS ReadWriteOnly (RWO) Volume\nStorageOS provisions a standard Volume that provides a block device for the file system of the NFS server. This means that every RWX Volume has its own RWO Volume. This allows RWX Volumes to leverage the synchronous replication and automatic failover functionality of StorageOS, providing the NFS server with high availability.\nNFS-Ganesha server\nFor each RWX Volume, an NFS-Ganesha server is spawned by StorageOS. The NFS server runs in user space on the Node containing the primary Volume. Each NFS server uses its own RWO Volume to store data so the data of each Volume is isolated.\nStorageOS binds an ephemeral port to the host network interface for each NFS-Ganesha server. The NFS export is presented using NFS v4.2. Check the prerequisites page to see the range of ports needed for StorageOS RWX Volumes.\nStorageOS API Manager\nStorageOS fully integrates with Kubernetes. The StorageOS API Manager Pod monitors StorageOS RWX Volumes to create and maintain a Kubernetes Service that points towards each RWX Volume\u0026rsquo;s NFS export endpoint. The API Manager is responsible for updating the Service endpoint when a RWX Volume failover occurs.\nProvisioning and using RWX PVCs The sequence in which a RWX PVC is provisioned and used is as follows:\n A PersistentVolumeClaim (PVC) is created with RWX access mode using any StorageOS StorageClass. StorageOS dynamically provisions the PV. A new StorageOS RWO Volume is provisioned internally (not visible in Kubernetes). When the RWX PVC is consumed by a pod, an NFS-Ganesha server is instantiated on the same Node as the primary Volume. The NFS-Ganesha server thus uses the RWO StorageOS Volume as its back end disk. The StorageOS API Manager publishes the host IP and port for the NFS service endpoint, by creating a Kubernetes Service that points to the NFS-Ganesha server export endpoint. StorageOS issues a NFS mount on the Node where the Pod using the PVC is scheduled. High availability RWX Volumes failover in the same way as standard RWO StorageOS Volumes. The replica Volume is promoted upon detection of Node failure and the NFS-Ganesha server is started on the Node containing the promoted replica. The StorageOS API Manager updates the endpoint of the Volume\u0026rsquo;s NFS service, causing traffic to be routed to the URL of the new NFS-Ganesha server. The NFS client in the application Node (where the user\u0026rsquo;s Pod is running) automatically reconnects.\nNotes All feature labels that work on RWO Volumes will also work on RWX Volumes. A StorageOS RWX Volume is matched one-to-one with a PVC. Therefore the StorageOS RWX Volume can only be accessed by Pods in the same Kubernetes namespace. StorageOS RWX Volumes support volume resize. Refer to the resize documentation for more details. ","excerpt":"Please note: StorageOS Project edition is required to create RWX Volumes.\n StorageOS supports …","ref":"/docs/concepts/rwx/","title":"ReadWriteMany"},{"body":" Please note: StorageOS Project edition is required to create RWX Volumes.\n StorageOS supports ReadWriteMany (RWX) access mode Persistent Volumes. A RWX PVC can be used simultaneously by many Pods in the same Kubernetes namespace for read and write operations.\nStorageOS RWX Volumes are based on a shared filesystem.\nTo create a ReadWriteMany (RWX) volume with StorageOS, create a Persistent Volume Claim (PVC) with an access mode of ReadWriteMany (see the First PVC documentation for examples of creating standard PVCs with StorageOS).\nThe following YAML manifest files provide an example:\nA 5Gi PVC with the StorageOS storageClassName of fast, with an accessMode of ReadWriteMany:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc-rwxspec:storageClassName:fastaccessModes:- ReadWriteManyresources:requests:storage:5GiA Deployment of 3 Pods, each of which consume this PVC:\napiVersion:apps/v1kind:Deploymentmetadata:name:sharedlabels:app:sharedspec:replicas:3selector:matchLabels:app:sharedtemplate:metadata:labels:app:sharedspec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mnt/name:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-rwxAfter creating the above resources, the PVC (here named pvc-rwx) should be bound and show an access mode of RWX:\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-rwx Bound pvc-59f7a152-8342-415c-a6ca-1cbb463410ab 5Gi RWX fast 60s Ensure that the Deployment Pods are running:\n$ kubectl get pods NAME READY STATUS RESTARTS AGE shared-1771418926-7o5ns 1/1 Running 0 1m shared-1771418926-r18az 1/1 Running 0 1m shared-1771418926-ds8f7 1/1 Running 0 1m The NFS-Ganesha service that exposes the StorageOS volume as RWX can now be viewed, showing the cluster IP of the service, and the default NFS port (2049).\n$ kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.96.0.1 \u0026lt;none\u0026gt; 443/TCP 120m pvc-59f7a152-8342-415c-a6ca-1cbb463410ab ClusterIP 10.107.10.0 \u0026lt;none\u0026gt; 2049/TCP 1m Features of the NFS volume can also be examined in the StorageOS UI. Under Volumes, the Attachment column shows an nfs tag. The Volume Details section provides information about the NFS Volume, such as the service endpoint and the node on which the underlying Volume is attached.\n","excerpt":"Please note: StorageOS Project edition is required to create RWX Volumes.\n StorageOS supports …","ref":"/docs/operations/rwx/","title":"ReadWriteMany"},{"body":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional durability to disk.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Redis on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:redisspec:selector:matchLabels:app:redisenv:prodserviceName:redisreplicas:1...spec:serviceAccountName:redis...volumeMounts:- name:datamountPath:/bitnami/redis/data...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the Redis examples folder and create the objects\ncd storageos-usecases kubectl create -f ./redis Confirm Redis is up and running.\n$ kubectl get pods -w -l app=redis NAME READY STATUS RESTARTS AGE redis-0 1/1 Running 0 1m Connect to the Redis client pod and connect to the Redis server through the service\n$ kubectl exec -it redis-0 -- redis-cli -a password Warning: Using a password with \u0026#39;-a\u0026#39; option on the command line interface may not be safe. 127.0.0.1:6379\u0026gt; CONFIG GET maxmemory 1) \u0026#34;maxmemory\u0026#34; 2) \u0026#34;0\u0026#34; Configuration If you need custom startup options, you can edit the ConfigMap file 15-redis-configmap.yaml with your desired Redis configuration settings.\n","excerpt":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional …","ref":"/docs/usecases/redis/","title":""},{"body":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than \u0026ldquo;latest\u0026rdquo;, and to perform upgrades only after reading the release notes.\nThe latest tagged release is v2.3.2. For installation instructions please see our Install page.\nThe latest CLI release is v2.3.0, available from Github.\nUpgrading To upgrade from version 1.x to 2.x, please contact support for assistance.\nv2.3.2 - Released 2020-11-25 Fixed (controlplane/rejoin) Failure to delete data causes re-advertise loop. (controlplane/rejoin) Handle timeout waiting for progress report. (dataplane/log) Change buffering of symmetra output to prevent stalls. v2.3.1 - Released 2020-11-16 Allows access to ReadWriteMany shared volumes when running containers as a non-root user. Fixed nfs: root squash to uid=0 is now configured on all shared volumes. v2.3.0 - Released 2020-10-31 This release adds production-grade shared file support to v2, previously a technology preview in v1.\nBreaking The v2.3.0 operator is no longer able to run StorageOS v1. New Adds support for ReadWriteMany shared volumes. See ReadWriteMany. Adds api-manager deployment to support shared volumes. See the api manager Github repository for more information. Kubernetes 1.19 support. Improved dataplane: Reduce replication thread usage by having the replication processes share the main thread pool. This helps ensure that there isn\u0026rsquo;t a spike in thread usage when a node recovers and begins re-syncing its volumes. This is particularly relevant on CRIO-based orchestrators such as Openshift where the default maximum allowed PID limit (which also governs the thread limit) is low. dataplane: Detect and log the effective maximum PID limit on startup. dataplane: Internal device presentation mappings are now ephemeral and are not persisted across reboots. dataplane: Disabled default verbose logging for fdatasync/flushWAL timers. dataplane: Log both volume inode and UUID in replication error messages for easier correlation. dataplane: On startup, ensure any remnant devices that may have been left after an unclean shutdown have been properly cleared. dataplane: Signal when all startup tasks complete. This ensures no IO can be initiated before this time. ha: Implement a backoff when attempting to repoint an attached volume after the master has failed. ha: Replicas can now rejoin after an asymmetric partition. This can occur when the master has not lost communication to the replica, but the replica can\u0026rsquo;t communicate with the master. Previously the replica would not be able to rejoin until the master determined it had failed. ha: A master that was partitioned can now re-join to the new master as a replica. api: node label changes update target node prior to committing new state. api: Validation errors now include more information on the failure and how to resolve. csi: Volume resize error messages (e.g. capacity exceeded) now passed through in CSI response. csi: Volume attachment is now verified prior to mount for more instructive error message. csi: Returns RESOURCE EXHAUSTED error when attempting to exceed maximum of 250 StorageOS volume attachments per node. diagnostics: Multiple improvements to bundle collection and collected data. ui: Allow collection of partial diagnostics bundles. ui: Tolerate clock skew when authenticating via the UI. licensing: Read-through cache added. Licence updates will take up to 60s to propagate to all nodes. cli: Set replicas output formatting. init: Checks the effective maximum PID limit and warns if less than the StorageOS recommended PID limit (32,768). CRIO-based distributions such as Openshift have a much lower default value (1024). Consult prerequisites for more information. Fixed dataplane: Fixes transport endpoint is not connected on startup after an unclean shutdown. csi: Volume unmount requests now succeed when the mountpoint has already been removed by the orchestrator. csi: Volume detach requests now succeed when the volume has already been deleted. Previously the volume would be stuck in Terminating status. v2.2.0 - Released 2020-08-18 This release focuses on performance. We analysed existing performance characteristics across a variety of real-world use cases and ended up with improvements across the board. Of particular note:\n Sequential reads have improved by up to 130% Sequential writes have improved by up to 737% Random reads have improved by up to 45% Random writes have improved by up to 135% I/O for large block sizes (128K) has improved by up to 353% We are extremely proud of our performance and we love to talk about it. Have a look at the Benchmarking section of the self-evaluation guide and consider sharing your results. Our PRE engineers are available to discuss in our slack channel.\nNew Data engine revamp focused on provable consistency and performance. Key characteristics:\n Metadata is stored in an optimised index, lowering I/O latency and improving performance for all workloads. Large block reads/writes are now be handled in a single operation. Applications like Kafka will go much faster. On-disk compression is now disabled by default as in most scenarios this offers better performance. To enable on-disk compression for a specific workload, see compression.\n Improved dataplane: The number of I/O threads are now determined by the number of processing cores available. This improves scalability and performance on larger servers. ha: Improve partition tolerance behaviour when a volume master that has lost its connection to etcd rejoins. ha: Allow replicas in unhealthy states to be remediated and re-used while maintaining partition tolerance. ha: When a master fails and the new master is not yet available, introduce a back-off to the redirection logic to avoid spamming the logs with connection failure errors. ha: Ignore health advertisements for local node. Local nodes are handled directly. node delete: Only refuse to delete a node if the node health can be authoritatively verified to be in use. api: Increase HTTP server write timeout. cli/ui: Allow partial diagnostic bundle downloads. ui: Namespace dropdown can now be scrolled. ui: Add \u0026ldquo;Job title\u0026rdquo; to UI licence form. logging: Log version at startup at INFO level. logging: Lower verbosity of SCSI warnings that do not apply to StorageOS. diagnostics: Include logs that have been rotated. diagnostics: Bundle collection across providers is now done in parallel. build: Update base image to RHEL 8.2. operator: Removed DB migration utility required for v1.3 -\u0026gt; v1.4 upgrades. operator: Automatically refreshes StorageOS API token without failing requests when the token expires. operator: Updated CSI attacher and provisioner to latest upstream. operator: Remove cluster.local suffix on Pod scheduler service address. This allows the scheduler to work in clusters with custom DNS configuration. operator: Defaults are now set for most CSI configuration options in the StorageOSCluster custom resource. Fixed csi: When unmount request is received for a volume that has already been unmounted, return success. csi: Verify volume is attached on the node before mounting it. xfs: Support older RHEL kernels which have an XFS library that does not allow reflinks/dedupe. dataplane: Reserve 1GiB of capacity on the target disk to allow manual cleanup operations, rather than filling target disk to capacity. operator: In some cases /var/lib/storageos could fail to unmount cleanly after a restart. This resulted in multiple entries in /proc/mounts. v2.1.0 - Released 2020-06-26 New csi: Volume expansion now supported in offline mode. To expand a volume, stop any workloads accessing the volume, then edit the PVC to increase the capacity. For more information, see our Volume Resize operations page and the CSI Volume Expansion page. api: Volume configuration including replica count can now be updated while the volume is in use. Other updateable fields include labels and description. failover: Before determining that a node is offline and performing recovery operations, the I/O path is also verified. This provides more robust failure detection and ensures that nodes that are still responding to I/O do not get replaced. This I/O path verification is in addition to the gossip-based failure detection. operator: Default tolerations are now set for the StorageOS node container. This helps ensure that the StorageOS node container does not get evicted when the node is running low on resources. Improved api: Added checks to prevent deletion of a node with active volumes, or if it is the master of at least one volume. This helps prevent orphaned volumes. cli: Add an --offline-delete flag to allow removal of volumes whose master and replica nodes are offline. This allows cleanup of orphaned volumes. ui: Add an offline volume delete option. ui: Volumes can now be detached from the UI. cli: Labels are no longer truncated. api: When a new node is added to the cluster, its capacity is available to use immediately. Fixed ui: Favicon was missing. ui: Duplicate volumes could be shown on the node details page. operator: During uninstall a ClusterRoleBinding was not removed. v2.0.0 - Released 2020-05-05 New operator: StorageOS containers now run in the kube-system namespace by default to allow the system-node-critical priority class to be set. This instructs Kubernetes to start StorageOS before application Pods, and to evict StorageOS only after application Pods have finished. This setting was previously recommended in documentation; it is now the default. operator: StorageOS CSI helper containers now run as privileged. This ensures that the CSI endpoint can be seen on systems with SELinux enabled. ui: replication progress for new or re-joining replicas is now displayed. ui: show warning for unlicensed clusters. cli: new commands: licence management get policy create namespace create policy describe user describe namespace describe policy delete user delete namespace delete policy licence: removed the default licence expiry date added for v2.0.0-rc.1. Improved dataplane: improved retry behaviour for network I/O. cli: \u0026ldquo;get volumes\u0026rdquo; for all namespaces should be done in parallel. cli: help text document config file ui: link node name and get to node details on the volume details page. ui: node details add available capacity spinner. ui: node list remove capacity values / address port. ui: node list show master/replica counts. ui: node list remove edit action. ui: format entity labels. ui: node details link volumes. ui: align buttons for licences. ui: k8s warning in \u0026ldquo;create volume\u0026rdquo; modal. ui: node list remove \u0026ldquo;API\u0026rdquo; from \u0026ldquo;API Address\u0026rdquo; ui: add some details about the Licence on the licence page. api: include valid for duration in login response. licence: restrict nodes which are unregistered after 24 hours. scheduler: return error for namespace/volume not found dataplane: start gRPC threads separately from rest of the supervisor. Fixed ui: centre licence types. ui: capacity in ui is per namespace. cli: fail gracefully if missing some output details (i.e. no node exists for id). v2.0.0-rc.1 - Released 2020-03-31 Initial release of version 2.x. See StorageOS v2.0 Release Blog for details.\n","excerpt":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than …","ref":"/docs/reference/release_notes/","title":"Release notes"},{"body":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous replication ensures strong consistency for applications such as databases and Elasticsearch, incurring one network round trip on writes.\nThe basic model for StorageOS replication is of a master volume with distributed replicas. Each volume can be replicated between 0 and 5 times, which are provisioned to 0 to 5 nodes, up to the number of remaining nodes in the cluster.\nIn this diagram, the master volume D was created on node 1, and two replicas, D2 and D3 on nodes 3 and 5.\nWrites that come into D (step 1) are written in parallel to D2 and D3 (step 2). When both replicas and the master acknowledge that the data has been written (step 3), the write operation return successfully to the application (step 4).\nFor most applications, one replica is sufficient (storageos.com/replicas=1).\nAll replication traffic on the wire is compressed using the lz4 algorithm, then streamed over tcp/ip to target port tcp/5703.\nIf the master volume is lost, a replica is promoted to master (D2 or D3 above) and a new replica is created and synced on an available node (Node 2 or 4). This is transparent to the application and does not cause downtime.\nIf a replica volume is lost and there are enough remaining nodes, a new replica is created and synced on an available node. While a new replica is created and being synced, the volume\u0026rsquo;s health will be marked as degraded.\nIf the lost replica comes back online before the new replica has finished synchronizing, then StorageOS will calculate which of the two synchronizing replicas has the smallest difference compared to the master volume and keep that replica. The same holds true if a master volume is lost and a replica is promoted to be the new master. If possible, a new replica will be created and begin to sync. Should the former master come back online it will be demoted to a replica and the replica will the smallest difference to the current master will be kept.\nWhile the replica count is controllable on a per-volume basis, some environments may prefer to set default labels on the StorageClass.\nNumber of nodes StorageOS replicas are distributed across available nodes. When a node fails, a new replica is provisioned and synced as described above. Failure to provision a new replica will eventually cause the volume to become read-only. StorageOS will re-attempt creation of the replica for 90 seconds. After that period, if the old replica is not available and a new replica cannot be provisioned, StorageOS cannot guarantee that the data is safe and stored on multiple nodes as requested by the user. It will therefore force the volume to be set to read-only.\nTo ensure that a new replica can always be created, an additional node should be available. To guarantee high availability, clusters using volumes with 1 replica must have at least 3 storage nodes. When using volumes with 2 replicas, at least 4 storage nodes, 3 replicas, 5 nodes, etc.\nMinimum number of storage nodes = 1 (primary) + N (replicas) + 1\nFor an application to be able to use the volume normally, the volume needs to recover from a degraded state. This is achieved by having enough nodes on which to place the data, and triggering a remount of the volumes - i.e. deleting the Pod mounting the volume once it is in a healthy state.\nDelta Sync StorageOS implements a delta sync between a volume master and its replicas. This means that if a replica for a volume goes offline, that when the replica comes back online only the regions with changed blocks need to be synchronized. This optimization reduces the time it takes for replicas to catch up, improving volume resilience. Additionally, it reduces network and IO bandwidth which can reduce costs when running in public clouds.\n","excerpt":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous …","ref":"/docs/concepts/replication/","title":"Replication"},{"body":"Managing Resources for StorageOS containers Kubernetes resource requests and limits are two optional Pod properties that allow you to specify how much of a resource a container in a Pod needs or can use. They are two main resources that you can specify requests and limits for, CPU and Memory.\nAs StorageOS is an infrastructure component, the health of other applications depends on being able to write to the StorageOS volumes. As such it is of paramount importance to avoid restarts of the StorageOS DaemonSet Pods. Restarting a StorageOS Pod results in the volumes of the node the StorageOS Pod is running on being marked as Read Only, and causes the failover of primary volumes on that node to their replicas. After a StorageOS Pod restart, once the StorageOS DaemonSet Pod is \u0026ldquo;READY\u0026rdquo;, the application Pods running on the node need to be restarted in order to trigger a mount of the filesystem hosted on the StorageOS volume and resume normal operations. To avoid restarts of the StorageOS main container by Kubernetes due to resource limits being reached, it is recommended to not set resource limits on the StorageOS DaemonSet. In addition to avoiding resource limits, StorageOS uses a high priority class when the DaemonSet is installed in the \u0026lsquo;kube-system\u0026rsquo; namespace. That avoids the DaemonSet Pods of being evicted.\nFor more information about managing resources for containers please see the Kubernetes documentation\nDefining Pod resource requests and reservations To add resource requests and reservations to the StorageOS DaemonSet configure them in the StorageOSCluster resource.\n","excerpt":"Managing Resources for StorageOS containers Kubernetes resource requests and limits are two optional …","ref":"/docs/reference/resource_requests_and_limits/","title":"Resource Requests and Limits"},{"body":"","excerpt":"","ref":"/search/","title":"Search Results"},{"body":"Backing up files from StorageOS volumes In this example use case we provide three different strategies for accessing files that have been written to a StorageOS persistent volume.\nIn the following examples the \u0026ldquo;application\u0026rdquo; container is the container main, which has a rsync, Nginx or sftp sidecar container. The StorageOS volume that the application is writing to will be mounted into the sidecar container so files written by the application are available for export. Files can be exported using Nginx as a web file server, transferred using rsync or accessed via SFTP.\nThe files create a stateful set that can be used AFTER a StorageOS cluster has been created. See our guide on how to install StorageOS on Kubernetes for more information.\nClone Repository In order to deploy the examples, clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/backup Before deploying the backup-example stateful set we recommend looking through the examples to understand how the different containers are configured\n Exfiltrating files through HTTP Deploy the Nginx example\n$ kubectl create -f nginx/ service/backup-example created configmap/nginx-config created statefulset.apps/backup-example created pod/busybox created Check that a backup-example pod is running\n$ kubectl get pods -w -l app=backup-example-nginx NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-nginx-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-nginx ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 80/TCP 46s Use wget to access the files served by Nginx. Nginx is sharing files from the same volume that the main application container is writing to. The connection to the Nginx container is made via the backup-example service.\n$ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;lost%2Bfound/\u0026#34;\u0026gt;lost+found/\u0026lt;/a\u0026gt; 12-Feb-2019 12:32 - \u0026lt;a href=\u0026#34;date.txt\u0026#34;\u0026gt;date.txt\u0026lt;/a\u0026gt; 12-Feb-2019 12:49 29 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx/date.txt Tue Feb 12 12:49:15 UTC 2019 Depending on what files have been written to the StorageOS volume the output of the index file will be different. In the example the date.txt file we created in Step 2 is present on the volume.\nExfiltrating files through Rsync Deploy the rsync example\n$ kubectl create -f rsync/ service/backup-example created configmap/rsync-config created secret/rsync-credentials created statefulset.apps/backup-example created pod/rsync created Check that a backup-example pod is running\n$ kubectl get pods -w -l app=backup-example-rsync NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-rsync-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-rsync NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-rsync ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 873/TCP 46s Use rsync to access the files shared by the rsync daemon. rsync is sharing files from the same volume that the main container is writing to. A username and password that are set in the rsync-credentials secret. The secret supplied in the example has the username and password set to username and password.\n$ kubectl exec -it rsync sh / # rsync --list-only rsync://username@backup-example-rsync/share Password: drwxr-xr-x 4,096 2019/02/12 12:49:15 . -rw-r--r-- 29 2019/02/12 12:49:15 date.txt drwx------ 16,384 2019/02/12 12:32:40 lost+found / # rsync -chavzP rsync://username@backup-example-rsync/share/date.txt . Password: receiving incremental file list date.txt 29 100% 28.32kB/s 0:00:00 (xfr#1, to-chk=0/1) sent 43 bytes received 135 bytes 50.86 bytes/sec total size is 29 speedup is 0.16 / # cat date.txt Tue Feb 12 12:49:15 UTC 2019 In the example above the list of available files was shown and a file called date.txt was synchronized to the rsync container.\nExfiltrating files through SFTP Deploy the sftp example\n$ kubectl create -f sftp/ Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-sftp-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-sftp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-sftp ClusterIP 100.70.50.56 \u0026lt;none\u0026gt; 22/TCP 2h Use SFTP to access the files shared by the SFTP container. If you have made no changes to the sftp-config secret the password is password.\n$ kubectl exec -it sftp -- bash root@sftp:/# sftp alex@backup-example-sftp alex@backup-example-sftp\u0026#39;s password: Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found sftp\u0026gt; get date.txt Fetching /date.txt to date.txt /date.txt 100% 29 15.9KB/s 00:00 sftp\u0026gt; bye root@sftp:/# cat date.txt Tue Feb 12 17:51:32 UTC 2019 In order to do this a SFTP user needs to be configured. The details for the user are stored in the sftp-config secret (see sftp/17-secret.yaml). The secret consists of base64 encoded username:password:uid:guid and the user is chroot\u0026rsquo;ed inside their home directory so the mount point for the StorageOS volume in the SFTP container in sftp/20-backup-pod.yaml needs to be configured.\nUsing custom SSH Keys The ConfigMap ssh-key-pub (see sftp/15-configmap.yaml) needs to be populated with a public key. The corresponding private key needs to be base64 encoded and put into the ssh-key-private secret (see sftp/17-secret.yaml). The user to connect as is determined by the user that is configured in the sftp-config configMap. To restrict logins to the SSH key edit the sftp-config secret so it contains no password (user::uid:guid).\n Connect to the sftp pod and connect through the service to the SFTP container running inside the backup-example pod.\n$ kubectl exec -it sftp -- bash root@sftp:/# sftp -i /home/alex/.ssh/id_rsa alex@backup-example-sftp Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found ","excerpt":"Backing up files from StorageOS volumes In this example use case we provide three different …","ref":"/docs/usecases/sidecar-backups/","title":""},{"body":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for instance, StorageOS. A StorageClass defines parameters to pass to the provisioner, which in the case of StorageOS can be translated into behaviour applied to the Volumes. Many StorageClasses can be provisioned to apply different feature labels to the StorageOS Volumes.\nBy default the StorageOS Cluster Operator installs the fast StorageClass at bootstrap of StorageOS. You can define its name in the StorageOS Cluster Resource.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:fastlabels:app:storageosprovisioner:csi.storageos.com# CSI DriverallowVolumeExpansion:trueparameters:pool:defaultcsi.storage.k8s.io/fstype:ext4# Change the NameSpace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/controller-expand-secret-name:csi-controller-expand-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-expand-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# NameSpace that runs StorageOS DaemonsetStorageClasses can be created to define default labels for StorageOS volumes, but also to map to any semantic aggregation of volumes that suits your use case, whether there are different roles (dev, staging, prod), or a StorageClass maps to a team or customer using the cluster.\nExamples You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic StorageClass definition in v2-storageclass-replicated.yaml.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# CSI DriverallowVolumeExpansion:trueparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the NameSpace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/controller-expand-secret-name:csi-controller-expand-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-expand-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# NameSpace that runs StorageOS DaemonsetThat StorageClass can be used by a PVC:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels.\nYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\n","excerpt":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for …","ref":"/docs/operations/storageclasses/","title":"Kubernetes StorageClasses"},{"body":"","excerpt":"","ref":"/","title":"StorageOS"},{"body":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for two different purposes.\nTelemetry Telemetry is made up a DNS version check query and a once per day report. Error reporting is the sentry.io crash dump reporting. sentry.io StorageOS sends crash reports to sentry.io. This information helps our developers monitor and fix crashes. Information is sent to sentry.io when a process inside the StorageOS container crashes.\n The crash report contains the signal that triggered the shutdown (e.g. SIGSEGV), the exit code and whether or not the crash generated a core dump. All StorageOS clusters with a routable connection to the internet will send crash reports to sentry.io over tcp/443. StorageOS respects environment variables that ProxyFromEnvironment uses.\nAn exhaustive list of information included in the crash report is below:\n StorageOS version Crash description string Anonymized Cluster ID Anonymized Node ID DNS Query StorageOS will perform a \u0026ldquo;latest version check\u0026rdquo; using a DNS query in order to inform administrators that a new version is available. StorageOS will also send anonymized node ids, cluster id and StorageOS version information to StorageOS using a DNS query. The information that we send in the query is encoded as well as being anonymized. This query allows us to inform Cluster admins when StorageOS upgrades are available in the StorageOS GUI and in the logs.\nThe DNS query includes:\n Anonymized StorageOS Cluster ID Anonymized StorageOS node ID StorageOS version number Once Per Day Report The once per day report contains information about the StorageOS cluster and Kubernetes versions to help StorageOS focus our development efforts on the most popular platforms. The once per day data is encrypted and sent to a StorageOS telemetry server so it is never processed outside of StorageOS assets.\nAn exhaustive list of information included in the once per day report is below:\n api_call_metrics cluster_disable_crash_reporting cluster_disable_version_check cluster_log_format cluster_log_level cluster_tls_provided k8s_distribution k8s_in_k8s k8s_scheduler_extender_enabled k8s_version node_available_bytes node_capacity node_crash_files_on_disk node_created_at_time node_etcd_config node_etcd_namespacing_enabled node_etcd_tls_enabled node_free_bytes node_health node_http_tls_enabled node_id node_labels node_status node_storageos_version node_system_clock_time node_total_bytes node_version volume_fs_type volume_id volume_labels volume_master_attach volume_master_delete_deployment volume_master_detach volume_master_failover_deployment volume_master_promote volume_master_provision volume_master_recover_replica volume_master_trigger_rejoin volume_metrics volume_placement_strategy volume_provision_source_user_agent volume_replica_delete_deployment volume_replica_failover_deployment volume_replica_promote volume_replica_provision volume_replicas volume_replica_trigger_rejoin volume_size_bytes Disable Telemetry It is possible to disable telemetry using the GUI, CLI, API, environment variables or the StorageOS Cluster Spec.\nStorageOS Cluster Spec Disable telemetry explicitly through the configurable spec parameters of the StorageOSCluster custom resource.\nEnvironment Variables You can use the following environmental variables to disable or enable telemetry.\nDISABLE_VERSION_CHECK # Disable the DNS version check DISABLE_TELEMETRY # Disable the once per day reporting DISABLE_ERROR_REPORTING # Disable sentry.io crash reports ","excerpt":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for …","ref":"/docs/reference/telemetry/","title":"StorageOS Telemetry"},{"body":"The support bundle is a tool that gathers information both about StorageOS and the environment in which it is operating. It provides information about the state and configuration of the cluster, nodes and other Kubernetes objects, as well as system performance metrics.\nThe support bundle is an addition rather than a replacement to the diagnostic bundle.\nInstall Kubectl StorageOS plugin The support bundle is generated with the StorageOS kubectl plugin, which can be installed as follows:\n$ curl -sL https://docs.storageos.com/sh/storageos-support-bundle-install.sh | bash If you prefer to run the installation commands individually, we provide them here:\n# Create a temporary directory in which to store the plugin binaries TMP_DIR=$(mktemp -d /tmp/storageos-kubectl-plugin-XXXXX) # Download the plugin binaries and extract to the temporary directory TARGET=\u0026#34;kubectl-storageos_linux_amd64.tar.gz\u0026#34; curl -sSL -o kubectl-storageos.tar.gz https://github.com/storageos/storageos.github.io/raw/master/sh/$TARGET tar -xf kubectl-storageos.tar.gz -C $TMP_DIR/ # Clean up the tar file rm -f kubectl-storageos.tar.gz # Add executable permissions for the plugin binaries and move into system path # For details of the bundle-generation tool\u0026#39;s functionality, visit: # https://docs.storageos.com/docs/bundles/support-bundle/ chmod +x $TMP_DIR/bin/kubectl-storageos-bundle sudo mv $TMP_DIR/bin/kubectl-storageos-bundle /usr/local/bin/ For MacOS build, use TARGET=kubectl-storageos_darwin_amd64.tar.gz\n Generate Bundle To generate a bundle, use the following command, the specification of which can be viewed and edited by obtaining the bundle-configuration.yaml file, which is publicly available.\n$ kubectl storageos bundle https://docs.storageos.com/yaml/bundle-configuration.yaml\n The tool expects there to be a StorageOS CLI running in kube-system as a Pod with the label run=cli. The tool will exec into this pod to get information from the StorageOS API. If the StorageOS CLI Pod does not match this criteria, you can either pull the YAML file and change the selector in the file, or add the label to the Pod. You can run the cli container following these instructions.\n Data collected in the bundle The data collected covers both the state of the StorageOS cluster, and in particular information regarding the infrastructure on which StorageOS is operating. It includes:\n StorageOS CLI information StorageOS Operator logs StorageOS logs Cluster metadata Cluster resources (DaemonSets, Events, Services, Pods, etc.) Backend disk configuration and performance statistics Load average Network checks across StorageOS nodes and ports Running processes Privacy StorageOS can only obtain the bundle if it is downloaded by the user and given to our engineering team, or uploaded for analysis. The data received by StorageOS is private and never leaves nor will leave StorageOS Inc.\nThe data contained in the support bundle has the sole purpose of helping customers troubleshoot their issues.\n","excerpt":"The support bundle is a tool that gathers information both about StorageOS and the environment in …","ref":"/docs/reference/bundles/support_bundle/","title":"Support bundle"},{"body":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base directory for storing configuration and blob files. Recommended host filesystem types are ext4 or xfs. If you require a specific filesystem please contact StorageOS.\nPersistent Volume Filesystems StorageOS provides a block device on which a file system can be created. The creation of the filesystem is either handled by StorageOS or by Kubernetes which affects what filesystems can be created.\nCSI Driver When using StorageOS with the CSI driver, StorageOS is responsible for running mkfs against the block device that pods mount. StorageOS is able to create ext2, ext3, ext4 and xfs file systems.\n","excerpt":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base …","ref":"/docs/reference/filesystems/","title":"Supported File systems"},{"body":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain node taints. Taints can be thought of as the opposite of Node affinity in that taints repel pods. Node taints are automatically applied by Kubernetes in response to node resources coming under contention. As StorageOS provides storage to pods it should not be evicted during periods of resource contention, as any pods using StorageOS volumes on the same node would need to be restarted.\nAs StorageOS runs as a daemonset some Kubernetes tolerations are added by Kubernetes while others are automatically added by the StorageOS operator.\nFor more information about tolerations please see the Kubernetes documentation.\ntolerations:# The unreachable and not-ready tolerations are added by Kubernetes to daemonsets automatically- key:\u0026#34;node.kubernetes.io/unreachable\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;- key:\u0026#34;node.kubernetes.io/not-ready\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;# The following tolerations are added to the StorageOS daemonset by the StorageOS operator- key:node.kubernetes.io/disk-pressureoperator:Exists- key:node.kubernetes.io/memory-pressureoperator:Exists- key:node.kubernetes.io/network-unavailableoperator:Exists- key:node.kubernetes.io/out-of-diskoperator:Exists- key:node.kubernetes.io/pid-pressureoperator:Exists- key:node.kubernetes.io/unschedulableoperator:ExistsAdding Custom Tolerations To add custom tolerations to the StorageOS daemonset configure them in the StorageOSCluster resource.\n","excerpt":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain …","ref":"/docs/reference/tolerations/","title":"Tolerations"},{"body":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to the StorageOS installation, integration with orchestrators or common misconfigurations.\nTools To be able to troubleshoot issues the StorageOS cli is required.\nPod in pending because of mount error Issue: The output of kubectl describe pod $POD_ID contains no such file or directory and references the StorageOS volume device file.\nroot@node1:~# kubectl -n kube-system describe $POD_ID (...) Events: (...) Normal Scheduled 11s default-scheduler Successfully assigned default/d1 to node3 Warning FailedMount 4s (x4 over 9s) kubelet, node3 MountVolume.SetUp failed for volume \u0026#34;pvc-f2a49198-c00c-11e8-ba01-0800278dc04d\u0026#34; : stat /var/lib/storageos/volumes/d9df3549-26c0-4cfc-62b4-724b443069a1: no such file or directory Reason: There are two main reasons this issue may arise:\n The StorageOS DEVICE_DIR location is wrongly configured when using Kubelet as a container Mount Propagation is not enabled (Option 1) Misconfiguration of the DeviceDir/SharedDir\nSome Kubernetes distributions such as Rancher, DockerEE or some installations of OpenShift deploy the Kubelet as a container, because of this, the device files that StorageOS creates to mount into the containers need to be visible to the kubelet. StorageOS can be configured to share the device directory.\nModern installations use CSI, which handles the complexity internally.\nAssert: root@node1:~# kubectl -n default describe stos | grep \u0026#34;Shared Dir\u0026#34; Shared Dir: # \u0026amp;lt;-- Shouldn\u0026#39;t be blank Solution: The Cluster Operator Custom Definition should specify the SharedDir option as follows.\nspec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container ... See example on how to configure the StorageOS Custom Resource.\n (Option 2) Mount propagation is not enabled.\n Applies only if Option 1 is configured properly.\n Assert: If not using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/storageos/volumes is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/storageos/volumes/ root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/storageos/volumes bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nIf using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/kubelet/plugins/kubernetes.io~storageos/devices is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/kubelet/plugins/kubernetes.io~storageos/devices root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/kubelet/plugins/kubernetes.io~storageos/devices bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nSolution: Older versions of Kubernetes need to enable mount propagation as it is not enabled by default. Most Kubernetes distributions allow MountPropagation to be enabled using FeatureGates. Rancher specifically, needs to enable it in the \u0026ldquo;View in API\u0026rdquo; section of your cluster. You need to edit the section \u0026ldquo;rancherKubernetesEngineConfig\u0026rdquo; to enable the Kubelet feature gate.\nPVC pending state - Failed to dial StorageOS A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: root@node1:~/# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE vol-1 Pending fast 7s kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 7s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: Get http://storageos-cluster/version: failed to dial all known cluster members, (10.233.59.206:5705) Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount volume can\u0026rsquo;t be transmitted to StorageOS, hence the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nIn this case, the Event message indicates that StorageOS API is not responding, implying that StorageOS is not running. For Kubernetes to define StorageOS pods ready, the health check must pass.\nAssert: Check the status of StorageOS pods.\nroot@node1:~/# kubectl -n kube-system get pod --selector app=storageos # for CSI add --selector kind=daemonset NAME READY STATUS RESTARTS AGE storageos-qrqkj 0/1 Running 0 1m storageos-s4bfv 0/1 Running 0 1m storageos-vcpfx 0/1 Running 0 1m storageos-w98f5 0/1 Running 0 1m If the pods are not READY, the service will not forward traffic to the API they serve hence PVC will remain in pending state until StorageOS pods are available.\n Kubernetes keeps trying to execute the action until it succeeds. If a PVC is created before StorageOS finish starting, the PVC will be created eventually.\n Solution: StorageOS health check takes 60 seconds of grace before reporting as READY. If StorageOS is starting properly after that period, the volume will be created when StorageOS finishes its bootstrap. If StorageOS is not running or is not starting properly, the solution would be to troubleshoot the installation. PVC pending state - Secret Missing A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 13s (x2 over 28s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: failed to get secret from [\u0026#34;storageos\u0026#34;/\u0026#34;storageos-api\u0026#34;] Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount a volume can\u0026rsquo;t be transmitted to StorageOS, and the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nThe StorageClass provisioned for StorageOS references a Secret from where it retrieves the API endpoint and the authentication parameters. If that secret is incorrect or missing, the connections won\u0026rsquo;t be established. It is common to see that the Secret has been deployed in a different namespace where the StorageClass expects it or that is has been deployed with a different name.\nAssert: Check the StorageClass parameters to know where the Secret is expected to be found.\nkubectl get storageclass fast -o yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: 2018-09-25T08:44:57Z labels: app: storageos name: fast resourceVersion: \u0026#34;108853\u0026#34; selfLink: /apis/storage.k8s.io/v1/storageclasses/fast uid: 48490a9b-c09f-11e8-ba01-0800278dc04d parameters: adminSecretName: storageos-api adminSecretNamespace: storageos description: Kubernetes volume fsType: ext4 pool: default provisioner: kubernetes.io/storageos reclaimPolicy: Delete Note that the parameters specify adminSecretName and adminSecretNamespace.\n Check if the secret exists according to those parameters\nkubectl -n kube-system get secret storageos-api No resources found. Error from server (NotFound): secrets \u0026#34;storageos-api\u0026#34; not found If no resources are found, it is clear that the Secret doesn\u0026rsquo;t exist or it is not deployed in the right location.\n Solution: Deploy StorageOS following the installation procedures. If you are using the manifests provided for Kubernetes to deploy StorageOS rather than using automated provisioners, make sure that the StorageClass parameters and the Secret reference match.\nNode name different from Hostname Issue: StorageOS nodes can\u0026rsquo;t join the cluster showing the following log entries.\ntime=\u0026#34;2018-09-24T13:47:02Z\u0026#34; level=error msg=\u0026#34;failed to start api\u0026#34; error=\u0026#34;error verifying UUID: UUID aed3275f-846b-1f75-43a1-adbfec8bf974 has already been registered and has hostname \u0026#39;debian-4\u0026#39;, not \u0026#39;node4\u0026#39;\u0026#34; module=command Reason: The StorageOS registration process to start the cluster uses the hostname of the node where the StorageOS container is running, provided by the Kubelet. However, StorageOS verifies the network hostname of the OS as a prestart check to make sure it can communicate with other nodes. If those names don\u0026rsquo;t match, StorageOS will be unable to start.\nSolution: Make sure the hostnames match with the Kubernetes advertised names. If you have changed the hostname of your nodes, make sure that you restart the nodes to apply the change.\nPeer discovery - Networking Issue: StorageOS nodes can\u0026rsquo;t join the cluster showing the following logs after one minute of container uptime.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover nodes in the cluster. When StorageOS starts, one or more nodes can be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nAssert: It is likely that ports are block by a firewall.\nSSH into one of your nodes and check connectivity to the rest of the nodes.\n# Successfull execution: [root@node06 ~]# nc -zv node04 5705 Ncat: Version 7.50 ( https://nmap.org/ncat ) Ncat: Connected to 10.0.1.166:5705. Ncat: 0 bytes sent, 0 bytes received in 0.01 seconds. StorageOS exposes network diagnostics in its API, viewable from the CLI. To use this feature, the CLI must query the API of a running node. The diagnostics show information from all known cluster members. If all the ports are blocked during the first bootstrap of the cluster, the diagnostics won\u0026rsquo;t show any data as nodes couldn\u0026rsquo;t register.\n StorageOS networks diagnostics are available for storageos-rc5 and storageos-cli-rc3 and above.\n # Example: root@node1:~# storageos cluster connectivity SOURCE NAME ADDRESS LATENCY STATUS MESSAGE node4 node2.nats 172.28.128.4:5708 1.949275ms OK node4 node3.api 172.28.128.5:5705 3.070574ms OK node4 node3.nats 172.28.128.5:5708 2.989238ms OK node4 node2.directfs 172.28.128.4:5703 2.925707ms OK node4 node3.etcd 172.28.128.5:5707 2.854726ms OK node4 node3.directfs 172.28.128.5:5703 2.833371ms OK node4 node1.api 172.28.128.3:5705 2.714467ms OK node4 node1.nats 172.28.128.3:5708 2.613752ms OK node4 node1.etcd 172.28.128.3:5707 2.594159ms OK node4 node1.directfs 172.28.128.3:5703 2.601834ms OK node4 node2.api 172.28.128.4:5705 2.598236ms OK node4 node2.etcd 172.28.128.4:5707 16.650625ms OK node3 node4.nats 172.28.128.6:5708 1.304126ms OK node3 node4.api 172.28.128.6:5705 1.515218ms OK node3 node2.directfs 172.28.128.4:5703 1.359827ms OK node3 node1.api 172.28.128.3:5705 1.185535ms OK node3 node4.directfs 172.28.128.6:5703 1.379765ms OK node3 node1.etcd 172.28.128.3:5707 1.221176ms OK node3 node1.nats 172.28.128.3:5708 1.330122ms OK node3 node2.api 172.28.128.4:5705 1.238541ms OK node3 node1.directfs 172.28.128.3:5703 1.413574ms OK node3 node2.etcd 172.28.128.4:5707 1.214273ms OK node3 node2.nats 172.28.128.4:5708 1.321145ms OK node1 node4.directfs 172.28.128.6:5703 1.140797ms OK node1 node3.api 172.28.128.5:5705 1.089252ms OK node1 node4.api 172.28.128.6:5705 1.178439ms OK node1 node4.nats 172.28.128.6:5708 1.176648ms OK node1 node2.directfs 172.28.128.4:5703 1.529612ms OK node1 node2.etcd 172.28.128.4:5707 1.165681ms OK node1 node2.api 172.28.128.4:5705 1.29602ms OK node1 node2.nats 172.28.128.4:5708 1.267454ms OK node1 node3.nats 172.28.128.5:5708 1.485657ms OK node1 node3.etcd 172.28.128.5:5707 1.469429ms OK node1 node3.directfs 172.28.128.5:5703 1.503015ms OK node2 node4.directfs 172.28.128.6:5703 1.484ms OK node2 node1.directfs 172.28.128.3:5703 1.275304ms OK node2 node4.nats 172.28.128.6:5708 1.261422ms OK node2 node4.api 172.28.128.6:5705 1.465532ms OK node2 node3.api 172.28.128.5:5705 1.252768ms OK node2 node3.nats 172.28.128.5:5708 1.212332ms OK node2 node3.directfs 172.28.128.5:5703 1.192792ms OK node2 node3.etcd 172.28.128.5:5707 1.270076ms OK node2 node1.etcd 172.28.128.3:5707 1.218522ms OK node2 node1.api 172.28.128.3:5705 1.363071ms OK node2 node1.nats 172.28.128.3:5708 1.349383ms OK Solution: Open ports following the prerequisites page.\nPeer discovery - Pod allocation Issue: StorageOS nodes can\u0026rsquo;t join the cluster and show the following log entries.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover the nodes in the cluster. When StorageOS starts, one or more active nodes must be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nIf there are no active StorageOS nodes, the bootstrap process will elect the first node in the JOIN variable as master, and the rest will try to discover from it. In case of that node not starting, the whole cluster will remain unable to bootstrap.\nInstallations of StorageOS use a DaemonSet, and by default do not schedule StorageOS pods to master nodes, due to the presence of the node-role.kubernetes.io/master:NoSchedule taint that is typically present. In such cases the JOIN variable must not contain master nodes or the StorageOS cluster will remain unable to start.\nAssert: Check that the first node of the JOIN variable started properly.\nroot@node1:~/# kubectl -n kube-system describe ds/storageos | grep JOIN JOIN: 172.28.128.3,172.28.128.4,172.28.128.5 root@node1:~/# kubectl -n kube-system get pod -o wide | grep 172.28.128.3 storageos-8zqxl 1/1 Running 0 2m 172.28.128.3 node1 Solution: Make sure that the JOIN variable doesn\u0026rsquo;t specify the master nodes. In case you are using the discovery service, it is necessary to ensure that the DaemonSet won\u0026rsquo;t allocate Pods on the masters. This can be achieved with taints, node selectors or labels.\nFor installations with the StorageOS operator you can specify which nodes to deploy StorageOS on using nodeSelectors. See examples in the Cluster Operator Examples page.\nFor more advanced installations using compute-only and storage nodes, check the storageos.com/deployment=computeonly label that can be added to the nodes through Kubernetes node labels, or StorageOS in the Labels page.\nLIO Init:Error Issue: StorageOS pods not starting with Init:Error\nkubectl -n kube-system get pod NAME READY STATUS RESTARTS AGE storageos-2kwqx 0/3 Init:Err 0 6s storageos-cffcr 0/3 Init:Err 0 6s storageos-d4f69 0/3 Init:Err 0 6s storageos-nhq7m 0/3 Init:Err 0 6s Reason: This indicates that since the Linux open source SCSI drivers are not enabled, StorageOS cannot start. The StorageOS DaemonSet enables the required kernel modules on the host system. If you are seeing these errors it is because that container couldn\u0026rsquo;t load the modules.\nAssert Check the logs of the init container.\nkubectl -n kube-system logs $ANY_STORAGEOS_POD -c storageos-init In case of failure, it will show the following output, indicating which kernel modules couldn\u0026rsquo;t be loaded or that they are not properly configured:\nChecking configfs configfs mounted on sys/kernel/config Module target_core_mod is not running executing modprobe -b target_core_mod Module tcm_loop is not running executing modprobe -b tcm_loop modprobe: FATAL: Module tcm_loop not found. Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and delete StorageOS pods, allowing the DaemonSet to create the pods again.\nLIO not enabled Issue: StorageOS node can\u0026rsquo;t start and shows the following log entries.\ntime=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;liocheck returned error\u0026#34; category=liocheck error=\u0026#34;exit status 1\u0026#34; module=dataplane stderr=\u0026#34;Sysfs root \u0026#39;/sys/kernel/config/target\u0026#39; is missing, is kernel configfs present and target_core_mod loaded? category=fslio level=warn\\nRuntime error checking stage \u0026#39;target_core_mod\u0026#39;: SysFs root missing category=fslio level=warn\\nliocheck: FAIL (lio_capable_system() returns failure) category=fslio level=fatal\\n\u0026#34; stdout= time=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;failed to start dataplane services\u0026#34; error=\u0026#34;system dependency check failed: exit status 1\u0026#34; module=command Reason: This indicates that one or more kernel modules required for StorageOS are not loaded.\nAssert The following kernel modules must be enabled in the host.\nlsmod | egrep \u0026#34;^tcm_loop|^target_core_mod|^target_core_file|^configfs\u0026#34; Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and restart the container.\n(OpenShift) StorageOS pods missing \u0026ndash; DaemonSet error StorageOS DaemonSet doesn\u0026rsquo;t have any pod replicas. The DaemonSet couldn\u0026rsquo;t allocate any Pod due to security issues.\nIssue: [root@master02 standard]# oc get pod No resources found. [root@master02 standard]# oc describe daemonset storageos (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedCreate 0s (x12 over 10s) daemonset-controller Error creating: pods \u0026#34;storageos-\u0026#34; is forbidden: unable to validate against any security context constraint: [provider restricted: .spec.securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used provider restricted: .spec.securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.volumes[0]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[1]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[2]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[3]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.initContainers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.initContainers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.initContainers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.initContainers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.containers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.containers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.containers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used] Reason: The OpenShift cluster has security context constraint policies enabled that forbid any pod, without an explicitly set policy for the service account, to be allocated.\nAssert: Check if the StorageOS ServiceAccount can create pods with enough permissions\noc get scc privileged -o yaml # Or custom scc with enough privileges (...) users: - system:admin - system:serviceaccount:openshift-infra:build-controller - system:serviceaccount:management-infra:management-admin - system:serviceaccount:management-infra:inspector-admin - system:serviceaccount:storageos:storageos \u0026amp;lt;-- - system:serviceaccount:tiller:tiller If the StorageOS sa system:serviceaccount:storageos:storageos is in the privileged scc it will be able to create pods.\nSolution: Add the ServiceAccount system:serviceaccount:storageos:storageos to a scc with enough privileges.\noc adm policy add-scc-to-user privileged system:serviceaccount:storageos:storageos Getting Help If our troubleshooting guides do not help resolve your issue, please see our support section for details on how to get in touch with us.\n","excerpt":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to …","ref":"/docs/operations/troubleshooting/","title":"Troubleshooting"},{"body":"This document details a step-by-step procedure to remove StorageOS from a Kubernetes cluster.\nRemember that StorageOS enables the stateful applications within your cluster. It is very important to remove any applications that rely on StorageOS before you remove StorageOS itself, or those applications will suffer unrecoverable errors.\nRemove Stateful Workloads and Data Delete any resources using StorageOS volumes\nDelete any statefulsets, deployments or pods that are using StorageOS Volumes.\n Delete PVCs using StorageOS\nDelete any Persistent Volume Claims that are using StorageOS.\n$ kubectl -n $NS delete pvc $PVC This will delete data held by StorageOS and won\u0026rsquo;t be recoverable.\n Remove StorageOS Cluster Delete StorageOS Cluster\n$ kubectl get storageoscluster --all-namespaces # Find the namespace where the Custom Resource runs $ kubectl -n $NS delete storageoscluster --all # Usually to be found in storageos-operator Wait until the StorageOS resources are gone\n$ kubectl -n kube-system get pod # NS: Namespace where StorageOS Daemonset is running, usually \u0026#39;kube-system\u0026#39; Uninstall the StorageOS Operator Delete the Cluster Operator once the StorageOS Pods are terminated\n Delete the StorageOS Operator deployment\n$ kubectl delete -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Procedure is finished. StorageOS is now uninstalled.\nRemove StorageOS contents and metadata (unrecoverable) The steps up until now have been recoverable - as long as the etcd backing StorageOS and the contents of /var/lib/storageos on your nodes are safe then StorageOS can be reinstalled. For complete removal and recovery of disk space, you can use the following procedure.\n The following steps will delete all data held by StorageOS and won\u0026rsquo;t be recoverable.\n Remove the StorageOS data directory\nThere are two ways to remove the StorageOS data directory:\n (Option 1) Login in to the hosts and execute the following commands\n$ sudo rm -rf /var/lib/storageos $ sudo umount /var/lib/kubelet/plugins_registry/storageos (Option 2) Execute the following command to deploy a DaemonSet that removes the StorageOS data directory.\n N.B This step is irreversible and once the data is removed it cannot be recovered.\n Run the following command where kubectl is installed and with the context set for your Kubernetes cluster.\n $ curl -s https://docs.storageos.com/sh/permanently-delete-storageos-data.sh | bash Flush Etcd Data\nThis will remove any keys written by StorageOS.\n$ export ETCDCTL_API=3 $ etcdctl --endpoints=http://$ETCD_IP:2379 del --prefix \u0026#34;storageos\u0026#34; If running Etcd with mTLS, you can set the certificates location with the following command.\n$ export ETCDCTL_API=3 $ etcdctl --endpoints=https://$ETCD_IP:2379 \\ --cacert=/path/to/ca.pem \\ --cert=/path/to/client-cert.pem \\ --key=/path/to/client-key.pem \\ del --prefix \u0026#34;storageos\u0026#34; ","excerpt":"This document details a step-by-step procedure to remove StorageOS from a Kubernetes cluster. …","ref":"/docs/operations/uninstall/","title":"Uninstall StorageOS"},{"body":"This document details a step-by-step procedure to upgrade a StorageOS v2 cluster.\nKeep in mind that upgrading a cluster will require minor downtime of applications using StorageOS volumes. However we will take steps to minimize the required downtime as much as possible.\nUpgrade StorageOS Ensure that you have read the PIDs prerequisite introduced in StorageOS v2.3 and that you check the init container logs to ensure your environments PID limits are set correctly.\n Warning: To reduce downtime, it is recommended to docker pull the new StorageOS container image storageos/node:v2.3.2 on the nodes beforehand so that the cluster spins up faster!\n First make sure you keep a backup of all the StorageOS yaml files. You can reuse the StorageOSCluster configuration file to easily upgrade your cluster. You can also backup the Statefulset yaml files to keep track of the replicas.\nkubectl get pod -n storageos-operator -o yaml \u0026gt; storageos_operator.yaml kubectl get storageoscluster -n storageos-operator -o yaml \u0026gt; storageos_cr.yaml kubectl get statefulset --all-namespaces \u0026gt; statefulset-sizes.yaml Scale all stateful applications that use StorageOS volumes to 0.\n Delete the StorageOSCluster CR.\nkubectl delete storageoscluster cluster-storageos -n storageos-operator If you have renamed the StorageOSCluster resource you can find it by using the command below.\nkubectl get storageoscluster --all-namespaces Deploy the new operator.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.3.1/storageos-operator.yaml Warning: If you have made changes to the CRDs, Service Account or Cluster Role, make sure you migrate those changes in the StorageOS operator yaml.\n Edit the StorageOSCluster Custom Resource (storageos_cr.yaml) with the new node image version.\nimages:nodeContainer:\u0026#34;storageos/node:v2.3.2\u0026#34;If you\u0026rsquo;re not using any of the hardcoded container image versions make sure to delete them as well and only leave the nodeContainer image as shown above.\nMake sure to also delete all metadata that you\u0026rsquo;re not using. The final metadata stanza should be something similar to that:\nmetadata:name:\u0026#34;cluster-storageos\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34; Re-create the StorageOSCluster\nkubectlcreate-fstorageos_cr.yaml Wait for all the StorageOS pods to enter the RUNNING state\nkubectl get pods -l app=storageos -A -w Scale your stateful applications back up.\n Congratulations, you now have the latest version StorageOS!\n","excerpt":"This document details a step-by-step procedure to upgrade a StorageOS v2 cluster.\nKeep in mind that …","ref":"/docs/operations/upgrade/","title":"Upgrade StorageOS"},{"body":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces using policies.\n Note: Users are created with access to the default namespace. This access is only revoked when a policy is created for the user or their group.\n Managing users Users can be created and deleted by navigating to the \u0026ldquo;Users\u0026rdquo; section of the GUI.\nAlternatively users can also be managed using the CLI.\nTo create a user with the CLI, run:\n$ storageos user create jim The above command will create a user named jim. The command will also prompt you to enter a password for the newly created user.\n","excerpt":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces …","ref":"/docs/operations/users/","title":"User Management"},{"body":"Velero Backups Using Velero to backup your StorageOS volumes Velero is an open source tool to safely backup and restore, perform disaster recovery, and migrate Kubernetes cluster resources and persistent volumes.\nVelero consists of the following components:\n A Velero Server in the cluster A CLI client A restic daemonset in the cluster Here\u0026rsquo;s a diagram on how Velero backups work:\nAnd here\u0026rsquo;s how Velero restores work:\nIn our case, we will use MinIO in the Kubernetes cluster as the \u0026ldquo;cloud provider\u0026rdquo; pictured in the backup diagram. MinIO is an object store that uses an S3 compatible API and as such can be used to store our backed up resources.\nWe will set up MinIO through a StatefulSet with a 50GiB StorageOS volume and a ClusterIP service. The UI can be accessed through the browser by port-forwarding the MinIO service.\nVelero uses Restic to backup Kubernetes volumes. Restic is a fast and secure backup program for filesystems whose documentation can be found here. The way it works is that it scans the volume directory for its files and then splits those files into blobs which are then sent to MinIO. Here\u0026rsquo;s how it integrates with Velero.\nPrerequisites Here are the prerequisites for running Velero in your Kubernetes cluster:\n Kubernetes cluster version 1.15 to 1.20 with DNS Kubectl installed Velero cli installed https://Velero.io/docs/main/basic-install/ Velero can also be installed from a helm chart\n Install MinIO with a StorageOS volume First, make sure to clone the StorageOS use cases repository and navigate to the Velero directory:\ngit clone https://github.com/storageos/use-cases.git cd use-cases/Velero Installing MinIO is really simple, just deploy it using the minio-deploy.yaml manifest file:\nkubectl apply -f ./minio Confirm that MinIO was deployed successfully:\n$ kubectl get pods -n velero NAME READY STATUS RESTARTS AGE minio-0 1/1 Running 0 3m48s minio-setup-zvcdg 0/1 Completed 1 3m47s You can access the web UI of MinIO by port-forwarding the MinIO service with this command:\nkubectl port-forward service/minio -n velero 9000 Install Velero Use the following command to install Velero via the Velero CLI or alternatively use the helm chart. To install it using the Velero cli, just run this command:\n The AWS plugin is being used because MinIO implements the S3 API. This is required even if you\u0026rsquo;re not using AWS.\n velero install \\ --provider aws \\ --plugins velero/velero-plugin-for-aws:v1.0.0 \\ --bucket velero \\ --secret-file ./credentials-Velero \\ --use-volume-snapshots=false \\ --backup-location-config region=minio,s3ForcePathStyle=\u0026#34;true\u0026#34;,s3Url=http://minio.velero.svc:9000 \\ --use-restic Make sure that Velero is installed correctly:\n$ kubectl logs deployment/velero -n velero ... time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Server started successfully\u0026#34; logSource=\u0026#34;pkg/cmd/server/server.go:881\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=restic-repository logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=restore logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=backup-sync logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=backup logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=backup-deletion logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Checking for expired DeleteBackupRequests\u0026#34; controller=backup-deletion logSource=\u0026#34;pkg/controller/backup_deletion_controller.go:551\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Done checking for expired DeleteBackupRequests\u0026#34; controller=backup-deletion logSource=\u0026#34;pkg/controller/backup_deletion_controller.go:579\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=schedule logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=downloadrequest logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=gc-controller logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=serverstatusrequest logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; Quiesce Before moving on to the MySQL use case, it is important to talk about quiescence. To quiesce is to pause or alter a device or application to achieve a consistent state, usually in preparation for a backup or other maintenance. Quiescence is very important when backing up real world applications because in flight or cached write operations of the filesystem during the backup process can cause corruption of the backed up data. It is a common mistake for engineers to back up their data without quiescing the filesystem and noticing corruption when they need to restore from that backup. A simple form of quiescence is the unix command fsfreeze, but in most cases, application specific quiescence should be used instead.\nMySQL use case For the our use case, we\u0026rsquo;ll use Velero with a real world application, MySQL.\n First deploy MySQL, based on the MySQL use case:\nkubectl apply -f ./mysql Notice that the Statefulset also includes 5 annotations:\nannotations:backup.velero.io/backup-volumes:datapre.hook.backup.velero.io/command:\u0026#39;[\u0026#34;/sbin/fsfreeze\u0026#34;, \u0026#34;--freeze\u0026#34;, \u0026#34;/var/lib/mysql\u0026#34;]\u0026#39;pre.hook.backup.velero.io/container:fsfreezepost.hook.backup.velero.io/command:\u0026#39;[\u0026#34;/sbin/fsfreeze\u0026#34;, \u0026#34;--unfreeze\u0026#34;, \u0026#34;/var/lib/mysql\u0026#34;]\u0026#39;post.hook.backup.velero.io/container:fsfreezeThe first annotation specifies which volume to backup using restic. The other annotations are used to perform an fsfreeze on the volume mount point using pre and post backup hooks, for more details about Velero pre/post backup hooks please see their documentation here.\nWe have to specify to use the fsfreeze ubuntu container since the MySQL container doesn\u0026rsquo;t support fsfreeze\n- name:fsfreezeimage:ubuntu:bionicsecurityContext:privileged:truevolumeMounts:- name:datamountPath:/var/lib/mysqlcommand:- \u0026#34;/bin/bash\u0026#34;- \u0026#34;-c\u0026#34;- \u0026#34;sleep infinity\u0026#34; Wait for the pod to spin up:\n$ kubectl get pods -n mysql NAME READY STATUS RESTARTS AGE client 1/1 Running 0 24m mysql-0 2/2 Running 0 24m Exec into the MySQL pod and populate it with data using the commands below.\n$ kubectl exec mysql-0 -n mysql -ti -c mysql -- mysql mysql\u0026gt; create database shop; mysql\u0026gt; use shop; mysql\u0026gt; create table books (title VARCHAR(256), price decimal(4,2)); mysql\u0026gt; insert into books value (\u0026#39;Gates of Fire\u0026#39;, 13.99); mysql\u0026gt; select * from books; +---------------+-------+ | title | price | +---------------+-------+ | Gates of Fire | 13.99 | +---------------+-------+ 1 row in set (0.00 sec) mysql\u0026gt; exit Create the Velero backup:\nvelero backup create mysql-backup --include-namespaces mysql --wait Confirm that all the Kubernetes objects are there and the restic backups completed successfully:\nvelero backup describe mysql-backup --details After the backup is finished, delete the StatefulSet and PVC.\n N.B. It\u0026rsquo;s important to make sure that the StatefulSet is deleted because the restore would be unable to complete if a StatefulSet pod is recreated during the restore process.\n kubectl delete statefulset mysql -n mysql kubectl delete pvc data-mysql-0 -n mysql Make sure that the pod is fully terminated:\n$ kubectl get pods -n mysql NAME READY STATUS RESTARTS AGE client 1/1 Running 0 25m Restore MySQL using Velero:\nvelero restore create --from-backup mysql-backup --wait Wait for the MySQL pod to spin up and see if the data is backed up:\n$ kubectl exec mysql-0 -n mysql -ti -c mysql -- mysql mysql\u0026gt; use shop; mysql\u0026gt; select * from books; +---------------+-------+ | title | price | +---------------+-------+ | Gates of Fire | 13.99 | +---------------+-------+ 1 row in set (0.00 sec) mysql\u0026gt; exit The data should be there and the backup was restored successfully!\n","excerpt":"Velero Backups Using Velero to backup your StorageOS volumes Velero is an open source tool to safely …","ref":"/docs/usecases/velero-backups/","title":""},{"body":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version [flags] Flags: -h, --help help for version Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version …","ref":"/docs/reference/cli/version/","title":"Version"},{"body":"StorageOS supports offline resize of volumes, either through editing a PVC storage request, or by updating the volume config via the CLI or UI. A volume cannot be resized while it is in use. To ensure that it is not, for a resize operation to take place the volume must not be attached to a node. This requires that any pods using a volume be scaled down for the resize to take place.\nPlease note that StorageOS only supports increasing volume size. For more information about how the resize works please see our Resize Concepts page.\nResizing a Volume In order to resize a PVC the storage request field must be updated.\napiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: fast accessModes: - ReadWriteOnce resources: requests: storage: 5Gi The Volume must be offline for this to take effect. Therefore, before performing the operation, scale down any pods using the volume. This will ensure that it is not in use.\nIn order to edit a PVC you can use kubectl edit or kubectl apply to make changes.\n N.B. Resizing a volume without updating the PVC directly will NOT result in the PVC being updated. The methods below are included for completeness. In Kubernetes environments editing the PVC is the preferred method for resizing a volume.\n To resize a volume using the StorageOS CLI use the volume update command\n$ storageos update volume size pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa 10GiB Name: pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa ID: 925e667f-91d3-465a-9391-8fdb56d0c9ff Size: 11 GB Description: AttachedOn: Replicas: 1x ready Labels: - csi.storage.k8s.io/pv/name pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa - csi.storage.k8s.io/pvc/name pvc-1 - csi.storage.k8s.io/pvc/namespace default - foo bar - pool default - storageos.com/replicas 1 Volume pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa (925e667f-91d3-465a-9391-8fdb56d0c9ff) updated. Size changed. To resize a volume using the StorageOS UI, navigate to the volumes section and click the edit pencil in order to update the volume config.\n","excerpt":"StorageOS supports offline resize of volumes, either through editing a PVC storage request, or by …","ref":"/docs/operations/resize/","title":"Volume Resize"},{"body":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard POSIX semantics. StorageOS presents volumes as mounts into containers via the Linux LIO subsystem.\nConceptually, StorageOS volumes have a frontend presentation, which is what the application sees, and a backend presentation, which is the actual on-disk format. Depending on the configuration, frontend and backend components may be on the same or different hosts.\nVolumes are formatted using the linux standard ext4 filesystem by default. Kubernetes users may change the default filesystem type to ext2, ext3, ext4, or xfs by setting the fsType parameter in their StorageClass (see Supported Filesystems for more information). Different filesystems may be supported in the future.\nStorageOS volumes are represented on disk in two parts. Actual volume data is written to blob files in /var/lib/storageos/data/dev[\\d+]. Inside these directories, each StorageOS block device gets two blob files of the form vol.xxxxxx.y.blob, where x is the inode number for the device, and y is an index between 0 and 1. We provide two blob files in order to ensure that certain operations which require locking do not impede in-flight writes to the volume.\nIn systems which have multiple /var/lib/storageos/data/dev[\\d+] directories, two blob files are created per block device. This allows us to load-balance writes across multiple devices. In cases where dev directories are added after a period of run time, later directories are favoured for writes until the data is distributed evenly across the blob files.\nMetadata is kept in directories named /var/lib/storageos/data/db[\\d+]. We maintain an index of all blocks written to the blob file inside the metadata store, including checksums. These checksums allow us to detect bitrot, and return errors on reads, rather than serve bad data. In future versions we may implement recovery from replicas for volumes with one or more replicas defined.\nStorageOS metadata requires approximately 2.7GB of storage per 1TB of allocated blocks in the associated volume. This size is consistent irrespective of data compression defined on the volume.\nTo ensure deterministic performance, individual StorageOS volumes must fit on a single node.\nVolume Resize StorageOS v2.1 supports offline resize of volumes. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure that the volume is not in use.\nThis means that if a Kubernetes pod is currently consuming a volume that a resize request has been issued for, the resize will not be actioned until the pod is terminated and the volume is detached from the node. The StorageOS controlplane will then attach the volume to the node that holds the master deployment and resize the underlying block device and then run resize2fs to expand the filesystem.\nFor a walk through of how to resize a volume please see the Volume Resize operations page.\n","excerpt":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard …","ref":"/docs/concepts/volumes/","title":"Volumes"},{"body":"Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.\nUsing StorageOS persistent volumes with Apache Zookeeper means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Zookeeper pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nAs Zookeeper has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Zookeeper to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Zookeeper on Kubernetes Pre-requisites StorageOS is assumed to have been installed; please check for the latest available version here. Helm To simplify the deployment of Zookeeper, we\u0026rsquo;ve used this Zookeeper helm chart (incubator) (version 1.2.2, app version 3.4.10) and rendered it into the example deployment files you can find in our GitHub repo.\nDeployment Clone the use cases repo You can find the latest files in the StorageOS use cases repository in /zookeeper/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion ---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:zookeeper...spec:replicas:3# \u0026lt;--- number of zookeeper pods...containers:- name:zookeeperimage:\u0026#34;gcr.io/google_samples/k8szk:v3\u0026#34;imagePullPolicy:IfNotPresent...volumeMounts:- name:datamountPath:/var/lib/zookeepervolumeClaimTemplates:- metadata:name:dataspec:accessModes:- \u0026#34;ReadWriteOnce\u0026#34;storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useresources:requests:storage:\u0026#34;5Gi\u0026#34;# \u0026lt;--- storage requested per podThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nCreate the kubernetes objects cd storageos-usecases kubectl apply -f ./zookeeper/ Confirm Zookeeper is up and running $ kubectl get pods NAME READY STATUS RESTARTS AGE zookeeper-0 1/1 Running 0 2m30s zookeeper-1 1/1 Running 0 112s zookeeper-2 1/1 Running 0 56s zookeeper-test-client 1/1 Running 0 2m30s Connect to Zookeeper Connect to the zookeeper client pod and list existing topics using the service endpoint\nkubectl exec -it zookeeper-test-client /bin/bash and issue a command to the zookeeper service\nzkCli.sh -server zookeeper ls /zookeeper ","excerpt":"Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration …","ref":"/docs/usecases/zookeeper/","title":""}]
\ No newline at end of file
diff --git a/search/index.html b/search/index.html
index 13fc0fae7..a377e30ca 100644
--- a/search/index.html
+++ b/search/index.html
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/offline-search-index.8be62cf583684b235acce140a3f1aa3d5e19b46a0ab8dfdd68644b7f0b5efeb1f57bc5d95e8bff7ee08daf3c2f1f4e03b9fe7728dff948879aa98dcecdd85882.json"
+ data-offline-search-index-json-src="/offline-search-index.82fab81ab1b93ff0b717ad5465254ecb5f63158c6dce4c9197d1f90d1339c8c017772675c0aa6d4dd2277e124d6b0d1f2a1fe5c486efbc2158c18e0a2eef47a3.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/404.html b/v1.x/404.html
index 5e7acc433..cbc0e39ee 100644
--- a/v1.x/404.html
+++ b/v1.x/404.html
@@ -181,7 +181,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/best-practices/index.html b/v1.x/docs/best-practices/index.html
index be5da6391..c89d8c4a5 100644
--- a/v1.x/docs/best-practices/index.html
+++ b/v1.x/docs/best-practices/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/architecture/index.html b/v1.x/docs/concepts/architecture/index.html
index 4eeb103bb..c33c8bddb 100644
--- a/v1.x/docs/concepts/architecture/index.html
+++ b/v1.x/docs/concepts/architecture/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/clusters/index.html b/v1.x/docs/concepts/clusters/index.html
index 099df7942..45ed0882e 100644
--- a/v1.x/docs/concepts/clusters/index.html
+++ b/v1.x/docs/concepts/clusters/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/compression/index.html b/v1.x/docs/concepts/compression/index.html
index 6ca13c858..13acdac4c 100644
--- a/v1.x/docs/concepts/compression/index.html
+++ b/v1.x/docs/concepts/compression/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/encryption/index.html b/v1.x/docs/concepts/encryption/index.html
index 6a75a33b1..845a6ebcd 100644
--- a/v1.x/docs/concepts/encryption/index.html
+++ b/v1.x/docs/concepts/encryption/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/fencing/index.html b/v1.x/docs/concepts/fencing/index.html
index f495a1771..2ce61ea22 100644
--- a/v1.x/docs/concepts/fencing/index.html
+++ b/v1.x/docs/concepts/fencing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/index.html b/v1.x/docs/concepts/index.html
index 72b64e711..5f896c18b 100644
--- a/v1.x/docs/concepts/index.html
+++ b/v1.x/docs/concepts/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/metrics/index.html b/v1.x/docs/concepts/metrics/index.html
index d7ba8aa84..6f9ebc359 100644
--- a/v1.x/docs/concepts/metrics/index.html
+++ b/v1.x/docs/concepts/metrics/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/namespaces/index.html b/v1.x/docs/concepts/namespaces/index.html
index 431227038..5dcae1635 100644
--- a/v1.x/docs/concepts/namespaces/index.html
+++ b/v1.x/docs/concepts/namespaces/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/nodes/index.html b/v1.x/docs/concepts/nodes/index.html
index 2af9b635d..f190e33d4 100644
--- a/v1.x/docs/concepts/nodes/index.html
+++ b/v1.x/docs/concepts/nodes/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/podlocality/index.html b/v1.x/docs/concepts/podlocality/index.html
index 70a903919..8a47bc5a6 100644
--- a/v1.x/docs/concepts/podlocality/index.html
+++ b/v1.x/docs/concepts/podlocality/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/policies/index.html b/v1.x/docs/concepts/policies/index.html
index f639d2e7c..4049f5c2f 100644
--- a/v1.x/docs/concepts/policies/index.html
+++ b/v1.x/docs/concepts/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/pools/index.html b/v1.x/docs/concepts/pools/index.html
index cfa618f1f..e0339bb91 100644
--- a/v1.x/docs/concepts/pools/index.html
+++ b/v1.x/docs/concepts/pools/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/replication/index.html b/v1.x/docs/concepts/replication/index.html
index 959a95d99..8db57e251 100644
--- a/v1.x/docs/concepts/replication/index.html
+++ b/v1.x/docs/concepts/replication/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/rules/index.html b/v1.x/docs/concepts/rules/index.html
index bb830c606..cb00cab8b 100644
--- a/v1.x/docs/concepts/rules/index.html
+++ b/v1.x/docs/concepts/rules/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/sharedfs/index.html b/v1.x/docs/concepts/sharedfs/index.html
index e258e0cd7..d95db597d 100644
--- a/v1.x/docs/concepts/sharedfs/index.html
+++ b/v1.x/docs/concepts/sharedfs/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/concepts/volumes/index.html b/v1.x/docs/concepts/volumes/index.html
index 99753fba7..8a0ec08e6 100644
--- a/v1.x/docs/concepts/volumes/index.html
+++ b/v1.x/docs/concepts/volumes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/index.html b/v1.x/docs/index.html
index 1ad618b43..173916548 100644
--- a/v1.x/docs/index.html
+++ b/v1.x/docs/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/index.xml b/v1.x/docs/index.xml
index da99d753f..a55f8bb20 100644
--- a/v1.x/docs/index.xml
+++ b/v1.x/docs/index.xml
@@ -1733,6 +1733,11 @@ following object exists.</p>
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -1765,7 +1770,7 @@ following object exists.</p>
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -1878,6 +1883,11 @@ following object exists.</p>
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v1.x/docs/install/index.html b/v1.x/docs/install/index.html
index 0559a14d1..4ced1627d 100644
--- a/v1.x/docs/install/index.html
+++ b/v1.x/docs/install/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/install/kubernetes/index.html b/v1.x/docs/install/kubernetes/index.html
index da0a3d455..11ed3f889 100644
--- a/v1.x/docs/install/kubernetes/index.html
+++ b/v1.x/docs/install/kubernetes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/install/openshift/index.html b/v1.x/docs/install/openshift/index.html
index c5e6cea07..df9d7dae1 100644
--- a/v1.x/docs/install/openshift/index.html
+++ b/v1.x/docs/install/openshift/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/install/rancher/index.html b/v1.x/docs/install/rancher/index.html
index 5888cbee0..18f5d3ddf 100644
--- a/v1.x/docs/install/rancher/index.html
+++ b/v1.x/docs/install/rancher/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/introduction/index.html b/v1.x/docs/introduction/index.html
index 29a748d1c..72aeecce6 100644
--- a/v1.x/docs/introduction/index.html
+++ b/v1.x/docs/introduction/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/introduction/platforms/index.html b/v1.x/docs/introduction/platforms/index.html
index 3d281fdc1..3a7033b79 100644
--- a/v1.x/docs/introduction/platforms/index.html
+++ b/v1.x/docs/introduction/platforms/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/introduction/quickstart/index.html b/v1.x/docs/introduction/quickstart/index.html
index c07ea0f35..1bcafb1fa 100644
--- a/v1.x/docs/introduction/quickstart/index.html
+++ b/v1.x/docs/introduction/quickstart/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/decommission-nodes/index.html b/v1.x/docs/operations/decommission-nodes/index.html
index 0f23f9be2..50649ac24 100644
--- a/v1.x/docs/operations/decommission-nodes/index.html
+++ b/v1.x/docs/operations/decommission-nodes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/encrypted-volumes/index.html b/v1.x/docs/operations/encrypted-volumes/index.html
index 3aa5f114b..408935f6d 100644
--- a/v1.x/docs/operations/encrypted-volumes/index.html
+++ b/v1.x/docs/operations/encrypted-volumes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/external-etcd/etcd-tls/index.html b/v1.x/docs/operations/external-etcd/etcd-tls/index.html
index 4875e3cbc..cb36055e7 100644
--- a/v1.x/docs/operations/external-etcd/etcd-tls/index.html
+++ b/v1.x/docs/operations/external-etcd/etcd-tls/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/external-etcd/index.html b/v1.x/docs/operations/external-etcd/index.html
index f44a92c02..cfa37aec6 100644
--- a/v1.x/docs/operations/external-etcd/index.html
+++ b/v1.x/docs/operations/external-etcd/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/fencing/index.html b/v1.x/docs/operations/fencing/index.html
index 948172df6..229cda410 100644
--- a/v1.x/docs/operations/fencing/index.html
+++ b/v1.x/docs/operations/fencing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/firstpvc/index.html b/v1.x/docs/operations/firstpvc/index.html
index cb90284b7..2459310f4 100644
--- a/v1.x/docs/operations/firstpvc/index.html
+++ b/v1.x/docs/operations/firstpvc/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/health/index.html b/v1.x/docs/operations/health/index.html
index 8fd471dc4..f5b4be629 100644
--- a/v1.x/docs/operations/health/index.html
+++ b/v1.x/docs/operations/health/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/index.html b/v1.x/docs/operations/index.html
index 915530d92..d01766bcc 100644
--- a/v1.x/docs/operations/index.html
+++ b/v1.x/docs/operations/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/licensing/index.html b/v1.x/docs/operations/licensing/index.html
index 0d0c4b254..4e480344d 100644
--- a/v1.x/docs/operations/licensing/index.html
+++ b/v1.x/docs/operations/licensing/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/maintenance/index.html b/v1.x/docs/operations/maintenance/index.html
index d44f96bb0..0f95e3a07 100644
--- a/v1.x/docs/operations/maintenance/index.html
+++ b/v1.x/docs/operations/maintenance/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/managing-host-storage/index.html b/v1.x/docs/operations/managing-host-storage/index.html
index c0189f90a..c3da50b0a 100644
--- a/v1.x/docs/operations/managing-host-storage/index.html
+++ b/v1.x/docs/operations/managing-host-storage/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/monitoring/index.html b/v1.x/docs/operations/monitoring/index.html
index 0818b8be7..8122cf05a 100644
--- a/v1.x/docs/operations/monitoring/index.html
+++ b/v1.x/docs/operations/monitoring/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/monitoring/prometheus-setup/index.html b/v1.x/docs/operations/monitoring/prometheus-setup/index.html
index bf6d12d54..b6735d78b 100644
--- a/v1.x/docs/operations/monitoring/prometheus-setup/index.html
+++ b/v1.x/docs/operations/monitoring/prometheus-setup/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/namespaces/index.html b/v1.x/docs/operations/namespaces/index.html
index 4a6d28be7..46f9cef6b 100644
--- a/v1.x/docs/operations/namespaces/index.html
+++ b/v1.x/docs/operations/namespaces/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/overcommitment/index.html b/v1.x/docs/operations/overcommitment/index.html
index 44d05289e..735487497 100644
--- a/v1.x/docs/operations/overcommitment/index.html
+++ b/v1.x/docs/operations/overcommitment/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/policies/index.html b/v1.x/docs/operations/policies/index.html
index 4819cdafd..58d76cdde 100644
--- a/v1.x/docs/operations/policies/index.html
+++ b/v1.x/docs/operations/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/rules/index.html b/v1.x/docs/operations/rules/index.html
index b8cd942e5..24705ad77 100644
--- a/v1.x/docs/operations/rules/index.html
+++ b/v1.x/docs/operations/rules/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/sharedfs/index.html b/v1.x/docs/operations/sharedfs/index.html
index 9991a668c..5be63a579 100644
--- a/v1.x/docs/operations/sharedfs/index.html
+++ b/v1.x/docs/operations/sharedfs/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/troubleshooting/index.html b/v1.x/docs/operations/troubleshooting/index.html
index a5d9d9a3f..bdbfc5990 100644
--- a/v1.x/docs/operations/troubleshooting/index.html
+++ b/v1.x/docs/operations/troubleshooting/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/updates/index.html b/v1.x/docs/operations/updates/index.html
index 1bd66b3c2..08cb0c3b7 100644
--- a/v1.x/docs/operations/updates/index.html
+++ b/v1.x/docs/operations/updates/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/upgrades/index.html b/v1.x/docs/operations/upgrades/index.html
index d7f971c10..6d31122a7 100644
--- a/v1.x/docs/operations/upgrades/index.html
+++ b/v1.x/docs/operations/upgrades/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/operations/users/index.html b/v1.x/docs/operations/users/index.html
index e72dae266..b03543b4d 100644
--- a/v1.x/docs/operations/users/index.html
+++ b/v1.x/docs/operations/users/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/platforms/aws-eks/index.html b/v1.x/docs/platforms/aws-eks/index.html
index 1f2413a55..ca8a6d674 100644
--- a/v1.x/docs/platforms/aws-eks/index.html
+++ b/v1.x/docs/platforms/aws-eks/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/platforms/azure-aks/index.html b/v1.x/docs/platforms/azure-aks/index.html
index 910bdcfee..9007a305c 100644
--- a/v1.x/docs/platforms/azure-aks/index.html
+++ b/v1.x/docs/platforms/azure-aks/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/platforms/dockeree/index.html b/v1.x/docs/platforms/dockeree/index.html
index f66b22ccb..700be3fc5 100644
--- a/v1.x/docs/platforms/dockeree/index.html
+++ b/v1.x/docs/platforms/dockeree/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/platforms/index.html b/v1.x/docs/platforms/index.html
index 746cf9b51..9e41cfcf7 100644
--- a/v1.x/docs/platforms/index.html
+++ b/v1.x/docs/platforms/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/platforms/openshift/index.html b/v1.x/docs/platforms/openshift/index.html
index 58fb026a6..2f6577c85 100644
--- a/v1.x/docs/platforms/openshift/index.html
+++ b/v1.x/docs/platforms/openshift/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/platforms/rancher/index.html b/v1.x/docs/platforms/rancher/index.html
index 38085cfb7..228c1e11c 100644
--- a/v1.x/docs/platforms/rancher/index.html
+++ b/v1.x/docs/platforms/rancher/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/prerequisites/firewalls/index.html b/v1.x/docs/prerequisites/firewalls/index.html
index 17cb964d6..223ebcc0b 100644
--- a/v1.x/docs/prerequisites/firewalls/index.html
+++ b/v1.x/docs/prerequisites/firewalls/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/prerequisites/index.html b/v1.x/docs/prerequisites/index.html
index d45c3b23e..b2eabf035 100644
--- a/v1.x/docs/prerequisites/index.html
+++ b/v1.x/docs/prerequisites/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/prerequisites/mountpropagation/index.html b/v1.x/docs/prerequisites/mountpropagation/index.html
index 106a0c962..bb684aba2 100644
--- a/v1.x/docs/prerequisites/mountpropagation/index.html
+++ b/v1.x/docs/prerequisites/mountpropagation/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/prerequisites/systemconfiguration/index.html b/v1.x/docs/prerequisites/systemconfiguration/index.html
index 1a70e9dce..a1db7f6db 100644
--- a/v1.x/docs/prerequisites/systemconfiguration/index.html
+++ b/v1.x/docs/prerequisites/systemconfiguration/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/api/index.html b/v1.x/docs/reference/api/index.html
index 2f1aefade..8f16207ee 100644
--- a/v1.x/docs/reference/api/index.html
+++ b/v1.x/docs/reference/api/index.html
@@ -189,7 +189,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/cluster/index.html b/v1.x/docs/reference/cli/cluster/index.html
index 11cc66b69..cce20adc9 100644
--- a/v1.x/docs/reference/cli/cluster/index.html
+++ b/v1.x/docs/reference/cli/cluster/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/index.html b/v1.x/docs/reference/cli/index.html
index daea0bc6a..45f8e6ece 100644
--- a/v1.x/docs/reference/cli/index.html
+++ b/v1.x/docs/reference/cli/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/licence/index.html b/v1.x/docs/reference/cli/licence/index.html
index e5db44b61..0e8aad70e 100644
--- a/v1.x/docs/reference/cli/licence/index.html
+++ b/v1.x/docs/reference/cli/licence/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/login/index.html b/v1.x/docs/reference/cli/login/index.html
index f6c96292d..e0b749c79 100644
--- a/v1.x/docs/reference/cli/login/index.html
+++ b/v1.x/docs/reference/cli/login/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/logs/index.html b/v1.x/docs/reference/cli/logs/index.html
index 16eabf749..f9ef45127 100644
--- a/v1.x/docs/reference/cli/logs/index.html
+++ b/v1.x/docs/reference/cli/logs/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/namespace/index.html b/v1.x/docs/reference/cli/namespace/index.html
index 0c665987f..378849288 100644
--- a/v1.x/docs/reference/cli/namespace/index.html
+++ b/v1.x/docs/reference/cli/namespace/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/node/index.html b/v1.x/docs/reference/cli/node/index.html
index 3525b836b..d142f0677 100644
--- a/v1.x/docs/reference/cli/node/index.html
+++ b/v1.x/docs/reference/cli/node/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/policy/index.html b/v1.x/docs/reference/cli/policy/index.html
index cc9478bcb..d040f461b 100644
--- a/v1.x/docs/reference/cli/policy/index.html
+++ b/v1.x/docs/reference/cli/policy/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/pool/index.html b/v1.x/docs/reference/cli/pool/index.html
index b6b7e9d02..2814a5ef5 100644
--- a/v1.x/docs/reference/cli/pool/index.html
+++ b/v1.x/docs/reference/cli/pool/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/rule/index.html b/v1.x/docs/reference/cli/rule/index.html
index 7713d838a..080f27d4b 100644
--- a/v1.x/docs/reference/cli/rule/index.html
+++ b/v1.x/docs/reference/cli/rule/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/user/index.html b/v1.x/docs/reference/cli/user/index.html
index 5a95bc431..fa52e017c 100644
--- a/v1.x/docs/reference/cli/user/index.html
+++ b/v1.x/docs/reference/cli/user/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cli/volume/index.html b/v1.x/docs/reference/cli/volume/index.html
index 89c7e5beb..23cd6acfa 100644
--- a/v1.x/docs/reference/cli/volume/index.html
+++ b/v1.x/docs/reference/cli/volume/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cluster-operator/configuration/index.html b/v1.x/docs/reference/cluster-operator/configuration/index.html
index 3ac258759..e444b63e6 100644
--- a/v1.x/docs/reference/cluster-operator/configuration/index.html
+++ b/v1.x/docs/reference/cluster-operator/configuration/index.html
@@ -30,7 +30,7 @@
-
+
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -1251,6 +1251,11 @@ StorageOSCluster Resource Confi
false |
+images.apiManagerContainer |
+StorageOS API Manager container image |
+storageos/api-manager:v1.0.0 |
+
+
images.csiClusterDriverRegistrarContainer |
CSI Cluster Driver Registrar Container image |
quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 |
@@ -1283,7 +1288,7 @@ StorageOSCluster Resource Confi
images.initContainer |
StorageOS init container image |
-storageos/init:1.0.0 |
+storageos/init:2.1.0 |
images.kubeSchedulerContainer |
@@ -1396,6 +1401,11 @@ StorageOSCluster Resource Confi
/var/lib/kubelet/plugins/kubernetes.io~storageos |
+storageClassName |
+The name of the default StorageClass created for StorageOS volumes |
+fast |
+
+
tlsEtcdSecretRefName |
Secret containing etcd client certificates |
|
diff --git a/v1.x/docs/reference/cluster-operator/examples/index.html b/v1.x/docs/reference/cluster-operator/examples/index.html
index dd886ec86..a0b1a6714 100644
--- a/v1.x/docs/reference/cluster-operator/examples/index.html
+++ b/v1.x/docs/reference/cluster-operator/examples/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cluster-operator/index.html b/v1.x/docs/reference/cluster-operator/index.html
index 0f444987a..db4c0f956 100644
--- a/v1.x/docs/reference/cluster-operator/index.html
+++ b/v1.x/docs/reference/cluster-operator/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cluster-operator/index.xml b/v1.x/docs/reference/cluster-operator/index.xml
index 05e4447b3..4feed84ec 100644
--- a/v1.x/docs/reference/cluster-operator/index.xml
+++ b/v1.x/docs/reference/cluster-operator/index.xml
@@ -108,6 +108,11 @@
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -140,7 +145,7 @@
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -253,6 +258,11 @@
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v1.x/docs/reference/cluster-operator/install/index.html b/v1.x/docs/reference/cluster-operator/install/index.html
index 447fc9478..77bac7f4f 100644
--- a/v1.x/docs/reference/cluster-operator/install/index.html
+++ b/v1.x/docs/reference/cluster-operator/install/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/cluster-operator/upgrade/index.html b/v1.x/docs/reference/cluster-operator/upgrade/index.html
index f312dd7f0..0815de4bf 100644
--- a/v1.x/docs/reference/cluster-operator/upgrade/index.html
+++ b/v1.x/docs/reference/cluster-operator/upgrade/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/clusterdiscovery/index.html b/v1.x/docs/reference/clusterdiscovery/index.html
index 4aea26d55..d1f449381 100644
--- a/v1.x/docs/reference/clusterdiscovery/index.html
+++ b/v1.x/docs/reference/clusterdiscovery/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/contributing/index.html b/v1.x/docs/reference/contributing/index.html
index 7c401798a..bc7075986 100644
--- a/v1.x/docs/reference/contributing/index.html
+++ b/v1.x/docs/reference/contributing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/envvars/index.html b/v1.x/docs/reference/envvars/index.html
index 393a1e30d..1fb043cff 100644
--- a/v1.x/docs/reference/envvars/index.html
+++ b/v1.x/docs/reference/envvars/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/filesystems/index.html b/v1.x/docs/reference/filesystems/index.html
index 28988da5f..8d6c98384 100644
--- a/v1.x/docs/reference/filesystems/index.html
+++ b/v1.x/docs/reference/filesystems/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/gui/index.html b/v1.x/docs/reference/gui/index.html
index 87eafbd6e..5fb620603 100644
--- a/v1.x/docs/reference/gui/index.html
+++ b/v1.x/docs/reference/gui/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/index.html b/v1.x/docs/reference/index.html
index 47d91c275..4d8076540 100644
--- a/v1.x/docs/reference/index.html
+++ b/v1.x/docs/reference/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/init-container/index.html b/v1.x/docs/reference/init-container/index.html
index d98d30970..7c79a4b0a 100644
--- a/v1.x/docs/reference/init-container/index.html
+++ b/v1.x/docs/reference/init-container/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/labels/index.html b/v1.x/docs/reference/labels/index.html
index 81c49384e..68975f477 100644
--- a/v1.x/docs/reference/labels/index.html
+++ b/v1.x/docs/reference/labels/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/open_source_attribution/index.html b/v1.x/docs/reference/open_source_attribution/index.html
index 36d70939d..647abc66c 100644
--- a/v1.x/docs/reference/open_source_attribution/index.html
+++ b/v1.x/docs/reference/open_source_attribution/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/prometheus/index.html b/v1.x/docs/reference/prometheus/index.html
index bbc8f4842..99fa2d597 100644
--- a/v1.x/docs/reference/prometheus/index.html
+++ b/v1.x/docs/reference/prometheus/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/release_notes/index.html b/v1.x/docs/reference/release_notes/index.html
index ef876843e..24e7c5d91 100644
--- a/v1.x/docs/reference/release_notes/index.html
+++ b/v1.x/docs/reference/release_notes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/scheduler/admission-controller/index.html b/v1.x/docs/reference/scheduler/admission-controller/index.html
index 807d96335..fbd6352f3 100644
--- a/v1.x/docs/reference/scheduler/admission-controller/index.html
+++ b/v1.x/docs/reference/scheduler/admission-controller/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/scheduler/examples/index.html b/v1.x/docs/reference/scheduler/examples/index.html
index 8d7308c12..5b176dbc5 100644
--- a/v1.x/docs/reference/scheduler/examples/index.html
+++ b/v1.x/docs/reference/scheduler/examples/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/scheduler/index.html b/v1.x/docs/reference/scheduler/index.html
index 4ee7decb7..b98dcf02b 100644
--- a/v1.x/docs/reference/scheduler/index.html
+++ b/v1.x/docs/reference/scheduler/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/reference/telemetry/index.html b/v1.x/docs/reference/telemetry/index.html
index 1d2f52445..9d3f77109 100644
--- a/v1.x/docs/reference/telemetry/index.html
+++ b/v1.x/docs/reference/telemetry/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/self-eval/index.html b/v1.x/docs/self-eval/index.html
index ca255a8ed..e39ab97d6 100644
--- a/v1.x/docs/self-eval/index.html
+++ b/v1.x/docs/self-eval/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/support/index.html b/v1.x/docs/support/index.html
index 71a00aee7..d90153b59 100644
--- a/v1.x/docs/support/index.html
+++ b/v1.x/docs/support/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/backups/index.html b/v1.x/docs/usecases/backups/index.html
index 61c7dd6b6..714dfcbba 100644
--- a/v1.x/docs/usecases/backups/index.html
+++ b/v1.x/docs/usecases/backups/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/cassandra/index.html b/v1.x/docs/usecases/cassandra/index.html
index e727a4909..2e44fa08b 100644
--- a/v1.x/docs/usecases/cassandra/index.html
+++ b/v1.x/docs/usecases/cassandra/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/elasticsearch/index.html b/v1.x/docs/usecases/elasticsearch/index.html
index 84b7c485b..bcafaaff6 100644
--- a/v1.x/docs/usecases/elasticsearch/index.html
+++ b/v1.x/docs/usecases/elasticsearch/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/index.html b/v1.x/docs/usecases/index.html
index 427918917..1daf3a9bb 100644
--- a/v1.x/docs/usecases/index.html
+++ b/v1.x/docs/usecases/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/jenkins/index.html b/v1.x/docs/usecases/jenkins/index.html
index 2723780f2..64a2c53fd 100644
--- a/v1.x/docs/usecases/jenkins/index.html
+++ b/v1.x/docs/usecases/jenkins/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/kafka/index.html b/v1.x/docs/usecases/kafka/index.html
index a7ed739c8..453b23516 100644
--- a/v1.x/docs/usecases/kafka/index.html
+++ b/v1.x/docs/usecases/kafka/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/kubevirt/index.html b/v1.x/docs/usecases/kubevirt/index.html
index 11e76117b..c7dd8aa80 100644
--- a/v1.x/docs/usecases/kubevirt/index.html
+++ b/v1.x/docs/usecases/kubevirt/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/mssql/index.html b/v1.x/docs/usecases/mssql/index.html
index 8350c2aac..e0da1af42 100644
--- a/v1.x/docs/usecases/mssql/index.html
+++ b/v1.x/docs/usecases/mssql/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/mysql/index.html b/v1.x/docs/usecases/mysql/index.html
index a1d386cf4..a95b8cba2 100644
--- a/v1.x/docs/usecases/mysql/index.html
+++ b/v1.x/docs/usecases/mysql/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/nginx/index.html b/v1.x/docs/usecases/nginx/index.html
index dbd218f4f..1a4202d4d 100644
--- a/v1.x/docs/usecases/nginx/index.html
+++ b/v1.x/docs/usecases/nginx/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/postgres/index.html b/v1.x/docs/usecases/postgres/index.html
index 9a2cd9633..db2acbf07 100644
--- a/v1.x/docs/usecases/postgres/index.html
+++ b/v1.x/docs/usecases/postgres/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/redis/index.html b/v1.x/docs/usecases/redis/index.html
index 73f27c0ad..415de9eb7 100644
--- a/v1.x/docs/usecases/redis/index.html
+++ b/v1.x/docs/usecases/redis/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/docs/usecases/zookeeper/index.html b/v1.x/docs/usecases/zookeeper/index.html
index 163192b00..e187b5a27 100644
--- a/v1.x/docs/usecases/zookeeper/index.html
+++ b/v1.x/docs/usecases/zookeeper/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/index.html b/v1.x/index.html
index a756a47f0..79f0a3da7 100644
--- a/v1.x/index.html
+++ b/v1.x/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/index.json b/v1.x/index.json
index 8f2832eab..5ff7d68cd 100644
--- a/v1.x/index.json
+++ b/v1.x/index.json
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json b/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json
new file mode 100644
index 000000000..212140e2d
--- /dev/null
+++ b/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json
@@ -0,0 +1 @@
+[{"body":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher OpenShift ","excerpt":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher …","ref":"/v1.x/docs/reference/cluster-operator/install/","title":"Install"},{"body":"Current Version The lastest version of StorageOS v1 is 1.5.4\nSections Our documentation is arranged into sections, accessible from the navigation bar on the left.\nIntroduction - Quickstart and support information.\nConcepts - Architectural and deep technical information.\nPrerequisites - We require certain prerequisites to be met for the product to function correctly. Please read these carefully and ensure they are implemented.\nPlatforms - Due to differences in the various orchestrators that StorageOS can run under, we list install guides and other platform specific operations here.\nOperations - Platform agnostic operations.\nUse Cases - A set of examples to get up and running with StorageOS quickly.\nReference - Information on our GUI, CLI, and other important information.\n","excerpt":"Current Version The lastest version of StorageOS v1 is 1.5.4\nSections Our documentation is arranged …","ref":"/v1.x/docs/introduction/","title":"Introduction"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed service such as EKS, AKS, GKE, DO or DockerEE platform can use the following Kubernetes guide to install StorageOS. Ensure the use of CSI for these platforms.\n 1.17 1.16 1.15 1.14 1.13 1.12 Install StorageOS on Kubernetes 1.17 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n Install StorageOS on Kubernetes 1.16 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n Install StorageOS on Kubernetes 1.15 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n Install StorageOS on Kubernetes 1.14 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n Install StorageOS on Kubernetes 1.13 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n Install StorageOS on Kubernetes 1.12 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed …","ref":"/v1.x/docs/install/kubernetes/","title":"Kubernetes"},{"body":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source implementation of the SCSI target.\nWe require the following modules to be loaded:\n target_core_mod tcm_loop target_core_file configfs target_core_user uio N.B. Other applications utilising TCMU cannot be run concurrently with StorageOS. Doing so may result in corruption of data. On startup, StorageOS will detect if other applications are using TCMU and fall back to FUSE. However if StorageOS is started first there is no mechanism for StorageOS to fallback to FUSE if another application begins to use TCMU. TCMU can be disabled using the DISABLE_TCMU StorageOSCluster spec parameter.\n Depending on the distribution, the modules are shipped as part of the base kernel package or as part of a kernel extras package which needs to be installed.\nDistribution Specifics The following distributions are supported by default:\n RHEL 7.5 CentOS 7 Debian 9 Ubuntu Azure RancherOS - Note CSI is not supported on RancherOS Ubuntu 16.04/18.04 requires the installation of additional packages.\n N.B. Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not provide the necessary linux-image-extra package - see below for more information\n Ubuntu Package Installation Ubuntu 16.04/18.04 Generic and Ubuntu 16.04 GCE require extra packages:\nUbuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Ubuntu With AWS Or GCE Kernels Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not yet provide the linux-image-extra package. As such you should either use Debian, CentOS or RHEL, or install the non-cloud-provider optimised Ubuntu kernel.\nInstalling the non-cloud-provider optimised Ubuntu kernel is something that should only be done with full understanding of potential ramifications.\nsudo apt -y update sudo apt install -y linux-virtual linux-image-extra-virtual sudo apt purge -y linux*aws # Reboot the machine sudo shutdown -r now Automatic Configuration Once required kernel modules are installed on the system, for convenience we provide a container which will ensure the appropriate modules are loaded and ready for use at runtime. On Docker installations, you will need to run the init container prior to starting StorageOS. Our installation guides for Kubernetes and OpenShift include this step.\n# Load the required kernel modules. The Kubernetes and OpenShift installations include this step. docker run --name enable_lio \\ --privileged \\ --rm \\ --cap-add=SYS_ADMIN \\ -v /lib/modules:/lib/modules \\ -v /sys:/sys:rshared \\ storageos/init:0.2 Manual Configuration For those wishing to manage their own kernel configuration, rather than using the init container, perform the following steps:\n Ensure kernel modules are all loaded per list above Ensure configfs is loaded and mounted at /sys/kernel/config ","excerpt":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source …","ref":"/v1.x/docs/prerequisites/systemconfiguration/","title":"System Configuration"},{"body":"StorageOS implements a MutatingAdmissionWebhook Admission Controller to ensure that Pods using StorageOS Volumes use the storageos-scheduler. An admission controller intercepts requests to the Kubernetes API server prior to persistence of the object, but after the request is authenticated and authorized.\nThe Admission Controller is responsible for mutating the PodSpec at creation time to populate the PodSpec.schedulerName field with the name of the StorageOS Scheduler - storageos-scheduler.\nDuring Pod creation, Kubernetes sends a web request to the StorageOS WebHook with the Pod specification. The PodSpec is only altered to use the StorageOS scheduler if the Pod uses a StorageOS volume.\nWeb Server The Web Server hosting the web hook is executed in the StorageOS Cluster Operator. Since only HTTPS requests are allowed, the Operator generates a self-signed x509 certificate every time it starts. The Cluster Operator will also renew certificates upon expiry (certs are valid for one year).\nThere is no manual intervention required regarding the SSL configuration as the setup is completely transparent between StorageOS and Kubernetes.\nSkipping Mutation To avoid scheduler mutation, the storageos.com/scheduler=false annotation can be added to resources that use StorageOS volumes.\nWhen using StatefulSets the annotation can be set on the spec.template.metadata.annotations field.\napiVersion:apps/v1kind:StatefulSetspec:...template:metadata:annotations:storageos.com/scheduler:\u0026#34;false\u0026#34;# N.B. the value must be a string and not a booleanWhen using Pods the annotation is set on the metadata.annotations field.\napiVersion:v1kind:Podmetadata:...annotations:storageos.com/scheduler:\u0026#34;false\u0026#34;# N.B. the value must be a string and not a boolean...Compatibility The Admission Controller doesn\u0026rsquo;t need to be enabled at Kubernetes cluster bootstrap time because it is a Dynamic Admission Controller. Hence, any cluster that has the MutatingAdmissionWebhook enabled is supported. Most Kubernetes cluster enable the Webhook admission controller by default.\nThe MutatingAdmissionWebhook is available from Kubernetes v1.13.\nYou can check your Kubernetes cluster compatibility by checking if the following object exists.\nkubectl api-versions | grep admissionregistration.k8s.io ","excerpt":"StorageOS implements a MutatingAdmissionWebhook Admission Controller to ensure that Pods using …","ref":"/v1.x/docs/reference/scheduler/admission-controller/","title":"Admission Controller"},{"body":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters of the StorageOSCluster custom resource and their default values.\n Parameter Description Default csi.deploymentStrategy CSI helper deployment strategy (statefulset or deployment) statefulset csi.enable Enable CSI setup false csi.enableControllerPublishCreds Enable CSI controller publish credentials false csi.enableNodePublishCreds Enable CSI node publish credentials false csi.enableProvisionCreds Enable CSI provision credentials false debug Enable debug mode for all the cluster nodes false disableFencing Disable Pod fencing false disableScheduler Disable StorageOS scheduler false disableTCMU Disable TCMU to allow co-existence with other TCMU users. Disabling TCMU degrades performance false disableTelemetry Disable telemetry reports false forceTCMU Forces TCMU to be enabled or causes StorageOS to abort startup false images.apiManagerContainer StorageOS API Manager container image storageos/api-manager:v1.0.0 images.csiClusterDriverRegistrarContainer CSI Cluster Driver Registrar Container image quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 images.csiExternalAttacherContainer CSI External Attacher Container image quay.io/k8scsi/csi-attacher:v1.0.1 images.csiExternalProvisionerContainer CSI External Provisioner Container image storageos/csi-provisioner:v1.0.1 ìmages.csiLivenessProbeContainer CSI Liveness Probe Container Image quay.io/k8scsi/livenessprobe:v1.0.1 images.csiNodeDriverRegistrarContainer CSI Node Driver Registrar Container image quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 images.hyperkubeContainer Deprecated field - HyperKube Container image Default dependent on Scheduler version images.initContainer StorageOS init container image storageos/init:2.1.0 images.kubeSchedulerContainer Kube scheduler container image Default dependent on Scheduler version images.nfsContainer StorageOS nfs container image storageos/nfs:1.0.0 images.nodeContainer StorageOS node container image storageos/node:1.5.4 ingress.annotations Annotations of the ingress used by the cluster ingress.enable Enable ingress for the cluster false ingress.hostname Hostname to be used in cluster ingress storageos.local ingress.tls Enable TLS for the ingress false k8sDistro The name of the Kubernetes distribution is use, e.g. rancher or eks kvBackend.address Comma-separated list of addresses of external key-value store. (1.2.3.4:2379,2.3.4.5:2379) kvBackend.backend (v2 deprecated) Name of the key-value store to use. Set to etcd for external key-value store. embedded namespace Namespace where storageos cluster resources are created kube-system nodeSelectorTerms Set node selector for storageos pod placement pause Pause the operator for cluster maintenance false resources Set resource requirements for the containers secretRefName Reference name of storageos secret secretRefNamespace Namespace of storageos secret service.annotations Annotations of the Service used by the cluster service.externalPort External port of the Service used by the cluster 5705 service.internalPort Internal port of the Service used by the cluster 5705 service.name Name of the Service used by the cluster storageos service.type Type of the Service used by the cluster ClusterIP sharedDir Path to be shared with kubelet container when deployed as a pod /var/lib/kubelet/plugins/kubernetes.io~storageos storageClassName The name of the default StorageClass created for StorageOS volumes fast tlsEtcdSecretRefName Secret containing etcd client certificates tlsEtcdSecretRefNamespace Namespace of the tlsEtcdSecretRefName tolerations Set pod tolerations for storageos pod placement ","excerpt":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters …","ref":"/v1.x/docs/reference/cluster-operator/configuration/","title":"Configuration"},{"body":" 4.x 3.11 3.9 Install StorageOS on OpenShift v4 Make sure the prerequisites for StorageOS are satisfied before proceeding.\n If you have installed OpenShift 4.3 in AWS ensure that the requisite ports are opened for the worker nodes\u0026rsquo; security group.\n Installing StorageOS on OpenShift 4.3 has fewer prerequisites as compared to previous OpenShift 3.x versions.\nThe recommended way to run StorageOS on OpenShift 4.3 is to deploy the StorageOS Cluster Operator using the OperatorHub and bootstrap StorageOS using a Custom Resource.\nOptions:\n OperatorHub (Recommended) Manual install OperatorHub install StorageOS has a RedHat OpenShift certified operator in the OpenShift OperatorHub. You can install the StorageOS operator through the OperatorHub.\n Select the OperatorHub from the Catalog sub menu and search for StorageOS\n Select StorageOS and click install\n Create the Operator subscription by clicking subscribe\n Wait until the Upgrade Status shows 1 installed\n Create a secret containing an apiUsername and an apiPassword key. The username and password defined in the secret will be used to authenticate when using the StorageOS CLI and GUI. Take note of which project you created the secret in.\n Go to Installed Operators and select the StorageOS operator. Select StorageOS Cluster and create a StorageOS cluster.\n The StorageOS cluster resource describes the StorageOS cluster that will be created. The secretRefName and secretRefNamespace should reference the secret containing the apiUsername and apiPassword that was previously created.\nAdditional spec parameters are available on the Cluster Operator configuration page.\napiVersion: storageos.com/v1 kind: StorageOSCluster metadata: name: storageos namespace: openshift-operators spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference the Secret created in the previous step secretRefNamespace: \u0026#34;openshift-operators\u0026#34; # Namespace of the Secret created in the previous step namespace: kube-system csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; k8sDistro: \u0026#34;openshift\u0026#34; Verify that the StorageOS Resource enters a running state.\n Set SELinux Permissions\nThe StorageOS CSI helper needs to mount a CSI Socket into the container so on each node add the svirt_sandbox_file_t flag to the CSI socket directory and CSI socket.\nchcon -Rt svirt_sandbox_file_t /var/lib/kubelet/plugins_registry/storageos If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\nManual install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 5. Set SELinux Permissions The StorageOS CSI helper needs to mount a CSI Socket into the container so on each node add the svirt_sandbox_file_t flag to the CSI socket directory and CSI socket.\nchcon -Rt svirt_sandbox_file_t /var/lib/kubelet/plugins_registry/storageos Install StorageOS on OpenShift 3.11 The recommended way to run StorageOS on an OpenShift 3.11 cluster is to deploy the StorageOS Cluster Operator and bootstrap StorageOS using a Custom Resource.\nPrerequisites Ensure any firewalls permit the appropriate ports.\n If your cluster enables SELinux, add the following permissions for each of the nodes that run StorageOS. bash setsebool -P virt_sandbox_use_fusefs on setsebool -P virt_use_fusefs on\n The -P option makes the change persistent after reboots.\n Ensure that your docker installation has mount propagation enabled per our mount propagation prerequisites.\n Enable the MountPropagation flag by appending feature gates to the API and controller (you can apply these changes using the Ansible Playbooks)\n Note: If you are using atomic installation rather than origin, the location of the yaml config files and service names might change.\n Add to the KubernetesMasterConfig section (/etc/origin/master/master-config.yaml):\nkubernetesMasterConfig: apiServerArguments: feature-gates: - MountPropagation=true controllerArguments: feature-gates: - MountPropagation=true Add to the feature-gates to the kubelet arguments (/etc/origin/node/node-config.yaml):\nkubeletArguments: feature-gates: - MountPropagation=true Warning: Restarting OpenShift services can cause downtime in the cluster.\n Restart services in the MasterNode/s\nmaster-restart api master-restart controllers # Restart kubelet systemctl restart atomic-openshift-node.service Restart service in all Nodes\n# Restart kubelet systemctl restart atomic-openshift-node.service Install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version disableScheduler: true resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-daemonset-75f6c 1/1 Running 0 3m storageos-daemonset-czbqx 1/1 Running 0 3m storageos-daemonset-zv4tq 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n Install StorageOS on OpenShift 3.9 The recommended way to run StorageOS on an OpenShift 3.9 cluster is to deploy the StorageOS Cluster Operator and bootstrap StorageOS using a Custom Resource.\nPrerequisites Ensure any firewalls permit the appropriate ports.\n If your cluster enables SELinux, add the following permissions for each of the nodes that run StorageOS. bash setsebool -P virt_sandbox_use_fusefs on setsebool -P virt_use_fusefs on\n The -P option makes the change persistent after reboots.\n Ensure that your docker installation has mount propagation enabled per our mount propagation prerequisites.\n Enable the MountPropagation flag by appending feature gates to the API and controller (you can apply these changes using the Ansible Playbooks)\n Note: If you are using atomic installation rather than origin, the location of the yaml config files and service names might change.\n Add to the KubernetesMasterConfig section (/etc/origin/master/master-config.yaml):\nkubernetesMasterConfig: apiServerArguments: feature-gates: - MountPropagation=true controllerArguments: feature-gates: - MountPropagation=true Add to the feature-gates to the kubelet arguments (/etc/origin/node/node-config.yaml):\nkubeletArguments: feature-gates: - MountPropagation=true Warning: Restarting OpenShift services can cause downtime in the cluster.\n Restart services in the MasterNode origin-master-api.service, origin-master-controllers.service and origin-node.service Restart service in all Nodes origin-node.service Usually through systemctl restart (origin-node.service|atomic-openshift-node.service)\n Install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version disableScheduler: true resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-daemonset-75f6c 1/1 Running 0 3m storageos-daemonset-czbqx 1/1 Running 0 3m storageos-daemonset-zv4tq 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"4.x 3.11 3.9 Install StorageOS on OpenShift v4 Make sure the prerequisites for StorageOS are …","ref":"/v1.x/docs/install/openshift/","title":"OpenShift"},{"body":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all nodes in the cluster:\n Port Number TCP/UDP Use 5701 tcp gRPC 5702 tcp Prometheus 5703 tcp DirectFS 5704 tcp Dataplane health check 5705 tcp REST API 5706 tcp ETCD service 5707 tcp ETCD service 5708 tcp NATS service 5709 tcp NATS service 5710 tcp NATS service 5711 tcp \u0026amp; udp Gossip service StorageOS also uses ephemeral ports to dial-out to these ports on other StorageOS nodes. For this reason, outgoing traffic should be enabled.\nFirewalls and VPS providers Some VPS providers (such as Digital Ocean) ship default firewall rulesets which must be updated to allow StorageOS to run. Some example rules are shown below - modify to taste.\nUFW For distributions using UFW, such as RHEL and derivatives:\nufw default allow outgoing ufw allow 5701:5711/tcp ufw allow 5711/udp Firewalld For distributions that enable firewalld to control iptables such as some installations of OpenShift.\nfirewall-cmd --permanent --new-service=storageos firewall-cmd --permanent --service=storageos --add-port=5700-5800/tcp firewall-cmd --add-service=storageos --zone=public --permanent firewall-cmd --reload Iptables For those using plain iptables:\n# Inbound traffic iptables -I INPUT -i lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I INPUT -m state --state ESTABLISHED,RELATED -m comment --comment \u0026#39;Permit established traffic\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 5701:5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p udp --dport 5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT # Outbound traffic iptables -I OUTPUT -o lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I OUTPUT -d 0.0.0.0/0 -m comment --comment \u0026#39;Permit outbound traffic\u0026#39; -j ACCEPT Please ensure that the iptables rules you have added above come before any default DROP or REJECT rules.\n","excerpt":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all …","ref":"/v1.x/docs/prerequisites/firewalls/","title":"Firewalls"},{"body":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and Password in base64 encoding.\nkubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z END This example contains a default password, for production installations, use a unique, strong password.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Create a cluster-config.yaml according to your needs from the examples below.\nkubectl create -f cluster-config.yaml Note that StorageOS will be deployed in spec.namespace (storageos by default), irrespective of what NameSpace the CR is defined in.\n Examples You can checkout all the parameters configurable in the configuration page.\n All examples must reference the storageos-api Secret.\nspec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference to the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret Check out Cluster Definition examples for full CR files.\nInstalling with an external etcd spec: kvBackend: address: \u0026#39;10.43.93.95:2379\u0026#39; # IP of the SVC that exposes ETCD # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can specify individual IPs of the etcd servers backend: \u0026#39;etcd\u0026#39; Installing to a subset of nodes In this case we select nodes that are workers. To make sure that StorageOS doesn\u0026rsquo;t start in Master nodes.\nYou can see the labels in the nodes by kubectl get node --show-labels.\nspec: nodeSelectorTerms: - matchExpressions: - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; operator: In values: - \u0026#34;true\u0026#34; # OpenShift uses \u0026#34;node-role.kubernetes.io/compute=true\u0026#34; # Rancher uses \u0026#34;node-role.kubernetes.io/worker=true\u0026#34; # Kops uses \u0026#34;node-role.kubernetes.io/node=\u0026#34; Different provisioners and Kubernetes distributions use node labels differently to specify master vs workers. Node Taints are not enough to make sure StorageOS doesn\u0026rsquo;t start in a node. The JOIN variable is defined by the operator by selecting all the nodes that match the nodeSelectorTerms.\n Enabling CSI spec: csi: enable: true deploymentStrategy: deployment # enableProvisionCreds: false # enableControllerPublishCreds: false # enableNodePublishCreds: false The credentials must be defined in the storageos-api Secret\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;default\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # Add base64 encoded creds below for CSI credentials. # csiProvisionUsername: # csiProvisionPassword: # csiControllerPublishUsername: # csiControllerPublishPassword: # csiNodePublishUsername: # csiNodePublishPassword: Specifying a shared directory for use with kubelet as a container spec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; Defining pod resource requests and reservations spec: resources: requests: memory: \u0026#34;256Mi\u0026#34; # cpu: \u0026#34;1\u0026#34; # limits: # memory: \u0026#34;4Gi\u0026#34; Limiting StorageOS can cause malfunction for IO to StorageOS volumes, therefore we do not currently recommend applying upper limits to resources for StorageOS pods.\n","excerpt":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and …","ref":"/v1.x/docs/reference/cluster-operator/examples/","title":"Cluster Operator examples"},{"body":"Scheduling modes There are two modes available to enforce pod locality for StorageOS Volumes. The mode is defined by labeling the Pods.\nLabels Locality Mode Label Description Preferred (Default) storageos.com/locality: preferred Best effort placement of the Pod along its StorageOS Volume Strict storageos.com/locality: strict Enforce placement of the Pod alongside the Primary Volume or do not schedule Preferred (Default) The Pod will be placed along with its data if possible. Otherwise, it will be placed along the volume replicas. If both scenarios are not possible, the Pod will start on a different node and StorageOS will grant accessibility to the data via network.\nYou can explicitly guarantee that mode by setting the label storageos.com/locality: \u0026quot;preferred\u0026quot;to the Pod.\nFor instance a Pod would set the labels as:\napiVersion: v1 kind: Pod metadata: name: pod labels: storageos.com/locality: \u0026#34;preferred\u0026#34; spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: persistent-volume # ----\u0026gt; StorageOS PVC Strict The Pod will be enforced to be placed along its data. Hence, in the same node as the Primary Volume. In case of that not being possible, the Pod will remain in pending state until that premise is fulfilled.\nYou can enable this mode by setting the label storageos.com/locality: \u0026quot;strict\u0026quot; to the Pod.\nFor instance an StatefulSet would set the labels as:\napiVersion: apps/v1 kind: StatefulSet metadata: name: mysql spec: selector: matchLabels: app: mysql storageos.com/locality: \u0026#34;strict\u0026#34; # Matching label to the template serviceName: mysql replicas: 1 template: metadata: labels: app: mysql storageos.com/locality: \u0026#34;strict\u0026#34; # Inherited label by the Pod spec: serviceAccountName: mysql containers: - name: mysql image: mysql:5.7 ... volumeClaimTemplates: - metadata: name: data labels: app: mysql spec: accessModes: [\u0026#34;ReadWriteOnce\u0026#34;] storageClassName: \u0026#34;storageos\u0026#34; # StorageClass for your cluster resources: requests: storage: 50Gi Explicit SchedulerName It is not necessary to explicitly set the SchedulerName as the Admission Controller automatically populates the PodSpec field. Set the SchedulerName in your manifests, manually, only if you disable or can\u0026rsquo;t execute the StorageOS Admission Controller.\n Kubernetes allows the use of different schedulers by setting the field .spec.schedulerName: storageos-scheduler.\nFor instance the Pod manifest utilising the StorageOS scheduler would look as follows:\napiVersion: v1 kind: Pod metadata: name: d1 spec: schedulerName: storageos-scheduler # --\u0026gt; StorageOS Scheduler # No need if using Admission Controller # (enabled by default) containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: persistent-volume # ----\u0026gt; StorageOS PVC ","excerpt":"Scheduling modes There are two modes available to enforce pod locality for StorageOS Volumes. The …","ref":"/v1.x/docs/reference/scheduler/examples/","title":"Scheduler Examples"},{"body":" Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by default.\n StorageOS requires mount propagation enabled to present devices as volumes for containers (see linux kernel documentation here).\nCertain versions of docker ship with a systemd manifest with MountFlags set to \u0026lsquo;slave\u0026rsquo;, thus preventing StorageOS from working. This can be removed or set to \u0026lsquo;shared\u0026rsquo; with a systemd drop in:\nmkdir -p /etc/systemd/system/docker.service.d/ cat \u0026lt;\u0026lt;EOF \u0026gt; /etc/systemd/system/docker.service.d/mount_propagation_flags.conf [Service] MountFlags=shared EOF # systemctl daemon-reload # systemctl restart docker.service To confirm behaviour, the following command should run without error.\ndocker run -it --rm -v /mnt:/mnt:shared busybox sh -c /bin/date Orchestrators such as Kubernetes or OpenShift have their own ways of exposing this setting. Kubernetes 1.10 and OpenShift 3.10 have mount propagation enabled by default. Previous versions require that feature gates are enabled on the Kubernetes master\u0026rsquo;s controller-manager and apiserver services and in the kubelet service on each node.\nInstallations of orchestrators using Docker require that mount propagation is enabled for both.\nRefer to our installation pages for the orchestrators to see details on how to check and enable mount propagation where appropriate.\n","excerpt":"Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by …","ref":"/v1.x/docs/prerequisites/mountpropagation/","title":"Mount Propagation"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n StorageOS transparently supports Rancher deployments on CentOS, RHEL, Debian, Ubuntu or RancherOS (CSI is not supported on RancherOS) and can support other Linux distributions as detailed on the System Configuration page if the appropriate kernel modules are available.\n Catalog Advanced Rancher Catalog Install StorageOS is a Certified application in the Rancher Catalog. You can install StorageOS using the Rancher application install.\n Select the System project of your cluster\n Select the Apps tab and click Launch Search for StorageOS and click on the App Define the StorageOS cluster installation\n A generic configuration for StorageOS is preset using the default values in the form. The values in the form can be changed to customize the installation. To customize the installation further, set Install StorageOS Cluster to false and use a yaml definition for the StorageOSCluster Custom Resource.\n The following options are exposed by the form to allow some simple customization of the StorageOS installation.\n Cluster Operator namespace : The Kubernetes namespace where the StorageOS Cluster Operator controller and other resources will be created. Container Images : By default images are pulled from DockerHub, you can specify the image URLs when using private registries. Conditional bootstrap of StorageOS : Controls the automatic deployment of StorageOS after installing the Cluster Operator. If set to false, the Operator will be created, but the Custom Resource will not be applied to the cluster. Launch the operator and proceed to the section Custom Resource definition. For more information check the Operator documentation and CR examples. StorageOS namespace : The Kubernetes namespace where StorageOS will be installed. Installing into the kube-system namespace will add StorageOS to a priority class to ensure high priority resource allocation. Installing StorageOS with the priority class prevents StorageOS from being evicted during periods of resource contention. Username/Password : Default Username and Password for the admin account to be created at StorageOS bootstrap. A random password will be generated by leaving the field empty or clicking the Generate button. Key-value store setup : Connection and configuration details for an external Etcd cluster. StorageOS can use an external key-value store to hold configuration. Settings such as external etcd with TLS termination are available. Node Selectors and Tolerations : Control placement of StorageOS DaemonSet Pods. StorageOS will only be installed on the selected nodes. Can be used in conjunction with tolerations. Verify the cluster bootstrap has successfully completed\n Custom Resource definition If Install StorageOS Cluster was set to false, StorageOS will not be bootstrapped automatically. After the StorageOS Operator is installed, you can now create a Custom Resource that describes the StorageOS cluster.\n Select the System Workloads and Import YAML Create the Secret and CustomResource This is an example.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z # Define your own user and password apiPassword: c3RvcmFnZW9z --- apiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;storageos\u0026#34; spec: k8sDistro: \u0026#34;rancher\u0026#34; namespace: \u0026#34;kube-system\u0026#34; secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret csi: enable: true deploymentStrategy: \u0026#34;deployment\u0026#34; images: nodeContainer: \u0026#34;storageos/node:{{ site.latest_node_version }}\u0026#34; # StorageOS version # kvBackend: # address: \u0026#39;storageos-etcd-client.etcd:2379\u0026#39; # Example address, change for your etcd endpoint # backend: \u0026#39;etcd\u0026#39; sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet runs as a container resources: requests: memory: \u0026#34;512Mi\u0026#34; nodeSelectorTerms: - matchExpressions: - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; operator: In values: - \u0026#34;true\u0026#34; Additional spec parameters are available on the [Cluster Operator configuration]({%link _docs/reference/cluster-operator/configuration.md %}) page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Advanced installation This installation procedure is available in case the default method does not meet your requirements. The following procedure requires more steps to complete in comparison to the default procedure and requires adjustment of more installation parameters.\nThe StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The account defined in the secret is also used by Kubernetes to authenticate against the StorageOS API when installing with the native driver.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; namespace: \u0026#34;kube-system\u0026#34; images: nodeContainer: \u0026#34;storageos/node:1.5.4\u0026#34; # StorageOS version csi: enable: true deploymentStrategy: deployment resources: requests: memory: \u0026#34;512Mi\u0026#34; # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 6 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n StorageOS transparently …","ref":"/v1.x/docs/install/rancher/","title":"Rancher"},{"body":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following yaml manifest.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/1.5.4/storageos-operator.yaml When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n Upgrade StorageOS Operator using Helm If you have installed the StorageOS Operator using the Helm Chart, then you can upgrade the operator using the following commands.\n$ helm list NAME REVISION STATUS CHART APP VERSION NAMESPACE storageos-v1 4 DEPLOYED storageos-operator-0.2.11 1.3.0 storageos-operator $ helm repo update $ helm upgrade $NAME storageos/storageos-operator When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n ","excerpt":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following …","ref":"/v1.x/docs/reference/cluster-operator/upgrade/","title":"Cluster Operator Upgrade"},{"body":"","excerpt":"","ref":"/v1.x/docs/concepts/","title":"Concepts"},{"body":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a 64-bit architecture. Container Runtime Engine: CRI-O, Containerd or Docker 1.10+ with mount propagation enabled. The necessary ports should be open. See the ports and firewall settings. A mechanism for device presentation. Recommended: At least three nodes for replication and high availability. Install the storageos CLI. If using Helm, make sure the tiller ServiceAccount has enough privileges to create resources such as Namespaces, ClusterRoles, etc. For instance, following this installation procedure. System clocks synchronized using NTP or similar methods. While our distributed consensus algorithm does not require synchronised clocks, it does help to more easily correlate logs across multiple nodes. ","excerpt":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a …","ref":"/v1.x/docs/prerequisites/","title":"Prerequisites"},{"body":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different distributions such as OpenShift, Rancher, EKS, AKS, GKE, etc. The user can provide standard PVC definitions and StorageOS will dynamically provision matching volumes. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nKubernetes and StorageOS communicate with each other to perform actions such as creation, deletion or mounting of volumes. The CSI (Container Storage Interface) driver is the standard method of communication. Using CSI, Kubernetes and StorageOS communicate over a Unix domain socket.\nIt is recommended to use the CSI StorageOS installation from Kubernetes 1.13 onwards. Although the former communication procedure, the native driver, is still in use and StorageOS maintains support for it, its use is discouraged as it is a deprecated method, to be removed in newer versions of Kubernetes.\nCSI (Container Storage Interface) Note CSI is the standard method of communication that enables storage drivers for Kubernetes to release on their own schedule. The CSI standard allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI was released GA from Kubernetes 1.13. StorageOS v2 only supports the use of CSI as a storage driver. In addition, the StorageOS Cluster Operator handles the configuration of the CSI driver and its complexity by detecting the version of the Kubernetes installed.\nCheck out the status of the CSI release cycle in relation with Kubernetes in the CSI project page.\nCSI communication is fully supported by StorageOS if the cluster is deployed with any supported Linux Distribution.\nKubernetes Upgrades on Managed Services Managed services that support in place upgrades are fully supported. However, upgrading Kubernetes using green/blue deployments is not supported. This is because nodes are replaced rather than being upgraded. Any data stored on the nodes is lost when new nodes replace the previous ones.\nDocker Some managed Kubernetes platforms such as Azure AKS, enable the \u0026lsquo;Live-Restore\u0026rsquo; Docker feature, enabling containers to continue running while Docker is stopped or upgraded. This feature can cause nodes to hang while shutting-down or rebooting, as rather than going through an orderly shutdown, StorageOS (and other processes) are killed before the disks are synced and unmounted. Devices in this inaccessible state will log a warning similar to:\nTransport endpoint not connected To prevent this behaviour, we advise disabling this feature by setting\n{ ... \u0026quot;live-restore\u0026quot;: false } in /etc/docker/daemon.json.\nHere\u0026rsquo;s an example Ansible snippet that might be used to achieve this\n... - name: configure /etc/docker/daemon.json lineinfile: path: /etc/docker/daemon.json regexp: '^.*\u0026quot;live-restore\u0026quot;: true,$' line: ' \u0026quot;live-restore\u0026quot;: false,' backrefs: yes notify: restart docker ... Note: Use at own risk; you may need to adapt the example to work in your environment\n ","excerpt":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different …","ref":"/v1.x/docs/install/","title":"Install"},{"body":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to a specific platform.\nStorageOS can be used in any Kubernetes distribution when the prerequisites are met. That includes Kubernetes manged services.\n","excerpt":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to …","ref":"/v1.x/docs/platforms/","title":"Platforms"},{"body":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential cluster metadata and manage distributed configuration state. An embedded etcd instance is included in the StorageOS container, but for production environments and testing of production workloads, we recommend deploying an external etcd cluster. For more details about, and an example of, how to run etcd, see the External etcd Operations page.\nIt is highly recommended to use external etcd for cloud environments and place the etcd cluster on stable nodes. Placing the etcd on nodes that are recycled often might affect the normal operations of StorageOS.\nSetup of storage on the hosts We recommend creating a separate filesystem for StorageOS to mitigate the risk of filling the root filesystem on nodes. This has to be done for each node in the cluster.\nFollow the managing host storage best practices page for more details.\nResource reservations StorageOS resource consumption depends on the workloads and the StorageOS features in use.\nThe recommended minimum memory reservation for the StorageOS Pods is 512MB for non-production environments. However it is recommended to prepare nodes so StorageOS can operate with at least with 1-2GB of memory. StorageOS frees memory when possible.\nFor production environments, we recommend 4GB of Memory and 1 CPU as a minimum and to test StorageOS using realistic workloads and tune resources accordingly.\nStorageOS Pods resource allocation will impact directly on the availability of volumes in case of eviction or resource limit triggered restart. It is recommended to not limit StorageOS Pods.\nStorageOS implements a storage engine, therefore limiting CPU consumption might affect the I/O throughput of your volumes.\nStorageOS API username/password The API grants full access to StorageOS functionality, therefore we recommend that the default administrative password of \u0026lsquo;storageos\u0026rsquo; is reset to something unique and strong.\nYou can change the default parameters by encoding the apiUsername and apiPassword values (in base64) into the storageos-api secret.\nTo generate a unique password, a technique such as the following, which generates a pseudo-random 24 character string, may be used:\n# Generate strong password PASSWORD=$(cat -e /dev/urandom | tr -dc \u0026#39;a-zA-Z0-9-!@#$%^\u0026amp;*()_+~\u0026#39; | fold -w 24 | head -n 1) # Convert password to base64 representation for embedding in a K8S secret BASE64PASSWORD=$(echo -n $PASSWORD | base64) Note that the Kubernetes secret containing a strong password must be created before bootstrapping the cluster. Multiple installation procedures use this Secret to create a StorageOS account when the cluster first starts.\nStorageOS Pod placement StorageOS must run on all nodes that will contribute storage capacity to the cluster or that will host Pods which use StorageOS volumes. For production environments, it is recommended to avoid placing StorageOS Pods on Master nodes.\nStorageOS is deployed with a DaemonSet controller, and therefore tolerates the standard unschedulable (:NoSchedule) action. If that is the only taint placed on master or cordoned nodes StorageOS pods might start on them (see the Kubernetes docs for more details). To avoid scheduling StorageOS pods on master nodes, you can add an arbitrary taint to them for which the StorageOS DaemonSet won\u0026rsquo;t have a toleration.\nDedicated instance groups Cloud environments give users the ability to quickly scale the number of nodes in a cluster in response to their needs. Because of the ephemeral nature of the cloud, StorageOS recommends setting conservative downscaling policies.\nFor production clusters, it recommended to use dedicated instance groups for Stateful applications that allow the user to set different scaling policies and define StorageOS pools based on node selectors to collocate volumes.\nLosing a few nodes at the same time could cause the loss of data even when volume replicas are being used.\nPort blocking StorageOS exposes ports to operate. It is recommended that the ports are not accessible from outside the scope of your cluster.\nStorageOS in Docker EE StorageOS does not support running on Swarm nodes nor on mixed (Kubernetes and Swarm) nodes. StorageOS volumes have to be provisioned and used from Kubernetes nodes.\n","excerpt":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential …","ref":"/v1.x/docs/best-practices/","title":"StorageOS Best Practices"},{"body":"","excerpt":"","ref":"/v1.x/docs/operations/","title":"Operations"},{"body":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or other Orchestrators that are derived from Kubernetes such as OpenShift or Rancher. This is useful for running stateful applications, such as databases or CI/CD applications, under the control of Kubernetes as Kubernetes can make scheduling decisions without the application data being lost.\nWhat we have outlined in the cookbooks below are some quick deployments of stateful applications into a Kubernetes cluster. These examples are not production ready but have been provided to give you some insight into how to use StorageOS with stateful applications.\nStatefulSets The examples we have provided use StatefulSets as a way to deploy applications into Kubernetes. The reason for this is that the StatefulSet controller is designed to manage stateful applications and it \u0026ldquo;provides guarantees about the ordering and uniqueness\u0026rdquo; of sets of pods.\nPractically this means that when a StatefulSet scales, pods are created in order from 0-(N-1), if a StatefulSet scales down then pods are deleted in reverse order from (N-1)-0.\nSecondly, it means that the behaviour of the StatefulSet controller differs from that of Deployment and ReplicaSet controllers. For Deployment and ReplicaSet controllers \u0026ldquo;\u0026hellip; at many points in the lifetime of a replica set there will be 2 copies of a pod\u0026rsquo;s processes running concurrently\u0026rdquo;. Having two different pods mount a volume at the same time can cause corruption of data. Currently Kubernetes accessModes only apply restrictions to nodes mounting volumes rather than pods, so it is important that StatefulSets are used with StorageOS volumes so the necessary pod uniqueness guarantees are maintained.\nStatefulSet Manifests The StorageOS specific part of the Kubernetes manifests for these examples lies in the VolumeClaimTemplate that\u0026rsquo;s part of the statefulset definition.\nVolumeClaimTemplate\napiVersion:apps/v1kind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1template:metadata:labels:app:mssqlenv:prodspec:serviceAccountName:mssqlcontainers:- name:fooimage:barvolumeMounts:- name:bazmountPath:/var/opt/barenvFrom:- configMapRef:name:mssqlvolumeClaimTemplates:- metadata:name:bazlabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiIn the StatefulSet definition above the container has a volume mount defined called baz. The definition for this volume is found in the VolumeClaimTemplate where the fast storageClass will be used to dynamically provision storage if the persistent volume does not already exist.\n","excerpt":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or …","ref":"/v1.x/docs/usecases/","title":"Use Cases: Install applications on Kubernetes with StorageOS"},{"body":"","excerpt":"","ref":"/v1.x/docs/reference/","title":"Reference"},{"body":"There are several ways to reach us if you require support. The fastest way to get in touch is to join our public Slack channel. \nYou can file a support ticket via email to support@storageos.com, or use our Help Desk portal\nTo help us provide effective support, we request that you provide as much information as possible when contacting us. The list below is a suggested starting point. Additionally, please include anything specific, such as log entries, that may help us debug your issue.\nInformation about the cluster can be automatically sent to StorageOS engineers as mentioned in the section StorageOS Cluster Report.\nPlatform Cloud provider/Bare metal OS distribution and version Kernel version docker version and installation procedure (distro packages or docker install) StorageOS Version of StorageOS storageos node ls storageos volume ls storageos volume inspect VOL_ID # in case of issues with a specific volume Orchestrator related (Kubernetes, OpenShift, etc) Version and installation method Managed or self managed? kubectl -n kube-system get pod kubectl -n kube-system logs -lapp=storageos -c storageos kubectl -n kube-system get storageclass Specific for your namespaces: kubectl describe pvc PVC_NAME Specific for your namespaces: kubectl describe pod POD_NAME StorageOS Cluster Report StorageOS has a cluster diagnostic function that aggregates cluster information.\nFor each node the following is collected:\n StorageOS logs output of storageos inspect node output of lshw output of dmesg StorageOS engineers might ask for a report to be generated during support cases.\nThe information given in the cluster report is only used for support purposes and it will be removed once the data is no longer needed for such purposes. In case the information is sensitive and can\u0026rsquo;t be given to StorageOS, please make sure that support engineers have as much information about your environment as possible.\nThe cluster report is created only when a user chooses to do so. For convenience the report can either be uploaded to StorageOS or downloaded to the machine running the browser. The report is uploaded from a StorageOS node to a StorageOS GCP encrypted bucket using a TLS encrypted connection. The upload takes place only after user confirmation.\nYou can generate a report through the StorageOS GUI by navigating to the Cluster menu. You can also directly connect to the cluster diagnostic API endpoint and retrieve the bundle. Note that only a user with the Admin role can create Diagnostic Bundles.\ncurl -u \u0026lt;ADMIN_USERNAME\u0026gt;:\u0026lt;ADMIN_PASSWORD\u0026gt; http://\u0026lt;NODE_IP\u0026gt;:5705/v1/diagnostics/cluster -o diagnostic.tar ","excerpt":"There are several ways to reach us if you require support. The fastest way to get in touch is to …","ref":"/v1.x/docs/support/","title":"Support"},{"body":"If you have any specific or more complex requirements please contact StorageOS as we\u0026rsquo;d be happy to organise a POC in conjunction with our Engineering team. You can join our slack channel or email us at: info@storageos.com\n Table of Contents StorageOS Operator StorageOS Operator features Native Driver vs CSI External Etcd Prerequisites Installing StorageOS Install etcd Install StorageOS Operator Install StorageOS Setup a Monitoring Stack StorageOS Features Volume Replication Fencing Benchmarking Considerations Application vs StorageOS replication StatefulSets Volume Placement How to land a volume and a pod on the same node Synthetic Benchmarks Application Benchmarks Conclusion Support for Self Evaluations Should you have questions or require support, there are several ways to get in touch with us. The fastest way to get in touch is to join our public Slack channel. You can also get in touch via email to info@storageos.com.\nFurthermore you can fill out the form below and we will get in touch.\n hbspt.forms.create({ portalId: \"3402546\", formId: \"a07fecd3-ce5b-4835-b136-51a94a35632b\", sfdcCampaignId: \"70158000000BAZzAAO\" }); Installation The first phase of the self-evaluation is to install StorageOS. This section of the document aims to layout what options are exposed to you during installation and why some options may be preferable to you over others.\nA standard StorageOS installations uses the StorageOS operator, so as much of the necessary configuration is handled for you. The StorageOS operator has been certified by Red Hat and is open source.\nStorageOS Operator The StorageOS operator is a Kubernetes native application that manages the StorageOS cluster lifecycle. It simplifies cluster installation, cluster removal and other operations.\nThe StorageOS operator watches for the creation of StorageOSCluster Custom Resources. A StorageOSCluster is a declarative representation of a StorageOS cluster. For example if CSI is enabled in the StorageOSCluster resource, a StorageOS cluster will be created that uses the CSI driver.\nStorageOS Operator features Specific configuration options for the StorageOS Operator that we believe to be important during a self-evaluation will be laid out in this guide.\nA set of example StorageOSCluster are listed here. For an exhaustive list of configuration settings for the StorageOS operator please see our documentation.\nNative Driver vs CSI Communication between Kubernetes and StorageOS can use one of two drivers; the StorageOS Native Driver or the CSI (Container Storage Interface) driver. CSI provides a standardized interface for storage providers to use and is considered GA from Kubernetes 1.13. Therefore, StorageOS uses CSI as the default driver for Kubernetes 1.13+.\nAs the StorageOS native driver is implemented in the Kubernetes trunk by StorageOS, it is tied to Kubernetes releases. Whereas using CSI we can iterate more quickly and make improvements independent of Kubernetes releases.\nImportantly CSI is only considered generally available in Kubernetes 1.13+ and is still in technology preview in Openshift 3.11.\nExternal Etcd StorageOS highly recommends an external etcd cluster is used for production deployments. In this configuration the etcd cluster would run on separate boxes from the rest of the Kubernetes and StorageOS cluster ensuring stability and resilience of the etcd cluster. However for the purposes of a self-evaluation it is acceptable to run etcd as a container inside Kubernetes.\nWe do not recommend running etcd on the same nodes as StorageOS when node failure will be tested, as if the majority of etcd nodes fail then the etcd cluster cannot be recovered automatically. Therefore it is better to run etcd on separate nodes.\nPrerequisites StorageOS has some prerequisites that must be met to complete a successful installation\n Machines intended to run StorageOS have at least 1 CPU core, 2GB RAM Docker 1.10 or later with mount propagation enabled TCP ports 5701-5710 and TCP \u0026amp; UDP 5711 open between all nodes in the cluster A 64bit supported operating system - By default StorageOS supports Debian 9, RancherOS, RHEL7.5 and CentOS7. Note Ubuntu 16.04 and 18.04 are supported but additional packages are required. Ubuntu 16.04/18.04 with the AWS kernel and Ubuntu 18.04 with the GCE kernel do not provide the required packages and are therefore NOT supported.\n To install the required kernel modules on Ubuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) To install the required kernel modules on Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Installing StorageOS Installation steps are as follows:\n Install etcd Install StorageOS Operator Create a Kubernetes secret detailing the default StorageOS administrator account Install StorageOS using a StorageOSCluster Custom Resource Install etcd In order to get a StorageOS cluster stood up quickly, a single node etcd cluster can be installed in Kubernetes, on a Kubernetes master. The reason for installing on a master is that master nodes generally have predictable lifetimes and low Pod scheduling churn. As such there is a lesser risk of the etcd pod being evicted ensuring a stable etcd cluster.\nNote that if the etcd pod is stopped for any reason the etcd cluster will cease to function pending manual intervention. Please take this into account during testing of failure scenarios.\n Download repo\n$ git clone https://github.com/coreos/etcd-operator.git Configure NS, Role and RoleBinding\n$ export ROLE_NAME=etcd-operator $ export ROLE_BINDING_NAME=etcd-operator $ export NAMESPACE=etcd Create Namespace\n$ kubectl create namespace $NAMESPACE Deploy Operator\n$ ./etcd-operator/example/rbac/create_role.sh $ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END The Kubernetes masters should then be labelled so a nodeSelector can be used in the EtcdCluster manifest\n$ kubectl label nodes \u0026lt;NODES\u0026gt; etcd-cluster=storageos-etcd Once the master node taints are known and the nodes have been labelled you can deploy an EtcdCluster manifest that contains tolerations for all taints on the master nodes and selects for the node label applied in the previous step. A sample manifest is below. Edit the size to match the number of masters you will deploy on and edit the tolerations to match all taints on the master nodes where etcd will be deployed.\n Create the EtcdCluster resource\n$ kubectl -n etcd create -f - \u0026lt;\u0026lt;END apiVersion: \u0026#34;etcd.database.coreos.com/v1beta2\u0026#34; kind: \u0026#34;EtcdCluster\u0026#34; metadata: name: \u0026#34;storageos-etcd\u0026#34; spec: size: 1 version: \u0026#34;3.4.7\u0026#34; pod: etcdEnv: - name: ETCD_QUOTA_BACKEND_BYTES value: \u0026#34;2147483648\u0026#34; # 2 GB - name: ETCD_AUTO_COMPACTION_RETENTION value: \u0026#34;100\u0026#34; # Keep 100 revisions - name: ETCD_AUTO_COMPACTION_MODE value: \u0026#34;revision\u0026#34; # Set the revision mode resources: requests: cpu: 200m memory: 300Mi securityContext: runAsNonRoot: true runAsUser: 9000 fsGroup: 9000 tolerations: - operator: \u0026#34;Exists\u0026#34; nodeSelector: etcd-cluster: storageos-etcd affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster operator: In values: - storageos-etcd topologyKey: kubernetes.io/hostname END Install StorageOS Operator In order to install the StorageOS operator download the requisite yaml manifests or apply them with kubectl.\n$ kubectl create -f https://github.com/storageos/cluster-operator/releases/download/1.5.3/storageos-operator.yaml You can verify the operator is running using the following command\n$ kubectl get pods -n storageos-operator Install StorageOS Once the StorageOS operator has been installed a StorageOS cluster can be generated by creating a StorageOSCluster resource.\nA StorageOSCluster resource describes the state of the StorageOS cluster that is desired and the StorageOS operator will create the desired StorageOS cluster. For examples of StorageOSCluster resources please see our examples page here. For a full list of the configurable spec parameters of the StorageOSCluster resource please see here.\n Create a secret defining the API username and password\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;default\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9zEND Create a StorageOSCluster resource\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: \u0026quot;storageos.com/v1\u0026quot; kind: StorageOSCluster metadata: name: \u0026quot;storageos\u0026quot; spec: secretRefName: \u0026quot;storageos-api\u0026quot; secretRefNamespace: \u0026quot;default\u0026quot; images: nodeContainer: \u0026quot;storageos/node:1.5.3\u0026quot; # StorageOS version resources: requests: memory: \u0026quot;512Mi\u0026quot; csi: enable: true deploymentStrategy: deployment kvBackend: address: 'storageos-etcd-client.etcd.svc:2379' backend: 'etcd' END Confirm that the cluster has been created and that StorageOS pods are running\n$ kubectl -n kube-system get pods StorageOS pods enter a ready state after a minimum of 65s has passed.\n Deploy the StorageOS CLI as a container\n$ kubectl -n kube-system run \\ --image storageos/cli:1.2.2 \\ --restart=Never \\ --env STORAGEOS_HOST=storageos \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sleep 999999 Confirm that StorageOS is working by creating a PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 annotations: volume.beta.kubernetes.io/storage-class: fast spec: accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Verify that the CLI is working. pvc-1 should be listed in the CLI output\n$ kubectl -n kube-system exec -it cli -- storageos volume ls The StorageOS web UI can also be used to display information about the state of the cluster. The StorageOS UI can be accessed on any node that is running a StorageOS pod on port 5705. The username/password for the UI is defined by the storageos-api secret. For this self-evaluation the username/password is storageos:storageos\nhttp://\u0026lt;NODE_IP\u0026gt;:5705 Create a pod that consumes the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Check that the pod starts successfully. If the pod starts successfully then the StorageOS cluster is working correctly\n$ kubectl get pod d1 -w The pod mounts a StorageOS volume under /mnt so any files written there will persist the lifetime of the pod. This can be demonstrated using the following commands.\n Execute a shell inside the pod and write some data to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo Hello World! \u0026gt; /mnt/hello root@d1:/# cat /mnt/hello Hello World! Delete the pod\n$ kubectl delete pod d1 Recreate the pod\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Open a shell inside the pod and check the contents of /mnt/hello\n$ kubectl exec -it d1 -- cat /mnt/hello Hello World! Now that StorageOS has been successfully installed, the cluster has a standard license by default which allows for the creation of 100GB of persistent volumes. If you register the cluster then a developer license will be applied and 500GB of persistent volumes can be created. Replicas do not count towards the license total so a 500GB license could be used to created a 500GB volume with 5 replicas. For the purposes of this self-evaluation the standard license is sufficient.\nSetup a Monitoring Stack StorageOS exposes many metrics about the running of StorageOS and perhaps most importantly the read/write performance of StorageOS volumes. Each StorageOS pod exposes a Prometheus endpoint that exposes metrics; these can be visualized with something like Grafana.\nA guided installation of Prometheus, using the Prometheus operator and Grafana, using helm, is available in our deploy repository.\n To install Prometheus you can run the install-prometheus script\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/prometheus $ ./install-prometheus.sh ``\n Grafana can be installed using yaml manifests\n$ ./install-grafana.sh ``\n In order to view the Grafana or Prometheus UI create the NodePort services in the manifests folder\n$ kubectl create -f manifests/prometheus/prometheus-svc.yaml.example $ kubectl create -f manifests/grafana/grafana-svc.yaml.example The Grafana UI is then available on `\u0026lt;NODE_IP\u0026gt;:3000`. The username and password are `admin`:`admin`. The username and password are set in a secret in the grafana-deployment.yaml file. Once logged in create the Prometheus data source by setting the URL to `http://prometheus-operated:9090` and configure the scrape interval to be 10s and set the query timeout to 30s. The [StorageOS example dashboard](https://grafana.com/dashboards/10093) can then be imported into Grafana. To confirm the dashboard is working try writing some data to the volume that was created previously\n$ kubectl exec -it d1 -- bash -c \u0026#39;dd if=/dev/zero of=/mnt/file count=1024 bs=1M oflag=direct\u0026#39; ``\n The StorageOS dashboard will show that a volume is being written to, giving metrics for IOPS and bandwidth.\nFor more information about how to interpret the metrics that we expose please see our documentation about Monitoring StorageOS. And for a full overview of the metrics that we expose please refer to our Prometheus documentation.\nWe have also created a dashboard for monitoring etcd pods which can be found here. It is important to defragment etcd before the on disk space exceeds the database quota, see the etcd documentation for more information about etcd maintenance.\n StorageOS Features Now that you have a correctly functioning StorageOS cluster we will explain some of our features that may be of use to you as you complete application and synthetic benchmarks.\nStorageOS features are all enabled/disabled by applying labels to volumes. These labels can be passed to StorageOS via persistent volume claims (PVCs) or can be applied to volumes using the StorageOS CLI or GUI.\nThe following is not an exhaustive feature list but outlines features which are commonly of use during a self-evaluation.\nVolume Replication StorageOS enables synchronous replication of volumes using the storageos.com/replicas label.\nThe volume that is active is referred to as the master volume. The master volume and its replicas are always placed on separate nodes. In fact if a replica cannot be placed on a node without a replica of the same volume, the volume will fail to be created. For example, in a three node StorageOS cluster a volume with 3 replicas cannot be created as the third replica cannot be placed on a node that doesn\u0026rsquo;t already contain a replica of the same volume.\nThe failure mode for a volume affects how many failed replicas can be tolerated before the volume is marked as offline. Replicas are also segregated according to the iaas/failure-domains node label. StorageOS will automatically place a master volume and its replicas in separate failure domains where possible.\nSee our replication documentation for more information on volume replication.\n To test volume replication create the following PersistentVolumeClaim\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-replicated labels: storageos.com/replicas: \u0026#34;1\u0026#34; annotations: volume.beta.kubernetes.io/storage-class: fast spec: accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Note that volume replication is enabled by setting the storageos.com/replicas label on the volume.\n Confirm that a replicated volume has been created by using the StorageOS CLI or UI\n$ kubectl -n kube-system exec -it cli -- storageos volume ls Create a pod that uses the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: replicated-pod spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-replicated END Write data to the volume\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# echo Hello World! \u0026gt; /mnt/hello root@replicated-pod:/# cat /mnt/hello Hello World! Find the location of the master volume and drain the node using the StorageOS CLI. Draining a node causes all volumes on the node to be evicted. For replicated volumes this immediately promotes a replica to become the new master, and for unreplicated volumes a replica is created and fully synchronized before the volume fails over\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-replicated Bound pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 5Gi RWO fast 1m $ kubectl exec -it -n kube-system cli -- storageos volume ls NAMESPACE/NAME SIZE MOUNT STATUS REPLICAS LOCATION default/pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 50GiB ip-10-0-11-175 active 1/1 ip-10-0-11-167 (healthy) $ kubectl exec -it -n kube-system cli -- storageos node drain ip-10-0-11-167 ip-10-0-11-167 Check the location of the master volume and notice that it is on a new node\n$ kubectl exec -it -n kube-system cli -- storageos volume ls NAMESPACE/NAME SIZE MOUNT STATUS REPLICAS LOCATION default/pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 50GiB ip-10-0-11-175 active 1/1 ip-10-0-11-189 (synching) Check that the data is still accessible to the pod\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# cat /mnt/hello Hello World! Fencing StorageOS enables fencing of pods using the storageos.com/fenced=true label. Pods must have the fencing label set and be using at least one StorageOS volume. Any StorageOS volume that the pod is using must have at least one healthy replica.\nStatefulSets are the de facto controller for stateful workloads in Kubernetes. They provide a variety of useful guarantees but chief among them is that pods are unique. This guarantee means that if Kubernetes detects that a node is segregated from the master, StatefulSet pods will not be rescheduled unless the StatefulSet pods on the failed node are manually force terminated. However as StorageOS pods communicate via a gossip protocol, StorageOS can determine whether the node is truly offline or just partitioned from the master. In the case that the node is no longer participating in gossip, StorageOS can intervene and terminate StatefulSet pods that are using StorageOS volumes thus improving the time to recover for StatefulSets.\nFor more information about StatefulSets and fencing please see our Fencing concepts page. For information on how to enable Fencing see our Fencing operations page.\n To test fencing create a StatefulSet from the StorageOS deploy repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./fencing Check what node the mysql-0 pod is running on and make that node unavailable e.g. shutdown the node or stop the kubelet on the node. Now watch as the mysql-0 pod is rescheduled onto a different node\n$ kubectl get pods -l app=mysql -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES client 1/1 Running 0 1m 10.244.2.4 ip-10-1-10-112.storageos.net \u0026lt;none\u0026gt; \u0026lt;none\u0026gt; mysql-0 1/1 Running 0 1m 10.244.1.6 ip-10-1-10-235.storageos.net \u0026lt;none\u0026gt; \u0026lt;none\u0026gt; Once the node is in a NotReady state you\u0026rsquo;ll see that the mysql-0 pod has been rescheduled on a different node\n$ kubectl get nodes NAME STATUS ROLES AGE VERSION ip-10-1-10-112.storageos.net Ready master 107m v1.14.3 ip-10-1-10-118.storageos.net Ready \u0026lt;none\u0026gt; 107m v1.14.3 ip-10-1-10-235.storageos.net NotReady \u0026lt;none\u0026gt; 107m v1.14.3 $ kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES client 1/1 Running 0 1m 10.244.2.4 ip-10-1-10-112.storageos.net \u0026lt;none\u0026gt; \u0026lt;none\u0026gt; mysql-0 1/1 Running 0 30s 10.244.1.6 ip-10-1-10-118.storageos.net \u0026lt;none\u0026gt; \u0026lt;none\u0026gt; Benchmarking As a rule the best performance is obtained by using unreplicated volumes that are co-located with the application or benchmarking tool writing to the volume. See Volume Placement below for more information.\nWhen running benchmarks in the cloud, benchmarks need to be run multiple times and nodes should be destroyed and recreated so that the underlying machine changes. This should be done to reduce the impact that noisy neighbours might have on benchmark results.\nConsiderations Application vs StorageOS replication Certain applications are able to natively replicate or shard data between application instances. When using these applications it is worth considering whether application replication, StorageOS replication or a mixture of both should be used.\nWhen StorageOS replication is enabled the time to recover in cases of node failure can be lessened. This is because StorageOS will promote a replica, Kubernetes will reschedule the application instance and the amount of data the application needs to catch up on is limited to whatever data was modified while the application was being rescheduled. Without StorageOS replication the application would have to rebuild an entire copy of data. Some applications have their performance greatly impacted by having to rebuild shards/replicas so this is also avoided.\nStatefulSets StatefulSets are the de facto controller for stateful applications. As such, when deploying applications that will use StorageOS volumes, StatefulSets should be used. You can find more information about StatefulSets here.\nVolume Placement StorageOS volumes give the best performance when the application pod and the master volume are co-located on the same node. When benchmarking applications, it is useful to take into account that using remote volumes and replicas impact the overall performance of a volume.\nGoing from 0 to 1 replica has the greatest performance impact for writes as now the latency of the operation is equal to the round trip time to the node with the replica over the network. Adding additional replicas poses less of a performance impact as writes to replicas are done in parallel, and the round trip time to each node is unlikely to greatly increase unless replicas land on nodes that are geographically distant to the master volumes\u0026rsquo; node.\nEven when volumes are replicated co-location of pod and master volume is still desirable because application writes are first sent to the master and then sent from the master volume to the replicas. Writing to a local master therefore saves network latency between the application and the master volume. As reads are always served from the master volume a remote master volume will add latency to reads as well as writes.\nWhen testing applications, such as databases, it is also necessary to run benchmarks for a sufficiently long time to account for caching, and cache flushing that databases do. We recommend running application benchmarks over a 20-30min period for this reason.\nHow to land a volume and a pod on the same node StorageOS has an automatic co-location feature on our development roadmap that we are calling pod locality. Until the feature is GA co-location of a master volume and a pod can be achieved by leveraging existing StorageOS and Kubernetes features.\nstorageos.com/hint.master is a volume label that influences the placement of a StorageOS master volume. By setting this label to the same value as a nodeSelector on a StatefulSet or Pod the master volume and the pod should co-locate on the same node. You can reference our FIO local volumes job for an example of how to do this.\nStorageOS Pools can be used to restrict volume placement to a subset of nodes. Nodes can be included in a specific pool by matching a pools nodeSelector. Pools can be created using the StorageOS GUI or CLI. The pool that a volume will be created from is specified in the StorageClass pool parameter.\nIt is also possible to cordon StorageOS nodes using the GUI or the CLI in order to force the placement of volumes on a specific node. A further possibility is to use of the storageos.com/auto-follow label. This label enables StorageOS to promote a replica volume to being a master when the pod and the replica volume are co-located. Synthetic Benchmarks Synthetic benchmarks using tools such as FIO are a useful way to begin measuring StorageOS performance. While not fully representative of application performance, they allow us to reason about the performance of storage devices without the added complexity of simulating real world workloads, and provide results easily comparable across platforms.\nAs with application benchmarks, when testing in public clouds multiple runs on newly created nodes should be considered to account for the impact of noisy neighbours.\nStorageOS has created a test suite for running FIO tests against StorageOS volumes that can be found here. The test suite can be deployed into a Kubernetes cluster using the instructions below.\n Clone the StorageOS use cases repo. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/FIO/local-volumes.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the FIO local-volumes folder\n$ cd storageos-usecases/FIO/local-volumes Get the name of the node that you wish the FIO pods and volumes to be created on. Make sure that the node name and the label kubernetes.io/hostname match and that the node has enough storage capacity to create 8Gi worth of volumes\n$ kubectl get node --show-labels Generate the FIO jobs by passing in the node name that the job should run on. The number is the number of volumes that FIO will test concurrently\n$ ./job-generator-per-volumecount.sh 4 $NODE_NAME Upload the FIO profiles as ConfigMaps\n$ ./upload-fio-profiles.sh Run the FIO tests\n$ kubectl create -f ./jobs Check the PVCs have been provisioned\n$ kubectl get pvc Use the StorageOS CLI to check the location of the volumes\n$ kubectl -n kube-system exec cli -- storageos volume ls Verify that the Pod is running on the same node\n$ kubectl get pod -owide FIO has a number of parameters that can be adjusted to simulate a variety of workloads and configurations. Particularly the queue depth, block size and the number of volumes used affect the FIO results. To tune the FIO parameters the profiles file can be edited or the ConfigMap that is created from the profiles file can be edited directly.\nStorageOS configuration also affects the overall volume performance. For example adding a replica to a volume will increase the latency for writes and affect IOPS and bandwidth for the volume.\nTo see the effect a StorageOS replica has on performance rerun an FIO test but add the storageos.com/replicas: \u0026quot;1\u0026quot; label to the PersistentVolumeClaims in the jobs spec. The greatest performance impact from adding replicas comes when moving from 0 to 1 replica. Adding additional replicas does not incur a significant performance penalty.\nThe remote volumes folder contains a guide for performing the same FIO tests against remote volumes. Application Benchmarks While synthetic benchmarks are useful for examining the behaviour of StorageOS with very specific workloads, in order to get a realistic picture of StorageOS performance actual applications should be tested.\nMany applications come with test suites which provide standard workloads. For best results, test using your application of choice with a representative configuration and real world data.\nAs an example of benchmarking an application the following steps lay out how to benchmark a Postgres database backed by a StorageOS volume.\n Start by cloning the StorageOS use cases repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/pgbench.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the Postgres examples folder\n$ cd storageos-usecases/pgbench Decide which node you want the pgbench pod and volume to be located on. The node needs to be labelled app=postgres\n$ kubectl label node \u0026lt;NODE\u0026gt; app=postgres Then set the storageos.com/hint.master label in 20-postgres-statefulset.yaml file to match the node name you have chosen before creating all the files\n$ kubectl create -f . Confirm that Postgres is up and running\n$ kubectl get pods -w -l app=postgres Use the StorageOS CLI or the GUI to check the master volume location and the mount location. They should match\n$ kubectl -n kube-system exec -it cli -- storageos v ls Exec into the pgbench container and run pgbench\n$ kubectl exec -it pgbench -- bash -c \u0026#39;/opt/cpm/bin/start.sh\u0026#39; Conclusion After completing these steps you will have benchmark scores for StorageOS. Please keep in mind that benchmarks are only part of the story and that there is no replacement for testing actual production or production like workloads.\nStorageOS invites you to provide feedback on your self-evaluation to the slack channel or by directly emailing us at info@storageos.com\n","excerpt":"If you have any specific or more complex requirements please contact StorageOS as we\u0026rsquo;d be …","ref":"/v1.x/docs/self-eval/","title":"Self Evaluation Guide"},{"body":"","excerpt":"","ref":"/v1.x/docs/operations/updates/","title":""},{"body":"","excerpt":"","ref":"/v1.x/index.json","title":""},{"body":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger.yaml\", dom_id: '#ohpen_swagger_ui', presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ] }) window.ui = ui } \t","excerpt":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger.yaml\", dom_id: …","ref":"/v1.x/docs/reference/api/","title":"API Reference"},{"body":"StorageOS is a software-defined storage platform for running stateful applications in containers.\nRead about the cloud native storage principles behind StorageOS.\nFundamentally, StorageOS aggregates storage attached to nodes in a cluster, creates a virtual pool across nodes, and presents virtual volumes from the pool into containers.\nIt is agnostic to the underlying storage and runs equally on bare metal, in virtual machines or on cloud providers.\nStorageOS is deployed as one container on each node that presents or consumes storage, available as storageos/node on the Docker Hub. In Kubernetes, this is typically managed as a daemonset, next to the applications. StorageOS runs entirely in user space.\nStorageOS is designed to feel familiar to Kubernetes and Docker users. Storage is managed through standard StorageClasses and PersistentVolumeClaims, and features are controlled by Kubernetes-style labels and selectors, prefixed with storageos.com/.\nStorageOS uses the storage capacity from the nodes where it is installed to provide thinly-provisioned volumes. That space is selected from the mount point of /var/lib/storageos/data on the host. It is recommended that disk devices are used exclusively for StorageOS, as described in Managing Host Storage \nAny container may mount a StorageOS virtual volume from any node, regardless of whether the container and volume are colocated on the same node or the volume is remote. Therefore, applications may be started or restarted on any node and access volumes transparently.\nVolumes are provisioned from a storage pool and are thinly provisioned.\nBy default, volumes are cached to improve read performance and compressed to reduce network traffic.\n Available memory % of overall memory reserved by StorageOS for caching 3 GB or less 3% 3-8 GB 5% 8-12 GB 7% 12 GB or more 10% ","excerpt":"StorageOS is a software-defined storage platform for running stateful applications in containers. …","ref":"/v1.x/docs/concepts/architecture/","title":"Architecture"},{"body":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our Kubernetes installation instructions page.\nAn EKS deployment of Kubernetes uses AWS Linux by default with an optimized kernel. As the requisite kernel modules are not available for StorageOS to use TCMU, FUSE will be used as a fallback. Using FUSE instead of TCMU has performance implications.\nFor more details about the OS Distributions check the System Configuration page.\n","excerpt":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our …","ref":"/v1.x/docs/platforms/aws-eks/","title":"AWS EKS"},{"body":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our Kubernetes installation instructions page.\nAn AKS deployment of Kubernetes uses Ubuntu by default with an optimized kernel. All versions of Ubuntu with a kernel version later than4.15.0-1029-azure meet the StorageOS prerequisites.\n","excerpt":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our …","ref":"/v1.x/docs/platforms/azure-aks/","title":"Azure AKS"},{"body":"In this example use case we provide three different strategies for accessing files that have been written to a StorageOS persistent volume.\nIn the following examples the \u0026ldquo;application\u0026rdquo; container is the container main, which has a rsync, Nginx or sftp sidecar container. The StorageOS volume that the application is writing to will be mounted into the sidecar container so files written by the application are available for export. Files can be exported using Nginx as a web file server, transferred using rsync or accessed via SFTP.\nThe files create a stateful set that can be used AFTER a StorageOS cluster has been created. See our guide on how to install StorageOS on Kubernetes for more information.\nClone Repository In order to deploy the examples, clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/backup Before deploying the backup-example stateful set we recommend looking through the examples to understand how the different containers are configured\n Exfiltrating files through HTTP Deploy the Nginx example $ kubectl create -f nginx/ service/backup-example created configmap/nginx-config created statefulset.apps/backup-example created pod/busybox created Check that a backup-example pod is running $ kubectl get pods -w -l app=backup-example-nginx NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file $ kubectl exec -it backup-example-nginx-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-nginx ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 80/TCP 46s Use wget to access the files served by Nginx. Nginx is sharing files from the same volume that the main application container is writing to. The connection to the Nginx container is made via the backup-example service. $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;lost%2Bfound/\u0026#34;\u0026gt;lost+found/\u0026lt;/a\u0026gt; 12-Feb-2019 12:32 - \u0026lt;a href=\u0026#34;date.txt\u0026#34;\u0026gt;date.txt\u0026lt;/a\u0026gt; 12-Feb-2019 12:49 29 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx/date.txt Tue Feb 12 12:49:15 UTC 2019 Depending on what files have been written to the StorageOS volume the output of the index file will be different. In the example the date.txt file we created in Step 2 is present on the volume.\nExfiltrating files through Rsync Deploy the rsync example $ kubectl create -f rsync/ service/backup-example created configmap/rsync-config created secret/rsync-credentials created statefulset.apps/backup-example created pod/rsync created Check that a backup-example pod is running $ kubectl get pods -w -l app=backup-example-rsync NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file $ kubectl exec -it backup-example-rsync-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-rsync NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-rsync ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 873/TCP 46s Use rsync to access the files shared by the rsync daemon. rsync is sharing files from the same volume that the main container is writing to. A username and password that are set in the rsync-credentials secret. The secret supplied in the example has the username and password set to username and password. $ kubectl exec -it rsync sh / # rsync --list-only rsync://username@backup-example-rsync/share Password: drwxr-xr-x 4,096 2019/02/12 12:49:15 . -rw-r--r-- 29 2019/02/12 12:49:15 date.txt drwx------ 16,384 2019/02/12 12:32:40 lost+found / # rsync -chavzP rsync://username@backup-example-rsync/share/date.txt . Password: receiving incremental file list date.txt 29 100% 28.32kB/s 0:00:00 (xfr#1, to-chk=0/1) sent 43 bytes received 135 bytes 50.86 bytes/sec total size is 29 speedup is 0.16 / # cat date.txt Tue Feb 12 12:49:15 UTC 2019 In the example above the list of available files was shown and a file called date.txt was synchronized to the rsync container.\nExfiltrating files through SFTP Deploy the sftp example $ kubectl create -f sftp/ Exec into the main container and write some data to a file $ kubectl exec -it backup-example-sftp-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-sftp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-sftp ClusterIP 100.70.50.56 \u0026lt;none\u0026gt; 22/TCP 2h Use SFTP to access the files shared by the SFTP container. If you have made no changes to the sftp-config secret the password is password. $ kubectl exec -it sftp -- bash root@sftp:/# sftp alex@backup-example-sftp alex@backup-example-sftp\u0026#39;s password: Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found sftp\u0026gt; get date.txt Fetching /date.txt to date.txt /date.txt 100% 29 15.9KB/s 00:00 sftp\u0026gt; bye root@sftp:/# cat date.txt Tue Feb 12 17:51:32 UTC 2019 In order to do this a SFTP user needs to be configured. The details for the user are stored in the sftp-config secret (see sftp/17-secret.yaml). The secret consists of base64 encoded username:password:uid:guid and the user is chroot\u0026rsquo;ed inside their home directory so the mount point for the StorageOS volume in the SFTP container in sftp/20-backup-pod.yaml needs to be configured.\nUsing custom SSH Keys The ConfigMap ssh-key-pub (see sftp/15-configmap.yaml) needs to be populated with a public key. The corresponding private key needs to be base64 encoded and put into the ssh-key-private secret (see sftp/17-secret.yaml). The user to connect as is determined by the user that is configured in the sftp-config configMap. To restrict logins to the SSH key edit the sftp-config secret so it contains no password (user::uid:guid).\n Connect to the sftp pod and connect through the service to the SFTP container running inside the backup-example pod. $ kubectl exec -it sftp -- bash root@sftp:/# sftp -i /home/alex/.ssh/id_rsa alex@backup-example-sftp Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found ","excerpt":"In this example use case we provide three different strategies for accessing files that have been …","ref":"/v1.x/docs/usecases/backups/","title":"Backing up files from StorageOS volumes"},{"body":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing StorageOS persistent volumes with Cassandra means that if a Cassandra pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up the pod data is immediately available. Should Kubernetes schedule the Cassandra pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not a StorageOS master is located on the same node.\nAs Cassandra has features to allow it to handle replication careful consideration of whether to allow StorageOS or Cassandra to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Cassandra on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiVersion:apps/v1kind:StatefulSetmetadata:name:cassandraspec:selector:matchLabels:app:cassandraserviceName:cassandrareplicas:3...spec:...volumeMounts:- name:cassandra-datamountPath:/var/lib/cassandra...volumeClaimTemplates:- metadata:name:cassandra-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Move into the Cassandra examples folder and create the objects\ncd storageos-usecases kubectl create -f ./cassandra Confirm Cassandra is up and running.\n$ kubectl get pods -w -l app=cassandra NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 8m32s cassandra-1 1/1 Running 0 7m51s cassandra-2 1/1 Running 0 6m36s Connect to the Cassandra client pod and connect to the Cassandra server through the service\n$ kubectl exec -it cassandra-0 -- cqlsh cassandra-0.cassandra Connected to K8Demo at cassandra-0.cassandra:9042. [cqlsh 5.0.1 | Cassandra 3.11.3 | CQL spec 3.4.4 | Native protocol v4] Use HELP for help. cqlsh\u0026gt; SELECT cluster_name, listen_address FROM system.local; cluster_name | listen_address --------------+---------------- K8Demo | 100.96.7.124 (1 rows) ","excerpt":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing …","ref":"/v1.x/docs/usecases/cassandra/","title":"Cassandra"},{"body":"The storageos command line interface (CLI) is used to manage cluster-wide configuration.\nInstallation # linux/amd64 curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/1.2.2/storageos_linux_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ # MacOS curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/1.2.2/storageos_darwin_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ You will need to provide the correct credentials to connect to the API. The default installation creates a single user with username storageos and password storageos:\nexport STORAGEOS_USERNAME=storageos STORAGEOS_PASSWORD=storageos export STORAGEOS_HOST=10.1.5.249 Usage $ storageos Usage:\tstorageos COMMAND Converged storage for containers. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Options: --config string Location of client config files (default \u0026#34;/home/$USER/.storageos\u0026#34;) -D, --debug Enable debug mode -d, --discovery string The discovery endpoint. Defaults to https://discovery.storageos.cloud (will override STORAGEOS_DISCOVERY env variable value) --help Print usage -H, --host string Node endpoint(s) to connect to (will override STORAGEOS_HOST env variable value) -p, --password string API password (will override STORAGEOS_PASSWORD env variable value) -u, --username string API username (will override STORAGEOS_USERNAME env variable value) -v, --version Print version information and quit Management Commands: cluster Manage clusters licence Manage the licence logs View and manage node logs on the active cluster namespace Manage namespaces node Manage nodes policy Manage policies pool Manage capacity pools rule Manage rules user Manage users volume Manage volumes Commands: install-bash-completion Install bash completion for the storageos cli login Store login credentials for a given storageos host logout Delete stored login credentials for a given storageos host version Show the StorageOS version information Run \u0026#39;storageos COMMAND --help\u0026#39; for more information on a command. Formatting CLI Output StorageOS CLI output can be formatted using the --format option. The strings that are passed to --format are based on Go templates.\nStorageOS CLI v1.2.0 added help for the --format option. The help can be accessed by passing --format help to StorageOS CLI commands to get context specific help.\n$ storageos volume ls --format help Cheatsheet Command Subcommand Description cluster create health inspect rm Cluster information. licence apply inspect rm Manage licences logs view View and manage node logs. namespace create inspect ls rm update Namespaces help different projects or teams organize volumes. node cordon drain health inspect ls uncordon undrain update Node information. policy create inspect ls rm Define how resources are accessed by users and groups. pool create inspect ls rm A collection of storage resources for provisioning volumes. rule create inspect ls rm update Rules define label-based policies to apply to volumes. user create inspect ls rm update User and group management. volume create inspect ls rm update StorageOS data volumes. Source is available on Github.\n","excerpt":"The storageos command line interface (CLI) is used to manage cluster-wide configuration. …","ref":"/v1.x/docs/reference/cli/","title":"Command line interface"},{"body":"$ storageos cluster Usage:\tstorageos cluster COMMAND Manage clusters Aliases: cluster, c Options: --help Print usage Commands: connectivity Display connectivity diagnostics for the cluster create Creates a cluster initialization token. health Displays the cluster\u0026#39;s health. inspect Display detailed information on one or more cluster maintenance Enable|disable maintenance mode for the cluster rm Remove one or more clusters Run \u0026#39;storageos cluster COMMAND --help\u0026#39; for more information on a command. storageos cluster connectivity To check the connectivity of the cluster\n$ storageos cluster connectivity SOURCE NAME ADDRESS LATENCY STATUS MESSAGE storageos-nodes2 storageos-nodes1.api 10.0.12.154:5705 1.085151ms OK storageos-nodes2 storageos-nodes1.directfs 10.0.12.154:5703 1.09232ms OK storageos-nodes2 storageos-nodes1.etcd 10.0.12.154:5707 1.142334ms OK storageos-nodes2 storageos-nodes1.nats 10.0.12.154:5708 1.172353ms OK storageos-nodes2 storageos-nodes1.serf 10.0.12.154:5711 1.11125ms OK storageos-nodes2 storageos-nodes2.api 10.0.12.148:5705 1.204403ms OK storageos-nodes2 storageos-nodes2.directfs 10.0.12.148:5703 1.134408ms OK storageos-nodes2 storageos-nodes2.etcd 10.0.12.148:5707 1.115885ms OK storageos-nodes2 storageos-nodes2.nats 10.0.12.148:5708 1.201178ms OK storageos-nodes2 storageos-nodes2.serf 10.0.12.148:5711 1.111379ms OK storageos-nodes2 storageos-nodes3.api 10.0.12.253:5705 1.143731ms OK storageos-nodes2 storageos-nodes3.directfs 10.0.12.253:5703 1.149442ms OK storageos-nodes2 storageos-nodes3.etcd 10.0.12.253:5707 1.083065ms OK storageos-nodes2 storageos-nodes3.nats 10.0.12.253:5708 1.090467ms OK storageos-nodes2 storageos-nodes3.serf 10.0.12.253:5711 1.158129ms OK storageos-nodes3 storageos-nodes1.api 10.0.12.154:5705 1.145954ms OK storageos-nodes3 storageos-nodes1.directfs 10.0.12.154:5703 1.114514ms OK storageos-nodes3 storageos-nodes1.etcd 10.0.12.154:5707 1.214016ms OK storageos-nodes3 storageos-nodes1.nats 10.0.12.154:5708 1.093753ms OK storageos-nodes3 storageos-nodes1.serf 10.0.12.154:5711 1.076079ms OK storageos-nodes3 storageos-nodes2.api 10.0.12.148:5705 1.206116ms OK storageos-nodes3 storageos-nodes2.directfs 10.0.12.148:5703 1.077688ms OK storageos-nodes3 storageos-nodes2.etcd 10.0.12.148:5707 1.079419ms OK storageos-nodes3 storageos-nodes2.nats 10.0.12.148:5708 1.090791ms OK storageos-nodes3 storageos-nodes2.serf 10.0.12.148:5711 1.15946ms OK storageos-nodes3 storageos-nodes3.api 10.0.12.253:5705 1.098104ms OK storageos-nodes3 storageos-nodes3.directfs 10.0.12.253:5703 1.154387ms OK storageos-nodes3 storageos-nodes3.etcd 10.0.12.253:5707 1.147184ms OK storageos-nodes3 storageos-nodes3.nats 10.0.12.253:5708 1.168365ms OK storageos-nodes3 storageos-nodes3.serf 10.0.12.253:5711 1.10147ms OK storageos-nodes1 storageos-nodes1.api 10.0.12.154:5705 1.141353ms OK storageos-nodes1 storageos-nodes1.directfs 10.0.12.154:5703 1.10065ms OK storageos-nodes1 storageos-nodes1.etcd 10.0.12.154:5707 1.143535ms OK storageos-nodes1 storageos-nodes1.nats 10.0.12.154:5708 1.142812ms OK storageos-nodes1 storageos-nodes1.serf 10.0.12.154:5711 1.125368ms OK storageos-nodes1 storageos-nodes2.api 10.0.12.148:5705 1.126621ms OK storageos-nodes1 storageos-nodes2.directfs 10.0.12.148:5703 1.114407ms OK storageos-nodes1 storageos-nodes2.etcd 10.0.12.148:5707 1.192261ms OK storageos-nodes1 storageos-nodes2.nats 10.0.12.148:5708 1.075251ms OK storageos-nodes1 storageos-nodes2.serf 10.0.12.148:5711 1.191951ms OK storageos-nodes1 storageos-nodes3.api 10.0.12.253:5705 1.080853ms OK storageos-nodes1 storageos-nodes3.directfs 10.0.12.253:5703 1.084045ms OK storageos-nodes1 storageos-nodes3.etcd 10.0.12.253:5707 1.117382ms OK storageos-nodes1 storageos-nodes3.nats 10.0.12.253:5708 1.15015ms OK storageos-nodes1 storageos-nodes3.serf 10.0.12.253:5711 1.075519ms OK storageos cluster create To create a cluster token for cluster discovery:\n$ storageos cluster create 207f0026-3844-40e0-884b-729d79c124b8 storageos cluster health To view the status of cluster nodes:\n$ storageos cluster health NODE CP_STATUS DP_STATUS storageos-1 Healthy Healthy storageos-2 Healthy Healthy storageos-3 Healthy Healthy To view the status in more detail there are additional format options which can be given to the --format flag:\n cp shows the status of control plane components dp shows the status of data plane components detailed shows the status of control plane and data plane components All the normal format options are available too. Run storageos cluster health --format help to see all the options for this command.\nstorageos cluster inspect To inspect a cluster:\n$ storageos cluster inspect 207f0026-3844-40e0-884b-729d79c124b8 [ { \u0026#34;id\u0026#34;: \u0026#34;207f0026-3844-40e0-884b-729d79c124b8\u0026#34;, \u0026#34;size\u0026#34;: 3, \u0026#34;createdAt\u0026#34;: \u0026#34;2017-07-14T13:17:29.226058526Z\u0026#34;, \u0026#34;updatedAt\u0026#34;: \u0026#34;2017-07-14T13:17:29.22605861Z\u0026#34; } ] storageos cluster maintenance inspect To view the maintenance status of a cluster:\n$ storageos cluster maintenance inspect [ { \u0026quot;enabled\u0026quot;: false, \u0026quot;updatedBy\u0026quot;: \u0026quot;\u0026quot;, \u0026quot;updatedAt\u0026quot;: \u0026quot;0001-01-01T00:00:00Z\u0026quot; } ] storageos cluster rm To remove a cluster:\nstorageos cluster rm 207f0026-3844-40e0-884b-729d79c124b8 207f0026-3844-40e0-884b-729d79c124b8 ","excerpt":"$ storageos cluster Usage:\tstorageos cluster COMMAND Manage clusters Aliases: cluster, c Options: …","ref":"/v1.x/docs/reference/cli/cluster/","title":"Cluster"},{"body":"On startup, you will need to specify whether a StorageOS node should bootstrap a new cluster or join an existing cluster. In order for a bootstrapping node to join or create a cluster, the node needs to know where to find the other nodes in the cluster. The node is informed of other nodes in the cluster via the JOIN environment variable.\nCluster Initialisation - Using StorageOS Operator For standard installs, the StorageOS operator will automatically populate the JOIN variable with appropriate values. For users with advanced requirements, the operator allows specification of a custom JOIN variable.\nCluster Initialisation - Advanced/Custom Installations StorageOS offers a public discovery service, which is a convenient way to pass clustering information to the StorageOS node.\n# Create a cluster discovery token. This token is not used after initialization $ storageos cluster create 017e4605-3c3a-434d-b4b1-dfe514a9cd0f # Add the returned cluster ID token to the JOIN variable JOIN=017e4605-3c3a-434d-b4b1-dfe514a9cd0f Alternatively, you can specify the IP addresses of nodes to join. If you provide a list of node IPs any new node joining the cluster will attempt to contact the node IPs specified. This means that if all the nodes in the JOIN are unavailable that new nodes will be unable to join the cluster.\n# Specify a node to connect to in an existing cluster JOIN=172.28.128.3 # Specify a list of nodes to attempt to connect to, in left-to-right order JOIN=172.28.128.3,172.28.128.9,172.28.128.15 # Specify both the discovery service and IP addresses, tried left-to-right JOIN=d53e9fae-7436-4185-82ea-c0446a52e2cd,172.28.128.3,172.28.128.9 The JOIN command line argument is always required, even in clusters with only one node. A blank JOIN variable will result in a non-functional cluster. This is to prevent non-obvious split-brain scenarios in multi-node clusters, where JOIN was mistakenly omitted.\n# Create a one-node cluster; note that replicas are unavailable. JOIN=$ADVERTISE_IP ","excerpt":"On startup, you will need to specify whether a StorageOS node should bootstrap a new cluster or join …","ref":"/v1.x/docs/reference/clusterdiscovery/","title":"Cluster discovery"},{"body":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator acts as a Kubernetes controller that watches the StorageOSCluster CR (Custom Resource). Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\nYou can find the source code in the cluster-operator repository.\nInstall the operator following orchestrator specific procedure.\nTo deploy a StorageOS cluster you will need to fulfil the following steps.\n Install operator Create storageos-api Secret Create a CR (Custom Resource) to deploy StorageOS ","excerpt":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS …","ref":"/v1.x/docs/reference/cluster-operator/","title":"Cluster Operator"},{"body":"StorageOS clusters represent groups of nodes which run a common distributed control plane, and aggregate their storage into one or more pools.\nTypically, a StorageOS cluster maps one-to-one to a Kubernetes (or similar orchestrator) cluster, and we expect our container to run on all worker nodes within the cluster that will consume or present storage.\nClusters use etcd to maintain state and manage distributed consensus between nodes. We offer a choice between an internally managed etcd suitable for test installations or the ability to interface with an external etcd, suitable for production deployments. We recommend the use of external etcd when production or production like workloads will be deployed on StorageOS.\n","excerpt":"StorageOS clusters represent groups of nodes which run a common distributed control plane, and …","ref":"/v1.x/docs/concepts/clusters/","title":"Clusters"},{"body":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is generally increased when compression is enabled due to fewer read/write operations taking place on the backend store (the volumes\u0026rsquo; blob files). Compression can be disabled by setting the label storageos.com/nocompress=true on a volume.\nStorageOS utilises the lz4 compression algorithm when writing to the backend store and when compressing replication traffic before it is sent across the network. Compression is granular per 4k block and data will remain compressed/uncompressed once written to a volume. Therefore, compression can be dynamically enabled and disabled by setting the storageos.com/nocompress label on a volume.\nStorageOS detects whether a block can be compressed or not by creating a heuristic that predicts the size of a compressed block. If the heuristic indicates that the compressed block is likely to be larger than the original block then the uncompressed block is stored. Block size increases post compression if the compression dictionary is added to a block that cannot be compressed. By verifying whether blocks can be compressed, disk efficiency is increased and CPU resources are not wasted on attempts to compress uncompressible blocks. StorageOS\u0026rsquo; patented on disk format is used to tell whether individual blocks are compressed without overhead. As such volume compression can be dynamically enabled/disabled even while a volume is in use.\nWhen compression and encryption are both enabled for a volume, blocks are compressed then encrypted.\n","excerpt":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is …","ref":"/v1.x/docs/concepts/compression/","title":"Compression"},{"body":"We are always looking to improve our documentation. If you like to help people and can write, read on for the process for submitting your contributions. If your guide is published, you\u0026rsquo;ll receive $250 per article by PayPal.\nContent guidelines A guide contains step by step instructions for how to accomplish a specific task using StorageOS. To be accepted, guides must be:\n Written in English. Relevant, accurate and complete. Technically correct and thoroughly tested. Follow the writing style guide. Guides should avoid:\n Duplicating an existing guide or other sources, such as blogs or forum posts. Including irrelevant material. Submission and review You should submit your guide as a pull request to the GitHub repo.\nYour submission will be left open for community review for two weeks. After this, your submission will be reviewed internally for about another week.\nIf accepted, your pull request will be approved and you will have 36 hours to send your submission title and PayPal account information. Non-response will be taken as a go-ahead to publish.\nLegal COPYRIGHT OWNERSHIP. The StorageOS Guides \u0026amp; Tutorials repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license.\nCREDIT. Nothing contained in this Agreement shall be deeded to require StorageOS to use the Work, or any part thereof, in connection with StorageOS Guides \u0026amp; Tutorials or otherwise. Credit for the Work shall read, “Contributed by writer’s name.”\nPAYMENT. Upon publication of a submission to the StorageOS Guides \u0026amp; Tutorials Repository, the writer will be paid the sum of USD $250.00 as an electronic payment.\n","excerpt":"We are always looking to improve our documentation. If you like to help people and can write, read …","ref":"/v1.x/docs/reference/contributing/","title":"Contributing to the docs"},{"body":"StorageOS nodes can be decommissioned and removed from the cluster using the StorageOS CLI.\n This functionality is only available when StorageOS is deployed with KV_BACKEND=etcd, so the KV store is external to StorageOS.\n There are safeguards to make sure data is not lost unintentionally. Only nodes in state Offline can be removed from the StorageOS cluster. Note that once removed from the cluster, nodes may not partake in StorageOS operations, and may not run container applications that require StorageOS backed persistent storage.\nThe recommended procedure is as follows.\n Cordon the node\n$ storageos node cordon node03 node03 Drain the node\n$ storageos node drain node03 node03 Wait until the node drain is finished. Check the volumes located on that node with storageos node ls and wait until there are no Masters or Replicas on the drained node. If there are no eligible nodes for replicas to be created on, the drained node will keep hosting them.\n Stop the node\n Delete the node from the cluster\n$ storageos node delete node03 node03 ","excerpt":"StorageOS nodes can be decommissioned and removed from the cluster using the StorageOS CLI.\n This …","ref":"/v1.x/docs/operations/decommission-nodes/","title":"Decommissioning StorageOS Nodes"},{"body":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page.\nDocker EE and the Universal Control Plane can run on different Linux distributions. StorageOS supports RHEL, CentOS, Debian, and selected Ubuntu images. For more details, check out the supported OSs in the prerequisites page.\nStorageOS only supports Kubernetes nodes managed by Docker Enterprise Edition, not those nodes running Swarm. Mixed nodes (those running Kubernetes and Swarm workloads) are not supported. As a consequence, StorageOS volumes can only be provisioned on Kubernetes nodes, and only these nodes should be used for stateful workloads.\n","excerpt":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page. …","ref":"/v1.x/docs/platforms/dockeree/","title":"Docker Enterprise Edition"},{"body":"","excerpt":"","ref":"/v1.x/docs/","title":"Documentation"},{"body":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, most popularly used to aggregate logs, but also to serve as a search backend to a number of different applications.\nUsing StorageOS persistent volumes with ElasticSearch (ES) means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Elasticsearch pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nElasticsearch has features to allow it to handle data replication, and as such careful consideration of whether to allow StorageOS or Elasticsearch to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Elasticsearch on Kubernetes Prerequisites Some OS tuning is required, which is done automatically when using our example from the use cases repository.\nElasticsearch requires vm.max_map_count to be increased to a minimum of 262144, which is a system wide setting. One way to achieve this is to run sysctl -w vm.max_map_count=262144 and update /etc/sysctl.conf to ensure it persists over a reboot. See ElasicSearch reference here.\nAdministrators should be aware that this impacts the behaviour of nodes and that there may be collisions with other application settings. Administrators are advised to centrally collate sysctl settings using the tooling of their choice.\nDeployment of the application StatefulSet defintion ---apiVersion:apps/v1kind:StatefulSetmetadata:name:esdata[...]spec:serviceAccountName:elasticsearchcontainers:- name:dataimage:elasticsearch:6.7.0imagePullPolicy:IfNotPresent[...]volumeMounts:- name:datamountPath:/usr/share/elasticsearch/data/data[...]volumeClaimTemplates:- metadata:name:\u0026#34;data\u0026#34;spec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- default StorageOS storage class nameresources:requests:storage:10Gi# \u0026lt;--- change this to the appropriate valueThis excerpt is from the StatefulSet definition (/elasticsearch/10-es-data.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nInstallation Clone the use cases repo You can find the latest files in the StorageOS use cases repostiory in /elasticsearch/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases Create the kubernetes objects\nThis will install an ES cluster with 3 master, 3 data and 3 coordinator nodes. Combined they will require ~ 14 GiB of available memory in your cluster, however, more may be used as the application is being used\nkubectl apply -f ./elasticsearch/ Once completed, an internal service object will have been created making the cluster available as http://elasticsearch:9200/ which is the default Kibana (when installed via Helm) will be using.\n Confirm Elasticsearch is up and running\nkubectl get pods -l component=elasticsearch NAME READY STATUS RESTARTS AGE elasticsearch-exporter-d86ffd94-zw45l 1/1 Running 0 5m44s es-coordinator-b7b984dd4-7wlz5 1/1 Running 0 5m44s es-coordinator-b7b984dd4-89w26 1/1 Running 0 5m44s es-coordinator-b7b984dd4-b4t6j 1/1 Running 0 5m44s es-master-78dfd5b49f-9gf5c 1/1 Running 0 5m44s es-master-78dfd5b49f-smsbw 1/1 Running 0 5m44s es-master-78dfd5b49f-z4qpj 1/1 Running 0 5m44s esdata-0 1/1 Running 0 5m44s esdata-1 1/1 Running 0 4m34s esdata-2 1/1 Running 0 3m22s Connect to ElasticSearch\nTo connect to ES directly, you can use the following port-forward command\nkubectl port-forward svc/elasticsearch 9200 and then access it via http://localhost:9200\n Kibana (optional) One of the most popular uses of ES is to use it for log aggregation and indexing, Kibana helps us visualize the data in these indices and can be easily used when installed via its Helm chart\n Install the helm chart.\nhelm install stable/kibana Once installed, use a port-foward to Kibana instead of directly to ES\nkubectl port-forward --namespace default $(kubectl get pods --namespace default -l \u0026#34;app=kibana\u0026#34; -o jsonpath=\u0026#34;{.items[0].metadata.name}\u0026#34;) 5601 and then access it via http://localhost:5601\n Monitoring (optional) As part of the example deployment, ES metrics are exposed and can be scraped by Prometheus on port 9108 (see 77-es-exporter.yaml). This is enabled by default, and should work with the default Prometheus install via Helm. If you\u0026rsquo;re using the Prometheus service monitors, you can monitor this installation by creating a monitor for the es-exporter service. For an example of how this is done to monitor StorageOS, please see prometheus-setup.\n","excerpt":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, …","ref":"/v1.x/docs/usecases/elasticsearch/","title":"Elasticsearch"},{"body":"Encryption is enabled on a per volume basis using the storageos.com/encryption label (for more information see Encrypted Volumes).\nStorageOS encrypts volumes on disk using AES-256 in XTS-AES mode with 512 bit keys as recommended by NIST, with encryption keys being derived using HKDF.\nKeys and Initialisation vectors are generated using the crypto/rand package.\n Key Size Usage Volume 512 bits random data Used by XTS-AES to encrypt and decrypt disk blocks Namespace 256 bits random data Used to encrypt Volume keys Initialisation Vector 256 bits random data Used as \u0026lsquo;salt\u0026rsquo; when encrypting volume keys with namespace keys The components required to derive the encryption key are stored in a Kubernetes secret. By default these secrets are stored in the namespace that StorageOS is installed into. As Kubernetes secrets are only base64 encoded, it is recommend to encrypt secrets at rest. Alternatively a KMS such as HashiCorp Vault can be used.\nEach namespace has its own key that is created when the namespace is initialized. The namespace keys are stored as Kubernetes secrets named ns-key.{Namespace Name}. The namespace key secrets are created in whatever namespace StorageOS is installed into.\nIn the example below there are two ns-key secrets in the storageos namespace because a StorageOS volume has been provisioned in the default and mongo namespaces. A ns-key is created for a namespace regardless of whether an encrypted volume exists in the namespace or not.\n$ kubectl get secrets -n kube-system | grep ns-key ns-key.default Opaque 1 4d ns-key.mongo Opaque 1 5h A volume is encrypted with a volume key that is a randomly generated 512 bit key. Rather than storing the volume key, StorageOS stores an encrypted version of the volume key, called the volume user key, which is generated by encrypting the volume key with a 256 bit namespace key and 256 bit initialization vector. Each namespace has a unique key and a unique initialization vector is generated for each volume.\nThe volume key is discarded to avoid the key that encrypts user data being compromised. Whenever the volume needs to be decrypted the volume key is derived by decrypting the volume user key using the namespace key and initialisation vector that are stored.\nIn order to check that the volume key has been correctly derived, a key digest of the volume key is stored to verify the derived volume key is identical to the original key.\nUltimately this means that the volume user key, initialisation vector, a digest of the volume key and the namespace key are stored in a Kubernetes secret. See Encrypted Volumes for best practices regarding backing up StorageOS secrets.\n","excerpt":"Encryption is enabled on a per volume basis using the storageos.com/encryption label (for more …","ref":"/v1.x/docs/concepts/encryption/","title":"Encryption"},{"body":"Volumes can be encrypted when they are created using the storageos.com/encryption label. The labels can be passed to StorageOS using PVCs or you can directly create volumes using the StorageOS CLI or GUI with the encryption label applied.\nFor more in depth discussion of how encryption works please see the Encryption concepts page.\nRequired labels The storageos.com/encryption label must be applied to the volume when it is created. The encryption status of a volume cannot be changed after a volume has been created.\nYou can pass the label using a PVC\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc0002labels:\u0026#34;storageos.com/encryption\u0026#34;: \u0026#34;true\u0026#34;annotations:volume.beta.kubernetes.io/storage-class:fastspec:accessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also pass the encryption label when creating volumes using the CLI\n$ storageos volume create encrypted-volume --label storageos.com/encryption=true You can also add the encryption label when creating a volume with the GUI\nBacking up Secrets StorageOS generates the cryptographic keys that are used to encrypt data (see Encryption for more details). The keys that are used to encrypt a volume are stored in a Kubernetes secret. As such, StorageOS does not have access to the keys that are used to encrypt a volume and if the keys are lost the volume cannot be decrypted.\nAs a precautionary measure it is recommended that you backup the Kubernetes secrets used to store the encryption keys.\nStorageOS will create one secret per encrypted volume and the secrets are created in whatever namespace StorageOS is installed into.\n$ kubectl get secrets -n kube-system NAME TYPE DATA AGE ns-key.default Opaque 1 20h vol-key.4276fc07-7d85-70bf-35a0-f0b005e55e0f Opaque 4 1m In the output above there is a ns-key.default and a vol-key.\n A ns-key is created for each StorageOS namespace in the format ns-key.{namespace}. A vol-key is created for every encrypted volume. The vol-keys are named vol-key.{volume-id}. The volume id can be retrieved by inspecting the volume.\n # Find the PVC name $ kubectl get pvc --show-labels NAME STATUS VOLUME STORAGECLASS AGE LABELS pvc0002 Bound pvc-1c68f013-40dd-11e9-91ad-0a57700a78b4 fast 10m storageos.com/encryption=true # Inspect the volume and find the volume ID $ storageos volume inspect default/pvc-1c68f013-40dd-11e9-91ad-0a57700a78b4 | grep -m1 id \u0026#34;id\u0026#34;: \u0026#34;4276fc07-7d85-70bf-35a0-f0b005e55e0f\u0026#34;, # Find the secret for PVC pvc0002 $ kubectl get secret vol-key.4276fc07-7d85-70bf-35a0-f0b005e55e0f NAME TYPE DATA AGE vol-key.4276fc07-7d85-70bf-35a0-f0b005e55e0f Opaque 4 12m StorageOS recommends that vol-key and ns-keys are backed up. This can be done by outputting the secrets as yaml and storing the resulting files securely. The example below will output the ns-key.default to a ns-key.default.yaml file.\n$ kubectl get secret ns-key.default -o yaml \u0026gt; ns-key.default.yaml The vol-key secret contains all the keys necessary to decrypt a volume so ensure that backups of the vol-keys are stored securely.\nRestoring Secrets In order to restore backed up secrets use kubectl to create them. The secrets have a namespace field in the file themselves so a namespace does not need to be specified.\n$ kubectl create -f ns-key.default.yaml Keys can be restored while StorageOS is running and will be used dynamically by StorageOS.\n","excerpt":"Volumes can be encrypted when they are created using the storageos.com/encryption label. The labels …","ref":"/v1.x/docs/operations/encrypted-volumes/","title":"Encrypted Volumes"},{"body":"Several aspects of StorageOS behaviour can be controlled via environment variables. These can be injected in via any of the usual mechanisms such as ConfigMaps.\n Variable Name Valid in versions Description ADVERTISE_IP 1.0+ IP address of the node for incoming connections. Defaults to first non-loopback address API_PORT 1.0+ Port for the API to listen on. Defaults to 5705 (IANA Registered) CSI_ENDPOINT 1.0+ If set, CSI compatibility is enabled. Typically set to unix://var/lib/kubelet/plugins_registry/storageos/csi.sock. Default is unset CSI_VERSION 1.1.0+ Added in 1.1.0 to define what version of CSI to use. Can be set to v0 or v1, defaults to v0 DESCRIPTION 1.0+ The node description for display purposes only. Default is unset DEVICE_DIR 1.0+ Where the volumes are exported. This directory must be shared into the container using the rshared volume mount option. Defaults to /var/lib/storageos/volumes DFS_PORT 1.0+ Port for DirectFS to listen on. Defaults to 5703 DISABLE_ERROR_REPORTING 1.0+ To disable error reporting across the cluster, set to true. Defaults to false. Errors are reported to help identify and resolve potential issues that may occur DISABLE_TCMU 1.3+ Prevents TCMU from being used. On systems where another program is already using TCMU this should be set to true. See System Configuration for more information. DISABLE_TELEMETRY 1.0+ To disable anonymous usage reporting across the cluster, set to true. Defaults to false. To help improve the product, data such as API usage and StorageOS configuration information is collected FORCE_TCMU 1.3+ Force the dataplane to use TCMU. If set to true StorageOS will not start if TCMU fails to start. HOSTNAME 1.0+ Hostname of the node, only if you wish to override it. In Kubernetes environments, typically set to spec.nodeName IN_K8S_CLUSTER 1.0+ Toggles enhanced Kubernetes integration. Defaults to true and will be disabled automatically if Kubernetes API is not accessible. Requires StorageOS to be deployed as a DaemonSet or Pod. JOIN 1.0+ Required A join token and/or list of cluster nodes to join. The first node will bootstrap the cluster. See cluster discovery. There is no default; this must be set for multiple-node clusters KUBECONFIG 1.0+ Path to local kubeconfig file. Not normally required. Default is unset KV_ADDR 1.0+ Comma separated list of etcd targets, in the form ip[:port]. Must be specified with KV_BACKEND=etcd. Prefer multiple direct endpoints over a single load-balanced endpoint KV_BACKEND 1.0+ Type of KV store to use. Defaults to embedded. etcd is supported with KV_ADDR set to an external etcd instance KV_CLIENT_PORT 1.0+ Port for the embedded Key/Value store. Defaults to 5706 KV_PEER_PORT 1.0+ Port for the embedded Key/Value store. Defaults to 5707 LABELS 1.0+ Comma separated list of node labels. e.g. LABELS=country=us,env=prod. Default is unset LOG_FILTER 1.0+ Used to discard log messages based on category. e.g. LOG_FILTER=cp=info,dp=info,etcd=debug. Default is unset LOG_FORMAT 1.0+ Logging output format, one of text or json. Defaults to json LOG_LEVEL 1.0+ One of debug, info, warning or error. Defaults to info NAMESPACE 1.0+ The orchestrator namespace that StorageOS is running in. Used as the location to store encryption keys in. Defaults to storageos NATS_CLUSTER_PORT 1.0+ Port for the NATS cluster service to listen on. Defaults to 5710 NATS_HTTP_PORT 1.0+ Port for the NATS HTTP server to listen on. Defaults to 5709 NATS_PORT 1.0+ Port for NATS messaging to listen on. Defaults to 5708 PASSWORD 1.0+ Password to authenticate to the API with. Defaults to storageos PROBE_INTERVAL 1.1.2+ The interval between node probes. Takes a time duration in string format e.g. 500ms or 2s. Setting this lower (more frequent) will cause the cluster to detect failed nodes more quickly at the expense of increased bandwidth usage. Defaults to 1000ms. Added in 1.1.2 replacing PROBE_INTERVAL_MS PROBE_INTERVAL_MS 1.1.1 The interval in milliseconds between node probes. Setting this lower (more frequent) will cause the cluster to detect failed nodes more quickly at the expense of increased bandwidth usage. Defaults to 1000ms. Added in 1.1.1 and deprecated in 1.1.2 See PROBE_INTERVAL PROBE_TIMEOUT 1.1.2+ The timeout to wait for an ack from a probed node before assuming it is unhealthy. Takes a time duration in string format e.g. 500ms or 2s. This should be set to 99-percentile of RTT (round-trip time) on your network. Defaults to 3000ms. Added in 1.1.2 replacing PROBE_TIMEOUT_MS PROBE_TIMEOUT_MS 1.1.1 The timeout to wait for an ack from a probed node before assuming it is unhealthy. This should be set to 99-percentile of RTT (round-trip time) on your network. Defaults to 3000ms. Added in 1.1.1 and deprecated in 1.1.2 See PROBE_TIMEOUT SERF_PORT 1.0+ Port for the Serf protocol to listen on. Defaults to 5711 TLS_ETCD_CA 1.2.0 The file path to the etcd server Certificate Authority certificate. The certificate should be mounted into the container using a Secret. See etcd TLS TLS_ETCD_CLIENT_CERT 1.2.0 The file path to the etcd server client certificate. The certificate should be mounted into the container using a Secret. See etcd TLS TLS_ETCD_CLIENT_KEY 1.2.0 The file path to the etcd server client key. The certificate should be mounted into the container using a Secret. See etcd TLS USERNAME 1.0+ Username to authenticate to the API with. Defaults to storageos ","excerpt":"Several aspects of StorageOS behaviour can be controlled via environment variables. These can be …","ref":"/v1.x/docs/reference/envvars/","title":"Environment Variables"},{"body":"StorageOS uses etcd to store cluster metadata. Because of the strong consistency model that etcd enforces, StorageOS metadata operations are guaranteed to be atomic and consistent.\nInstallation options Before installing StorageOS, an etcd cluster needs to be prepared. There are different topologies that fulfil this prerequisite.\n External etcd (Production) etcd as Pods (Testing) Production Testing External Etcd The production topology is designed to provide the highest stability for the etcd cluster. It is necessary for normal StorageOS function to have a reliable metadata cluster. Otherwise, central operations such as provisioning, attachment or failover of volumes cannot be performed. In the event that etcd becomes unavailable, StorageOS clusters become read only, allowing access to data but preventing metadata changes.\nIt is recommended to install etcd out of the scope of the orchestrator wherever possible. Following CoreOS best practices, a minimum of 3 independent nodes should be dedicated to etcd. StorageOS doesn\u0026rsquo;t require a high performance etcd cluster as the throughput of metadata to the cluster is low. Depending on the level of redundancy you feel comfortable with you can install etcd on the Kubernetes Master nodes. Take extreme care to avoid collisions of the StorageOS etcd installation with the Kubernetes etcd when using the Kubernetes Master nodes. Precautions such as changing the default configuration for the client and peer ports, and ensuring the etcd data directory is modified. The ansible playbook below will default the etcd installation directory to /var/lib/storageos-etcd.\nInstallation If you are familiar with etcd, you can proceed with the CoreOS instructions to install etcd, otherwise this section lays out out an example installation using Ansible.\n Clone StorageOS Helper repository\ngit clone https://github.com/storageos/deploy.git cd k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd Edit the inventory file\n Target the nodes that install etcd, where the file hosts.example serves as an example. The ip parameter is needed for each node.\n $ cat hosts.example [nodes] centos-1 ip=172.28.128.14 centos-2 ip=172.28.128.15 centos-3 ip=172.28.128.16 # Edit the inventory file $ vi hosts.example # Or your own inventory file Edit the etcd configuration\n If targeting Kubernetes Master nodes, you must change etcd_port_client, etcd_port_peers\n $ cat group_vars/all etcd_version: \u0026#34;3.3.18\u0026#34; etcd_port_client: \u0026#34;2379\u0026#34; etcd_port_peers: \u0026#34;2380\u0026#34; etcd_quota_bytes: 8589934592 # 8 GB etcd_auto_compaction_mode: \u0026#34;revision\u0026#34; etcd_auto_compaction_retention: \u0026#34;100\u0026#34; members: \u0026#34;{{ groups[\u0026#39;nodes\u0026#39;] }}\u0026#34; installation_dir: \u0026#34;/var/lib/storageos-etcd\u0026#34; $ vi group_vars/all Install\nansible-playbook -i hosts.example site.yml Verify installation\n The playbook installs the etcdctl binary on the nodes, at /usr/local/bin.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 member list 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Managed Services When running StorageOS on Managed Kubernetes services it may not be possible to deploy with the Production etcd topology described above. It is therefore recommended to deploy etcd on its own as much as possible, even if that means deploying 3 independent VMs for etcd to run on.\nAs managed services treat nodes as ephemeral resources, if the orchestration deletes the 3 nodes hosting etcd, the result will be catastrophic and a restore from a backup will be needed.\nIf it is not possible to deploy independent VMs for etcd, etcd can be deployed as pods, inside the cluster. This configuration requires an awareness of the stability that etcd requires. You can use the etcd-as-pods installation option, but be aware of the precautions that need to be taken.\nWhy External Etcd etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. In the case that quorum is lost, an etcd node stops and marks its contents as read-only. It cannot guarantee that the data being held is valid. Another peer might have a newer version that has not been delivered. Quorum is fundamental for etcd operations.\nIn a Kubernetes environment, applications are scheduled across and in some scenarios such as \u0026ldquo;DiskPressure\u0026rdquo; they may need to be evicted from a node, and be scheduled onto a different node. With an application such as etcd, the scenario described can result in quorum being lost, making the cluster unable to recover automatically. Usually a 3 node etcd cluster can survive losing one node and recover. However, losing a second node at the same time or even having a network partition between them will result in quorum lost.\nBind Etcd IPs to Kubernetes Service Kubernetes external services use a DNS name to reference external endpoints. You can use the example from the helper github repository to deploy the external Service. That might be of use when monitoring etcd from Prometheus.\n Etcd as Pods etcd can be deployed in Kubernetes using the official etcd-operator.\nDeploying etcd in Kubernetes makes the etcd installation very easy, however be aware that even though the official etcd-operator is maintained by RedHat, it hasn\u0026rsquo;t been under active development since 2019. As such it may be considered an archived project. For an actively maintained etcd Operator you might want to check the Improbable etcd Operator.\nExamples of deploying etcd clusters using the etcd-operator on Kubernetes and OpenShift are available.\nSince Kubernetes 1.16 the deployment api uses \u0026ldquo;apps/v1\u0026rdquo;. Once you have cloned the coreos etcd operator repository, you will need to change the apiVersion of the file \u0026ldquo;examples/deployment.yaml\u0026rdquo; from extensions/v1beta1 to apps/v1.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. Make sure you take frequent backups of the etcd cluster as it holds all the StorageOS cluster metadata.\nKnown etcd-operator issues This topology is only recommended for deployments where isolated nodes cannot be used.\netcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd in pods it is therefore important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster is not possible unless a backup is restored.\n StorageOS and Etcd When installing StorageOS, the etcd endpoints are passed in a StorageOSCluster Custom Resource.\nFor instance:\napiVersion: \u0026quot;storageos.com/v1\u0026quot; kind: StorageOSCluster metadata: name: \u0026quot;storageos\u0026quot; spec: secretRefName: \u0026quot;storageos-api\u0026quot; # Reference from the Secret created in the previous step secretRefNamespace: \u0026quot;default\u0026quot; # Namespace of the Secret (...) kvBackend: address: 'storageos-etcd-client.etcd:2379' # Example address, change for your etcd endpoint #address: '10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379' # You can set etcd server ips backend: 'etcd' Note the kvBackend.address section.\n For full Custom Resource documentation check StorageOSCluster resource definition.\nBest practices StorageOS uses etcd as a service, whether it is deployed following the above instructions or as a custom installation. It is expected that the user maintains the availability and integrity of the etcd cluster.\nIt is highly recommended to keep the cluster backed up and ensure high availability of its data. It is also important to keep the latency between StorageOS nodes and the etcd replicas low. Deploying an etcd cluster in a different data center or region can make StorageOS detect etcd nodes as unavailable due to latency. A 10ms latency between StorageOS and etcd would be the maximum threshold for proper functioning of the system.\nMonitoring It is highly recommended to add monitoring to the etcd cluster. etcd serves Prometheus metrics on the client port http://etc-url:2379/metrics.\nYou can use StorageOS developed Grafana Dashboards for etcd. When using etcd for production, you can use the etcd-cluster-as-service, while the etcd-cluster-as-pod can be used when using etcd from the operator.\nDefragmentation etcd uses revisions to store multiple versions of keys. Compaction removes all key revision prior to a certain revision from etcd. Typically the etcd configuration enables the automatic compaction of keys to prevent performance degradation and limit the storage required. Compaction of revisions can create fragmentation that means space on disk is available for use by etcd but is unavailable for use by the file system. In order to reclaim this space, etcd can be defragmented.\nReclaiming space is important because when the etcd database file grows over the \u0026ldquo;DB_BACKEND_BYTES\u0026rdquo; parameter, the cluster triggers an alarm and sets itself read only and only allows reads and deletes. To avoid hitting the db backend bytes limit, compaction and defragmentation are required. How often defragmentation is required depends on the churn of key revisions in etcd.\nThe Grafana Dashboards mentioned above indicate when nodes require defragmentation. Be aware that defragmentation is a blocking operation that is performed per node, hence the etcd node will be locked for the duration of the defragmentation. Defragmentation usually takes a few milliseconds to complete.\n","excerpt":"StorageOS uses etcd to store cluster metadata. Because of the strong consistency model that etcd …","ref":"/v1.x/docs/operations/external-etcd/","title":"Etcd"},{"body":"In order to understand what Fencing is and why it is a useful feature it\u0026rsquo;s important to understand the behaviour of StatefulSets.\nKubernetes does reschedule pods from some controllers when nodes become unavailable. The default behaviour is that when a node becomes unavailable its status becomes \u0026ldquo;Unknown\u0026rdquo; and after the pod-eviction-timeout has passed pods are scheduled for deletion. By default, the pod-eviction-timeout is five minutes.\nStatefulSets are the de facto Kubernetes controller to use for stateful applications. The StatefulSet controller offers guarantees around pod uniqueness, sticky identities and the persistence of PVCs beyond the lifetime of the pod. As such, StatefulSets have different characteristics and provide different guarantees than Deployments.\nDeployments guarantee the amount of healthy replicas by reconciling the state of the cluster with the declared desired state. Attempts to align the cluster state with the desired state happen as fast as possible by aggressively initializing and terminating pods. If one pod is terminating, another will be automatically scheduled to start even if the first pod is not yet completely terminated. Stateless applications benefit from this behaviour as one pod executes the same work as any other in the deployment.\nStatefulSets, on the other hand, guarantee that every pod scheduled has a unique identity, which is to say that only a single copy of the pod is running in the cluster at any one time. Whenever scheduling decisions are made, the StatefulSet controller ensures that only one copy of a pod is running. If a pod is deleted, a new pod will not be scheduled until the first pod is fully terminated. This is an important guarantee considering that FileSystems need to be unmounted before they can be remounted in a new pod. Any PVC defining a device requires this behaviour to ensure the consistency of the data and thus the PVC.\nAs a consequence of the guarantee of unique pod identity, StatefulSet pods don\u0026rsquo;t get rescheduled upon node failures. This is because Kubernetes is unable to reason about whether the node is temporarily unavailable due to a network partition or if the node has crashed. Therefore, the StatefulSet controller cannot guarantee that if it reschedules an unavailable pod that the pod is not still running. The original pod would be running on the partitioned node and the rescheduled pod would be running on a different node, in violation of the StatefulSet guarantee of pod uniqueness. Instead, the StatefulSet controller slates the pod on the partitioned node for termination, but since there is no communication between the node and the control plane, the termination cannot be actioned by the partitioned node. The control plane will then mark the pod status as \u0026ldquo;Unknown\u0026rdquo; while it waits for the partition to heal or for a manual intervention by the cluster operator. The partitioned node doesn\u0026rsquo;t make any changes to its pods as there\u0026rsquo;s no communication between the node and the Kubernetes control plane.\nFor more information on the rationale behind the design of StatefulSets please see the Kubernetes design proposal for Pod Safety\nHA for StatefulSet applications can be achieved with the StorageOS Fencing feature.\nStorageOS implements a feature known as Fencing. With Fencing, pods that are scheduled on a failed node can be terminated by StorageOS allowing the pods to be scheduled on a different node. StorageOS can determine if a pod should be rescheduled by leveraging StorageOS health checks that are already used to ensure high availability of data and failover. Without Fencing, the pod will be slated for termination but this can only be actioned when the unavailable node rejoins the cluster or the unavailable node is deleted from the cluster.\nAs explained in StatefulSet Behaviour, the StatefulSet controller is conservative by design, given the constraints of various types of persistent volumes that can be managed in Kubernetes. For certain workloads, when StorageOS has declared a node offline it may be desirable to promote faster pod rescheduling by allowing StorageOS to Fence pods on the unavailable node. By enabling Fencing, StatefulSet pods have a much shorter time to recover (TTR) than usual and no manual intervention is required for StatefulSet pods on failed nodes to be rescheduled. Hence, the combination of StorageOS volume failover and StorageOS Fencing makes an application more resilient to node failures with automatic recovery and a 30-60 second TTR.\nAs a StorageOS pod runs on each node in the cluster that consumes or presents storage, and these nodes communicate using a gossip protocol, StorageOS has additional insight into whether a node cannot communicate with the master or if the node is truly unavailable. Additionally due to the synchronous replication of StorageOS volumes, any writes made to the volume on the partitioned node will fail as the writes cannot be acknowledged by replica volumes. Therefore, in a scenario where the node is unreachable StorageOS knows that the pod will lose access to its data so it is safe for StorageOS to force the pod to be rescheduled.\nFor more information about how to enable Fencing please see our Fencing Operations page.\n","excerpt":"In order to understand what Fencing is and why it is a useful feature it\u0026rsquo;s important to …","ref":"/v1.x/docs/concepts/fencing/","title":"Fencing"},{"body":"For information regarding the fencing feature please see our Fencing concepts page.\nEnabling StorageOS to fence a pod In order to allow StorageOS to fence a pod scheduled on an unavailable node, a pod must have the following:\n storageos.com/fenced=true label At least one StorageOS volume mounted Each StorageOS volume the pod mounts must have at least one healthy replica. Fencing is not disabled across the StorageOS cluster - DISABLE_FENCING environment variable is not set to true N.B. Any pod that is to be fenced must meet the criteria above\n A pod created by the StatefulSet manifest below would be able to be fenced. The pod has the storageos.com/fenced=true label, mounts a StorageOS volume - vol and the StorageOS volume has a replica. Note that volumeClaimTemplates inherit labels from the StatefulSet i.e. the replica label.\napiVersion:apps/v1kind:StatefulSetmetadata:name:debian-statefulspec:selector:matchLabels:app:\u0026#34;debian-stateful\u0026#34;storageos.com/fenced:\u0026#34;true\u0026#34;storageos.com/replicas:\u0026#34;1\u0026#34;serviceName:\u0026#34;default\u0026#34;replicas:1template:metadata:labels:app:\u0026#34;debian-stateful\u0026#34;storageos.com/fenced:\u0026#34;true\u0026#34;storageos.com/replicas:\u0026#34;1\u0026#34;spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- name:volmountPath:/mntvolumeClaimTemplates:- metadata:name:volspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:1GiDisable Fencing Although fencing is enabled in a StorageOS cluster by default, pods will not be fenced unless the conditions above are met.\nHowever, to completely disable fencing in a StorageOS cluster the environment variable DISABLE_FENCING=true can be set.\n","excerpt":"For information regarding the fencing feature please see our Fencing concepts page.\nEnabling …","ref":"/v1.x/docs/operations/fencing/","title":"Fencing"},{"body":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. StorageOS implements dynamic provisioning, so the creation of a PVC will automatically provision a PV (PersistentVolume) that can be used to persist data written by a Pod.\nCreate the PersistentVolumeClaim You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic PVC definition\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;fast\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC will dynamically provision a 5GB volume using the fast StorageClass. This StorageClass was created during the StorageOS install and triggers creation of a PeristentVolume by StorageOS.\nFor installations with CSI, you can create multiple StorageClasses in order to specify default labels.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volumeprovisioner:storageos# Provisioner when using CSIThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels. For simplicities sake this example will use unreplicated volumes.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# Reference to the StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1labels:storageos.com/replicas:\u0026#34;1\u0026#34;spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC has the storageos.com/replicas label set. This label tells StorageOS to add a replica for the volume that is created. For the sake of keeping this example simple an unreplicated volume will be used.\n Move into the examples folder and create a PVC using the PVC definition above.\n$ # from storageos-usecases/00-basic $ kubectl create -f ./pvc-basic.yaml You can view the PVC that you have created with the command below\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-1 Bound pvc-f8ffa027-e821-11e8-bc0b-0ac77ccc61fa 5Gi RWO fast 1m Create a pod that mounts the PVC created in step 2.\n$ kubectl create -f ./pod.yaml The command above creates a Pod that uses the PVC that was created in step 1.\napiVersion:v1kind:Podmetadata:name:d1spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mntname:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-1In the Pod definition above volume v1 references the PVC created in step 2, and is mounted in the pod at /mnt. In this example a debian image is used for the container but any container image with a shell would work for this example.\n Confirm that the pod is up and running\n$ kubectl get pods NAME READY STATUS RESTARTS AGE d1 1/1 Running 0 1m Execute a shell inside the container and write some contents to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo \u0026#34;Hello World!\u0026#34; \u0026gt; /mnt/helloworld root@d1:/# cat /mnt/helloworld Hello World! By writing to /mnt inside the container, the StorageOS volume created by the PVC is being written to. If you were to kill the pod and start it again on a new node, the helloworld file would still be avaliable.\nIf you wish to see more use cases with actual applications please see our Use Cases documentation.\n ","excerpt":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. …","ref":"/v1.x/docs/operations/firstpvc/","title":"StorageOS Volume Guide"},{"body":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any of the nodes in the cluster. Initally you can log in as the default administrator, using the username storageos and password storageos.\nManage cluster nodes and pools The nodes and pools page allow you to manage cluster nodes and storage pool. In this example, this cluster consists of three nodes with 35.9GB capacity each. The default storage pool contains all three nodes, giving a total of 107.6GB.\nCreate and view volumes You can create volumes, including replicated volumes, and view volume details:\nManaging volumes with namespaces and rules Volumes can be namespaced across different projects or teams, and you can switch namespace using the left hand panel:\nData policy and placement is enforced using rules:\n","excerpt":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any …","ref":"/v1.x/docs/reference/gui/","title":"Graphical user interface (GUI)"},{"body":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the status of the components on nodes in the cluster.\n$ storageos cluster health NODE ADDRESS CP_STATUS DP_STATUS storageos-1 192.168.50.100 Healthy Healthy storageos-2 192.168.50.101 Healthy Healthy storageos-3 192.168.50.102 Healthy Healthy The API server reports its status at the /v1/health endpoint.\n$ curl -v http://localhost:5705/v1/health * Trying ::1... * Connected to localhost (::1) port 5705 (#0) \u0026gt; GET /v1/health HTTP/1.1 \u0026gt; Host: localhost:5705 \u0026gt; User-Agent: curl/7.47.0 \u0026gt; Accept: */* \u0026gt; \u0026lt; HTTP/1.1 200 OK \u0026lt; Access-Control-Allow-Headers: Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization \u0026lt; Access-Control-Allow-Methods: POST, GET, OPTIONS, PUT, DELETE \u0026lt; Access-Control-Allow-Origin: * \u0026lt; Content-Type: application/json \u0026lt; Date: Fri, 11 Aug 2017 12:07:55 GMT \u0026lt; Content-Length: 539 \u0026lt; * Connection 0 to host localhost left intact {\u0026#34;submodules\u0026#34;:{\u0026#34;kv\u0026#34;:{\u0026#34;status\u0026#34;:\u0026#34;alive\u0026#34;,\u0026#34;message\u0026#34;:\u0026#34;\u0026#34;,\u0026#34;updatedAt\u0026#34;:\u0026#34;2017-08-16T14:24:59.898288145Z\u0026#34;,\u0026#34;changedAt\u0026#34;:\u0026#34;2017-08-16T13:06:18.672362683Z\u0026#34;},\u0026#34;kv_write\u0026#34;:{\u0026#34;status\u0026#34;:\u0026#34;alive\u0026#34;,\u0026#34;message\u0026#34;:\u0026#34;\u0026#34;,\u0026#34;updatedAt\u0026#34;:\u0026#34;2017-08-16T14:24:59.898289093Z\u0026#34;,\u0026#34;changedAt\u0026#34;:\u0026#34;2017-08-16T13:06:27.475859537Z\u0026#34;},\u0026#34;nats\u0026#34;:{\u0026#34;status\u0026#34;:\u0026#34;alive\u0026#34;,\u0026#34;message\u0026#34;:\u0026#34;\u0026#34;,\u0026#34;updatedAt\u0026#34;:\u0026#34;2017-08-16T14:24:59.898287588Z\u0026#34;,\u0026#34;changedAt\u0026#34;:\u0026#34;2017-08-16T13:06:27.475858077Z\u0026#34;},\u0026#34;scheduler\u0026#34;:{\u0026#34;status\u0026#34;:\u0026#34;alive\u0026#34;,\u0026#34;message\u0026#34;:\u0026#34;\u0026#34;,\u0026#34;updatedAt\u0026#34;:\u0026#34;2017-08-16T14:24:59.898288556Z\u0026#34;,\u0026#34;changedAt\u0026#34;:\u0026#34;2017-08-16T13:06:27.475859095Z\u0026#34;}}} ","excerpt":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the …","ref":"/v1.x/docs/operations/health/","title":"Cluster health"},{"body":"StorageOS uses the storage available on the nodes where it is installed to present as available for volumes.\nIn order to mitigate against problems caused by filling the host root disk, we recommend mounting a separate device into the /var/lib/storageos directory. StorageOS is agnostic to the type of filesystem mounted in /var/lib/storageos.\nExtending Available Storage StorageOS uses subdirectories of /var/lib/storageos/data to hold user data. By default, the directory /var/lib/storageos/data/dev1 will be created when a node is bootstrapped, and used for pool data. It is possible to shard the data by creating more directories into this structure. StorageOS will save data in any directory that conforms to the pattern /var/lib/storageos/data/dev[0-9]+, such as /var/lib/storageos/data/dev2 or /var/lib/storageos/data/dev5. This functionality enables operators to mount different devices into devX directories and StorageOS will recognise them as available storage automatically.\nThere are two possible options to expand the available disk space for StorageOS to allocate:\n Mount filesystem in /var/lib/storageos/data/devX Use LVM to expand the logical volume available to StorageOS Option 1: Mount Additional Devices This option enables operators to expand the cluster\u0026rsquo;s available space at any time without having to stop applications or forcing operational downtime. The expansion of disk is transparent for applications and StorageOS Volumes. StorageOS will use the new available space to create new data files.\n Context\nWe assume that there is a disk available in our Linux system without formatting in addition to the root filesystem. StorageOS data dir dev1 (/var/lib/storageos/data/dev1) is using /dev/xvda1. We will use the device /dev/xvdf to expand StorageOS available space.\nList available block devices in the host.\nroot@ip-172-20-58-239:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\nroot@ip-172-20-58-239:~# storageos node ls --format=\u0026quot;table {{.Name}}\\t{{.Capacity}}\u0026quot; NAME TOTAL ip-172-20-119-113.eu-west-1.compute.internal 128.7GB ip-172-20-58-239.eu-west-1.compute.internal 128.7GB ip-172-20-68-139.eu-west-1.compute.internal 128.7GB ip-172-20-84-11.eu-west-1.compute.internal 128.7GB Format device\nroot@ip-172-20-58-239:/var/lib/storageos/data# mkfs -t ext4 /dev/xvdf mke2fs 1.42.12 (29-Aug-2014) Creating filesystem with 26214400 4k blocks and 6553600 inodes Filesystem UUID: 380712fa-6f82-477a-81a5-d7466d4c6b7f Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 4096000, 7962624, 11239424, 20480000, 23887872 Allocating group tables: done Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done Mount filesystem\nroot@ip-172-20-58-239:~# mkdir -p /var/lib/storageos/data/dev2 root@ip-172-20-58-239:~# mount /dev/xvdf /var/lib/storageos/data/dev2 Verify available storage\nIn less than 30 seconds, StorageOS will see the new available capacity.\nroot@ip-172-20-58-239:~# storageos node ls --format=\u0026quot;table {{.Name}}\\t{{.Capacity}}\u0026quot; NAME TOTAL ip-172-20-119-113.eu-west-1.compute.internal 128.7GB ip-172-20-58-239.eu-west-1.compute.internal 227.3GB ip-172-20-68-139.eu-west-1.compute.internal 128.7GB ip-172-20-84-11.eu-west-1.compute.internal 128.7GB root@ip-172-20-58-239:/var/lib/storageos/data/dev2# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk /var/lib/storageos/data/dev2 Note that the node ip-172-20-58-239.eu-west-1.compute.internal has increased the TOTAL capacity in 100Gi.\n Persist the mount at boot by adding the mount endpoint to /etc/fstab\n Option 2: Expand Existing Devices Backed by LVM This option enables operators to take advantage of LVM to manage disks.\n Context\nWe assume that /var/lib/storageos is mounted onto an LVM volume. We are using a volumegroup named storageos and logical volume called data. There is a second physical disk /dev/xvdg unused.\nList available block devices in the host.\nroot@ip-172-20-84-11:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk `-storageos-data 254:0 0 99G 0 lvm /var/lib/storageos xvdg 202:96 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\nroot@ip-172-20-84-11:~# storageos node ls --format=\u0026quot;table {{.Name}}\\t{{.Capacity}}\u0026quot; NAME TOTAL ip-172-20-119-113.eu-west-1.compute.internal 128.7GB ip-172-20-58-239.eu-west-1.compute.internal 128.7GB ip-172-20-68-139.eu-west-1.compute.internal 128.7GB ip-172-20-84-11.eu-west-1.compute.internal 100.3GB # --\u0026gt; LVM storageos/data volume Add physical disk to LVM\nroot@ip-172-20-84-11:~# vgextend storageos /dev/xvdg Volume group \u0026quot;storageos\u0026quot; successfully extended The volume group storageos must have 2 physical volumes (#PV)\nroot@ip-172-20-84-11:~# vgs VG #PV #LV #SN Attr VSize VFree storageos 2 1 0 wz--n- 199.99g 104.99g Extend logical volume data\nroot@ip-172-20-84-11:~# lvextend -L+100G /dev/storageos/data Size of logical volume storageos/data changed from 95.00 GiB (24320 extents) to 195.00 GiB (49920 extents). Logical volume data successfully resized Resize the FileSystem\n Your filesystem must support the option to be expanded, and to do so while in use. Otherwise, you need to unmount first.\n root@ip-172-20-84-11:~# resize2fs /dev/storageos/data resize2fs 1.42.12 (29-Aug-2014) Filesystem at /dev/storageos/data is mounted on /var/lib/storageos; on-line resizing required old_desc_blocks = 6, new_desc_blocks = 13 The filesystem on /dev/storageos/data is now 51118080 (4k) blocks long. Check new available space\nThe mounted file system to /var/lib/storageos has increased its size.\nroot@ip-172-20-84-11:~# df -h /dev/mapper/storageos-data Filesystem Size Used Avail Use% Mounted on /dev/mapper/storageos-data 192G 60M 183G 1% /var/lib/storageos StorageOS available storage has increased too.\nroot@ip-172-20-84-11:~# storageos node ls --format=\u0026quot;table {{.Name}}\\t{{.Capacity}}\u0026quot; NAME TOTAL ip-172-20-119-113.eu-west-1.compute.internal 128.7GB ip-172-20-58-239.eu-west-1.compute.internal 128.7GB ip-172-20-68-139.eu-west-1.compute.internal 128.7GB ip-172-20-84-11.eu-west-1.compute.internal 201GB # --\u0026gt; 100G more available Persist the mount at boot by adding the mount point to /etc/fstab\n ","excerpt":"StorageOS uses the storage available on the nodes where it is installed to present as available for …","ref":"/v1.x/docs/operations/managing-host-storage/","title":"Managing Host Storage"},{"body":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init container that sets the system configuration for StorageOS. The container also manages configuration changes required when upgrading StorageOS versions.\nThe container belongs to the DaemonSet that the StorageOS Cluster Operator starts when a StorageOSCluster resource is created. The storageos-init container is executed as an initContainer as part of a Kubernetes Pod. Therefore, only successful execution of the storageos-init container processes will result in the main container starting.\nScript Framework The code responsible for fulfilling the requirements is based on a Script Framework.\nThe script framework executes a set of scripts, performing checks, verifications and other procedures needed for StorageOS to be able to start. The scripts stdout and stderr are written to the stdout and stderr of the init app. The container logs contain all the logs of the individual scripts that run. The exit statuses of the scripts are used to determine initialization failure or success. Any non-zero exit status is logged as an event in the Kubernetes Pod events.\nIf any of the scripts fail, the storageos-init container will propagate the failure to Kubernetes, showing the status of the Pod as Init:Err.\nTo view the output of all storageos-init containers the following command can be used:\nkubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init For more details, check the StorageOS init container project.\nScripts executed The storageos-init container executes the following scripts.\n enable-lio dbupgrade ","excerpt":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init …","ref":"/v1.x/docs/reference/init-container/","title":"Init container"},{"body":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a StorageOS persistent volume being mounted on /var/jenkins_home. Deploying Jenkins using StorageOS offers multiple benefits. Firstly Jenkins can spin up multiple build pods at once to allow concurrent builds of different projects. Secondly Jenkins configuration is on a PersistentVolume so even if the Jenkins pod is rescheduled the configuration will persist.\nUsing StorageOS volume replicas allows for failure of nodes holding the PersistentVolume without interrupting Jenkins. Lastly by enabling StorageOS fencing Jenkins time to recover, in case of node failures, is greatly reduced.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information\nDeploying Jenkins on Kubernetes You can find the latest files in the StorageOS example deployment repository.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./jenkins Confirm that Jenkins is up and running\n$ kubectl get pods -w -l app=jenkins NAME READY STATUS RESTARTS AGE jenkins-0 1/1 Running 0 1m Connect to the Jenkins UI through the Jenkins service.\nYou can do this by port forwarding the Jenkins Kubernetes service to your localhost and accessing the UI via your browser. Alternatively if you have network access to your Kubernetes nodes then you can create a NodePort service and access Jenkins like that. A NodePort service has been left in 10-service.yaml commented out.\nTo port-foward the Jenkins service use the following command.\n$ kubectl port-foward svc/jenkins 8080 To login to the Jenkins UI use the credentials specified in 07-config.yaml, unless these have been changed from the defaults the username/password is admin/password.\n Create a Jenkins job.\nOnce you are logged into the UI you can create a job that will be farmed out to a Kubernetes plugin build agent. Click New Item, enter a name for the project and select Freestyle project. Next add an Execute shell build step. As a proof of concept you can use the bash below to have the pod execute a sleep.\n#!/bin/bash sleep 1000 Save the project and select Schedule a build of your project. You can watch for the appearance of a build pod using kubectl get pods -l jenkins=agent -w. Once the pod is created you should see the Build Executor status in the Jenkins UI display the pod.\nTo see multiple projects being built at once create another project and try scheduling a build of both projects at the same time.\n ","excerpt":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a …","ref":"/v1.x/docs/usecases/jenkins/","title":"Jenkins"},{"body":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub and traditional queues.\nUsing StorageOS persistent volumes with Apache Kafka means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the kafka pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nKafka has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Kafka to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Apache Zookeeper is required by Kafka to function; we assume it to already exist and be accessible within the Kubernetes cluster as zookeeper, see how to run Zookeeper with StorageOS here StorageOS is assumed to have been installed; please check for the latest available version here Kafka pods require 1536 MB of memory for successful scheduling Helm To simplify the deployment of kafka, we\u0026rsquo;ve used this Kafka helm chart (incubator) (version 0.13.8, app version 5.0.1) and rendered it into the example deployment files you can find in our GitHub repo.\nClone the use cases repo You can find the latest files in the StorageOS use cases repository in /kafka/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition\n---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:kafkalabels:app:kafka...spec:serviceName:kafka-headlesspodManagementPolicy:OrderedReadyupdateStrategy:type:OnDeletereplicas:3# \u0026lt;--- number of kafa pods to runtemplate:...spec:serviceAccountName:kafkacontainers:...- name:kafka-brokerimage:\u0026#34;confluentinc/cp-kafka:5.0.1\u0026#34;imagePullPolicy:\u0026#34;IfNotPresent\u0026#34;...volumeMounts:- name:datadirmountPath:\u0026#34;/var/data\u0026#34;volumes:- name:jmx-configconfigMap:name:kafka-metricsterminationGracePeriodSeconds:60volumeClaimTemplates:- metadata:name:datadirspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]resources:requests:storage:10Gi# \u0026lt;--- storage requested for each podstorageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Create the kubernetes objects\nkubectl apply -f ./kafka/ Confirm kafka is up and running\n$ kubectl get pods -l app=kafka NAME READY STATUS RESTARTS AGE kafka-0 2/2 Running 0 10m kafka-1 2/2 Running 0 9m26s kafka-2 2/2 Running 0 7m59s Connect to kafka\nConnect to the kafka test client pod and send some test data to kafka through its service endpoint\n Connect to the pod\nkubectl exec -it kafka-test-client /bin/bash Create a topic\n/usr/bin/kafka-topics --zookeeper zookeeper:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1 Send some test data\n/usr/bin/kafka-run-class org.apache.kafka.tools.ProducerPerformance --topic test-rep-one --num-records 5000 --record-size 100 --throughput -1 --print-metrics --producer-props acks=1 bootstrap.servers=kafka:9092 buffer.memory=67108864 batch.size=8196 ","excerpt":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from …","ref":"/v1.x/docs/usecases/kafka/","title":"Kafka"},{"body":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual machines (VMs) in Kubernetes pods.\nDeploying Kubevirt using StorageOS offers multiple benefits. Kubevirt can spin up VMs as Kubernetes pods, using images on StorageOS persistent volumes. Doing this allows the VM data to persist through restarts and rescheduling. Using StorageOS volume replicas also allows for failure of nodes holding the PersistentVolume without interrupting the VM running off the PersistentVolume. Containerized Data Importer (CDI) can also be used to prepare StorageOS volumes with disk images in an automated fashion. Simply by declaring that a VirtualMachine will use a DataVolume and providing the disk image URL, a StorageOS volume can be dynamically provisioned and automatically prepared with the disk image.\nThis usecase will guide you through installing KubeVirt and CDI on your Kubernetes cluster, and create a VM. By the end of the guide you\u0026rsquo;ll be able to launch a shell inside the KubeVirt VM that\u0026rsquo;s running as a Kubernetes pod.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Please ensure you have met the Kubevirt prerequisites, please see the Kubevirt installation instructions for more information.\nAs part of this installation it is assumed that you are running a Kubernetes cluster on VMs. As such nested virtualization or hardware emulation need to be enabled.\nDeploying KubeVirt on Kubernetes For ease of installation we have enabled hardware emulation. If your VMs support nested virtualization then edit the Kubevirt ConfigMap ./kubevirt-install/10-cm.yaml, removing the line debug.useEmulation: \u0026quot;true\u0026quot;.\n In order to deploy Kubevirt you just need to clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/kubevirt $ kubectl create -f ./kubevirt-install Check that the Kubevirt pods are running.\n$ kubectl get pods -w -n kubevirt NAME READY STATUS RESTARTS AGE virt-api-57546d479b-p26d4 1/1 Running 0 1m virt-api-57546d479b-zs5dw 1/1 Running 0 1m virt-controller-56b5498854-7xsfz 1/1 Running 1 1m virt-controller-56b5498854-bz559 1/1 Running 1 1m virt-handler-6z4kq 1/1 Running 0 1m virt-handler-7szhl 1/1 Running 0 1m virt-handler-jmm6w 1/1 Running 0 1m virt-operator-79c9bdd859-8xq98 1/1 Running 0 1m virt-operator-79c9bdd859-kfjz6 1/1 Running 0 1m Once Kubevirt is running install CDI.\n$ kubectl create -f ./cdi Check that the CDI pods are running correctly.\n$ kubectl get pods -n cdi NAME READY STATUS RESTARTS AGE cdi-apiserver-8668f888df-s6pp4 1/1 Running 0 1m cdi-deployment-5cf794896b-whh4j 1/1 Running 0 1m cdi-operator-5887f96c-dz2hg 1/1 Running 0 1m cdi-uploadproxy-97fbbfcbf-6f9xs 1/1 Running 0 1m Now that CDI and Kubevirt are running, VMs can be created. In this example VMs running Cirros, a small and lightweight OS, will be created. The vm-cirros.yaml manifest creates a VirtualMachine that uses a DataVolume. This means that CDI will create a StorageOS backed PVC and download the image that the VirtualMachineInstance (VMI) will boot from onto the PVC.\n$ kubectl create -f ./vm-cirros.yaml Check that the VMI is running. Note that the VMI will only be created after CDI has downloaded the Cirros disk image onto a StorageOS persistent volume so depending on your connection speed this may take some time.\n$ kubectl get vmi NAME AGE PHASE IP NODENAME cirros 1m Running 10.244.2.12 ip-10-1-10-154.storageos.net $ kubectl get pods NAME READY STATUS RESTARTS AGE virt-launcher-cirros-drqhr 1/1 Running 0 1m Connect to the VM console.\nThis example uses the virtctl kubectl plugin in order to connect to the VMs console. The escape sequence ^] is ctrl + ]\n$ kubectl virt console cirros Successfully connected to cirros console. The escape sequence is ^] login as \u0026#39;cirros\u0026#39; user. default password: \u0026#39;gocubsgo\u0026#39;. use \u0026#39;sudo\u0026#39; for root. cirros login: cirros Password: $ Cloning Volumes CDI allows for images to be cloned using a DataVolume manifest. Verify that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n N.B. Ensure that the VMI is stopped before continuing!\n Verify that the VMI is stopped before continuing, and that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE cirros Bound pvc-f4833060-5a77-420c-927e-6bc518d9df3c 12Gi RWO fast 1m Once the PVC\u0026rsquo;s existence is confirmed then create a new DataVolume that uses the cirros PVC as its source.\n$ kubectl create -f ./cloned.yaml Watch as the CDI pods are created.\n$ kubectl get pods -w You\u0026rsquo;ll see that a cdi-upload-cloned-datavolume pod is created and then a cdi-clone-source pod is created. The cdi-source pod mounts the original cirros volume and sends the contents of the volume to the cdi-upload pod. The cdi-upload pod creates and mounts a new PVC and writes the contents of the original volume to it.\n ","excerpt":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual …","ref":"/v1.x/docs/usecases/kubevirt/","title":"Kubevirt"},{"body":"Feature labels are a powerful and flexible way to control storage features, especially when combined with rules.\nLabels can be applied to various StorageOS artefacts. Applying specific feature labels triggers compression, replication and other storage features. No feature labels are present by default.\nStorageOS Node labels Nodes do not have any feature labels present by default. When StorageOS is run within Kubernetes with the Cluster Operator, any node labels set on Kubernetes nodes are available within StorageOS. Node labels may also be set with the CLI or UI.\n Feature Label Values Description Deployment type storageos.com/deployment strings [computeonly,mixed] Specifies whether a node should be computeonly where it only acts as a client and does not host volume data locally, or mixed (the default), where the node can operate in both client and server modes. Region iaas/region string Set automatically in AWS, Azure and GCE. e.g. eu-west-1. Not currently used by StorageOS but available for use in rules. Failure domain iaas/failure-domain string Used to spread master and replicas across different failure domains. Set automatically in AWS, Azure and GCE, e.g. eu-west-1b Update domain iaas/update-domain string Set by some cloud providers to perform sequential updates/reboots. Not currently used by StorageOS but available for use in rules. Size iaas/size string The node hardware configuration, as set by the cloud provider, e.g. m5d.xlarge. Not currently used by StorageOS but available for use in rules. To add a label to a node:\nstorageos node update --label-add storageos.com/deployment=computeonly nodename StorageOS Pod Labels Feature Label Values Description Fencing storageos.com/fenced true / false Enables StorageOS fencing of pods on unavailable nodes. For more information about fencing prerequisites please see the Fencing operations page. To add the fencing label to a pod use kubectl:\nkubectl label pod \u0026lt;POD_NAME\u0026gt; key=value kubectl label pod mydb-pod storageos.com/fenced=true StorageOS Pool labels Pools do not have any labels present by default.\n Feature Label Values Description Overcommitment storageos.com/overcommit integers [+] Sets the percentage of overcommitment allowed for the pool (see here). To add overcommit labels to a pool:\nstorageos pool update --label-add storageos.com/overcommit=20 default StorageOS Volume labels Volumes do not have any feature labels present by default\n Feature Label Values Description Caching storageos.com/nocache true / false Switches off caching. Compression storageos.com/nocompress true / false Switches off compression of data at rest and in transit. Encryption storageos.com/encryption true / false Enables volume encryption, more details here Failure mode storageos.com/failure.mode strings [soft,hard,alwayson] Soft failure mode works together with the failure tolerance. Hard is a mode where any loss in desired replicas count will mark volume as unavailable. AlwaysOn is a mode where as long as master is alive volume will be writable. Failure tolerance storageos.com/failure.tolerance integers [0, 4] Specifies how many failed replicas to tolerate, defaults to (Replicas - 1) if Replicas \u0026gt; 0, so if there are 2 replicas it will default to 1. Placement storageos.com/hint.master Node hostname or uuid Requests master volume placement on the specified node. Will use another node if request can\u0026rsquo;t be satisfied. QoS storageos.com/throttle true / false Deprioritizes traffic by reducing the rate of disk I/O, when true. Replication storageos.com/replicas integers [0, 5] Replicates entire volume across nodes. Typically 1 replica is sufficient (2 copies of the data); more than 2 replicas is not recommended. To create a volume with a feature label:\nstorageos volume create --label storageos.com/throttle=true --label storageos.com/replicas=1 volumename When using the Kubernetes CSI driver (available from Kubernetes 1.10), volume labels can also be added to the parameters section of the StorageClass. This means that all volumes created with the specific StorageClass will have StorageOS volume labels applied to them.\nFor example the StorageClass below will create ext4 formatted volumes with a single StorageOS replica, in the default pool.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;provisioner:storageos","excerpt":"Feature labels are a powerful and flexible way to control storage features, especially when combined …","ref":"/v1.x/docs/reference/labels/","title":"StorageOS Feature labels"},{"body":"$ storageos licence Usage:\tstorageos licence COMMAND Manage the licence Options: --help Print usage Commands: apply Apply a new licence, Either provide the filename of the licence file or write to stdin. E.g. \u0026#34;storageos licence apply --filename=licence\u0026#34; E.g. \u0026#34;cat licence | storageos licence apply --stdin\u0026#34; inspect Display detailed information on the licence rm Remove the current licence Run \u0026#39;storageos licence COMMAND --help\u0026#39; for more information on a command. storageos licence apply To apply a new licence from a licence file to override the existing one, run:\n$ storageos licence apply --filename=licence To apply a new licence from clipboard to override the existing one, run:\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos licence apply --stdin storageos licence inspect To display detailed information on the current licence, run:\n$ storageos licence inspect [ { \u0026#34;clusterID\u0026#34;: \u0026#34;ea0f97a1-9fa5-4977-9919-e9fb4bbd8708\u0026#34;, \u0026#34;storage\u0026#34;: 100, \u0026#34;validUntil\u0026#34;: \u0026#34;9999-01-01T00:00:00Z\u0026#34;, \u0026#34;licenceType\u0026#34;: \u0026#34;basic\u0026#34;, \u0026#34;features\u0026#34;: { \u0026#34;HA\u0026#34;: true }, \u0026#34;unregistered\u0026#34;: true } ] storageos licence rm To delete the previously applied licence from the system, run:\n$ storageos licence rm ","excerpt":"$ storageos licence Usage:\tstorageos licence COMMAND Manage the licence Options: --help Print usage …","ref":"/v1.x/docs/reference/cli/licence/","title":"Licence"},{"body":"A newly installed StorageOS cluster includes an unregistered Basic licence, which caps usable storage space at 100GB. To utilise more storage space, we offer either a Developer (free with registration - 500GB) licence or an Enterprise (unlimited capacity - see below) licence. This document explains how to upgrade your licence using either the GUI or CLI.\nObtaining a Developer licence via the GUI You can obtain and apply a Developer licence in the StorageOS web GUI automatically by creating or logging in with a StorageOS account on the StorageOS portal via the licence page of the StorageOS web GUI: http://ADVERTISE_IP:5705/#/licence.\nWait a few seconds for the licence generation process to complete, at which point your licence will be visible. To inspect your licence, click on the \u0026ldquo;DETAILS\u0026rdquo; button as follows:\nApplying a previously obtained licence via the GUI Occasionally we will issue licences directly, e.g. by email or some other off-line method. To apply such keys, via the web GUI, visit http://ADVERTISE_IP:5705/#/licence and click on the tab \u0026ldquo;ENTER KEY\u0026rdquo;, then paste the licence key and click on \u0026ldquo;UPLOAD KEY TO CLUSTER\u0026rdquo;. Note that you could also view your cluster ID on the same page.\nObtaining a Developer licence via the CLI Before getting a licence, you need to know the ID of your StorageOS cluster.\nThis CLI command can print the cluster ID:\nstorageos licence inspect | jq -r .[].clusterID To obtain a licence for your StorageOS cluster, create a new StorageOS account or log into StorageOS Portal, go to the Licences page and follow the instructions on the page to get the licence key for your cluster. Make sure that you input the correct cluster ID before generating the licence key.\nThen copy the licence key to clipboard and apply the licence by the CLI command:\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos licence apply --stdin Read the licence CLI command reference for further information.\nObtaining an Enterprise licence Please contact sales@storageos.com to discuss pricing for our Enterprise licence.\n","excerpt":"A newly installed StorageOS cluster includes an unregistered Basic licence, which caps usable …","ref":"/v1.x/docs/operations/licensing/","title":"Licensing"},{"body":"$ storageos login --help Usage:\tstorageos login [HOST] Store login credentials for a given storageos host Options: --help Print usage -p, --password string The password to use for this host (will override value of the global option --password) -u, --username string The username to use for this host (will override value of the global option --username) The storageos CLI provides a simple credentials helper to aid in cluster management. In addition to the use of environment variables STORAGEOS_USERNAME and STORAGEOS_PASSWORD the credentials stored by this command are available for use to authenticate with a cluster. The CLI will automatically use the stored credentials when contacting a known host (if not overridden by -u or -p).\nTo store credentials for a host use the login command:\n$ storageos login 10.1.5.249 Username: storageos Password: Credentials verified These credentials are then stored in the file $HOMEDIR/.storageos/config.json\n{ \u0026#34;knownHosts\u0026#34;: { \u0026#34;10.1.5.249:5705\u0026#34;: { \u0026#34;username\u0026#34;: \u0026#34;storageos\u0026#34;, \u0026#34;password\u0026#34;: \u0026#34;c3RvcmFnZW9z\u0026#34; } } } On Windows and Linux these credentials are stored in plain-text equivalent base-64 so users should take appropriate measures to protect the contents of this file. On the Mac platform, osx-keychain integration is provided, enabling secure credential storage.\n{ \u0026#34;knownHosts\u0026#34;: { \u0026#34;10.1.5.249:5705\u0026#34;: { \u0026#34;username\u0026#34;: \u0026#34;storageos\u0026#34;, \u0026#34;useKeychain\u0026#34;: true } } } Copying credentials between machines On Windows and Linux, migrating credentials to another machine is simple. Just copy the file $HOMEDIR/.storageos/config.json to the same path on the new machine.\nFor Mac machines, the contents stored in the keychain must also be copied. Information on how to do this can be found on Apple\u0026rsquo;s site. For user convenience, all credentials stored in the keychain by the storageos CLI will use the service name storageos_cli.\nOnce credentials for a cluster are no-longer needed, use the logout command to forget the credentials.\n$ storageos logout 10.1.5.249 ","excerpt":"$ storageos login --help Usage:\tstorageos login [HOST] Store login credentials for a given storageos …","ref":"/v1.x/docs/reference/cli/login/","title":"Login"},{"body":"Usage:\tstorageos logs COMMAND View and manage node logs Options: --clear-filter Clears the filter --filter string Set the logging filter -f, --follow Tail the logs for the given node, or all nodes if not specified --format string Output format (raw or table) or a Go template (default \u0026#34;raw\u0026#34;) --help Print usage -l, --log-level string Set the logging level (\u0026#34;debug\u0026#34;|\u0026#34;info\u0026#34;|\u0026#34;warn\u0026#34;|\u0026#34;error\u0026#34;|\u0026#34;fatal\u0026#34;) -q, --quiet Only display volume names -t, --timeout int Timeout in seconds. (default 5) Commands: view Show logging configuration Run \u0026#39;storageos logs COMMAND --help\u0026#39; for more information on a command. The logs command is intended to assist with troubleshooting a running cluster.\n--log-level controls the level of detail shown in the logs with info the default. The available options are debug, info, warn, error and fatal. During normal operation info level is recommended.\nFilters fine-tune the amount of detail shown. They allow you to set the log level to debug level, then set specific categories at a higher level (e.g. info) so there is less noise while troubleshooting an issue.\nFor example, the filter dp=info,cp=info,etcd=debug would set all dataplane and controlplane categories to info level, then only enable debug on the etcd category. Filters are evaluated from left to right. The log level must also be set to debug to show the etcd category at debug level.\nCategories are tags on log messages that relate to the component that generated them. A typical log message looks like:\ntime=\u0026quot;2018-01-11T12:42:58Z\u0026quot; level=info msg=\u0026quot;lost leadership election, waiting\u0026quot; action=election category=leader error=\u0026quot;already exists\u0026quot; module=ha storageos logs view Displays the current log levels and filters for the whole cluster.\nstorageos logs view NODE LEVEL FILTER storageos-1 debug cp=info,dp=info,leader=debug storageos-2 info storageos-3 debug cp=info,dp=info,ha=debug Set log verbosity To set the log level on all nodes in the cluster:\nstorageos logs --log-level debug OK To set the log level on specific nodes, append a list of node names:\nstorageos logs --log-level debug storageos-1 storageos-2 OK Set filter To set a filter on a single node:\nstorageos logs --filter cp=info,dp=info,leader=debug storageos-1 OK If no node names are given, the filter will be applied to all nodes.\nClear filter To remove the filter, use the --clear-filter flag.\nTo clear on all nodes:\nstorageos logs --clear-filter OK You can clear the filter on specific nodes by appending one or more node names:\nstorageos logs --clear-filter storageos-1 OK Viewing logs --follow flag allows you to view the cluster logs as they are generated.\nstorageos logs --follow storageos-1 storageos logs --follow ","excerpt":"Usage:\tstorageos logs COMMAND View and manage node logs Options: --clear-filter Clears the filter …","ref":"/v1.x/docs/reference/cli/logs/","title":"Logs"},{"body":"Draining a node Draining a node reschedules volumes to other nodes and marks the node as unschedulable.\n If the volume does not have any replicas, a new replica will be created on a different node and promoted to master. The previous master will be removed.\n If the volume has replicas, one will be promoted to master. The previous master will become a replica and be relocated to a different node.\n If there are not enough available nodes, StorageOS will keep trying to evict all volumes while the node is in the drained state. Once a new node is added to the cluster, the volume will be moved automatically.\n Performing a node drain will not remove the StorageOS mounts living in that node. Any volume mounted in that specific node will be evicted but still hold the StorageOS mount making the data transparently available to the client, with zero downtime.\nTo drain a node you can use the GUI (see the image below) or the StorageOS CLI\n$ storageos node drain node01 node01 Cordoning a node Cordoning a node marks the node as unschedulable without rescheduling any volumes running on the node. New volumes are unable to be scheduled nor can replicas be promoted on cordoned nodes.\nTo cordon a node you can use the StorageOS CLI\n$ storageos node cordon node01 node01 Or using the GUI go to Nodes and use the cordon toggle. Cluster Maintenance Mode Entering Cluster maintenance mode, or freezing a StorageOS cluster means that no volumes will be moved, or replicas promoted while the cluster is frozen.\nIn order to enter cluster maintenance mode you can either use the GUI or the StorageOS API.\nUsing the GUI go to Cluster and toggle Cluster Maintenance Mode on: To enter Cluster maintenance mode using the API you can post to the following endpoint:\n$ curl -u storageos:storageos -X POST 10.1.10.165:5705/v1/cluster/maintenance {\u0026#34;enabled\u0026#34;:true,\u0026#34;updatedBy\u0026#34;:\u0026#34;storageos\u0026#34;,\u0026#34;updatedAt\u0026#34;:\u0026#34;2018-11-13T15:57:34.605480403Z\u0026#34;}% To leave Cluster maintenance mode using the API you can delete from the following endpoint:\n$ curl -u storageos:storageos -X DELETE 10.1.10.165:5705/v1/cluster/maintenance {\u0026#34;enabled\u0026#34;:false,\u0026#34;updatedBy\u0026#34;:\u0026#34;storageos\u0026#34;,\u0026#34;updatedAt\u0026#34;:\u0026#34;2018-11-13T15:59:09.115797194Z\u0026#34;}% Updates Please see Updates operations for more information on how to apply StorageOS updates.\n","excerpt":"Draining a node Draining a node reschedules volumes to other nodes and marks the node as …","ref":"/v1.x/docs/operations/maintenance/","title":"Maintenance"},{"body":"StorageOS believes in exposing many metrics about the functioning and performance of StorageOS processes to help users instrument applications consuming StorageOS volumes and StorageOS itself. To this end StorageOS exposes metrics via a Prometheus endpoint on each StorageOS pod. See our Prometheus Endpoint reference page for specific details about what metrics are exposed.\nStorageOS metrics are exposed in Prometheus text format, so collectors such as Prometheus, Telegraf or Sensu can be used. Prometheus text format exposes data as time series where each time series can be one of four Prometheus metric types.\n Metric Name Description Example Metric Counter Cumulative metric that only increases. Can be reset to zero on a restart storageos_volume_backend_read_bytes_total Gauge Metric that can increase or decrease storageos_volume_size_bytes Histogram Cumulative metric that includes information about the distribution of samples storageos_local_leader_known_nodes_sync_seconds Summary Similar to a histogram but calculates quantiles over certain time windows go_gc_duration_second Each time series is identified by a metric name and key-value labels. The key-value labels allow multiple dimensions for a metric to be exposed. For example the storageos_volume_backend_read_bytes_total metric is given for each volume.\nUsing Prometheus to scrape metrics endpoints in Kubernetes is quite elegant as Prometheus can be configured to scrape metrics from Kubernetes Services. This is important because StorageOS metrics are intended to be scraped from every StorageOS pod in the cluster and then aggregated.\nFor an example of how to visualize StorageOS metrics please see our Monitoring StorageOS page for a link to a sample Grafana dashboard.\n","excerpt":"StorageOS believes in exposing many metrics about the functioning and performance of StorageOS …","ref":"/v1.x/docs/concepts/metrics/","title":"Metrics"},{"body":"Ingesting StorageOS Metrics StorageOS metrics are exposed on each cluster node at http://ADVERTISE_IP:5705/metrics. For a full list of metrics that the endpoint provides please see Prometheus Endpoint. Metrics are exported in Prometheus text format, so collectors such as Prometheus, Telegraf or Sensu can be used. The examples on this page will reference Prometheus semantics.\nFor an example Prometheus and Grafana setup monitoring StorageOS please see the example here.\nAnalysing Metrics There are many metrics exposed by the Prometheus endpoint, but without a good understanding of what each metric is measuring, they may be difficult to interpret. To aid the visualisation of metrics a Grafana dashboard has been made available here.\nStorageOS Volume Metrics Measuring IOPS One of the most popular ways to measure the efficacy of a device is to measure the number of Input/Output Operations per Seconds (IOPS) the device can achieve. storageos_volume_frontend_write_total and storageos_volume_frontend_read_total can be used to calculate the IOPS rate using builtin Prometheus functions.\nThe metrics themselves are counters that report the total read/write operations for a volume from the application perspective. As a counter can only increase over time, the prometheus rate() function needs to be applied to get a measure of operations over time.\nrate(storageos_volume_frontend_write_total[2m]) The Prometheus rate function calculates the per-second average rate of increase for a counter, over the 2 minute time period given. So, the function above gives the per-second average of writes over two minutes. Therefore, if the rate of both read and write totals is taken they can be summed to give IOPS.\nMeasuring Bandwidth While IOPS is a measure of operations per second, bandwidth provides a measure of throughput, usually in MB/s. storageos_volume_frontend_write_bytes_total and storageos_volume_frontend_read_bytes_total are exposed as a way to calculate bandwidth from the application\u0026rsquo;s perspective.\nThese metrics are counters that report the total bytes read from/written to a volume. As with IOPS, a rate can be calculated to give the average number of bytes per second.\nrate(storageos_volume_frontend_write_bytes_total[2m]) As with IOPS, the function above gives the per-second average increase in bytes written to a volume, therefore if the rate of read and write byte totals is summed you have the total volume bandwidth.\nFrontend vs Backend Metrics The StorageOS Prometheus endpoint exposes both frontend and backend volume metrics. The frontend metrics relate to I/O operations against a StorageOS volume\u0026rsquo;s filesystem. These operations are those executed by applications consuming StorageOS volumes. Backend metrics relate to I/O operations that the StorageOS container runs against devices that store the blob files. They are affected by StorageOS features such as compression and encryption which the application is unaware of.\nStorageOS Node Metrics The metrics endpoint exposes a standard set of metrics for every process that the StorageOS container starts, including the metrics below.\nUptime The StorageOS control plane is the first process that starts when a StorageOS pod is created. The storageos_control_process_start_time_seconds is a gauge that provides the start time of the control plane process since the Unix epoch.\ntime() - storageos_control_process_start_time_seconds{alias=~\u0026#34;$node\u0026#34;} By subtracting the control plane start time from the current time since the Unix epoch, the total uptime of the process can be derived.\nCPU Usage The StorageOS container will spawn a number of different processes. To calculate the total CPU footprint of the StorageOS container, these processes need to be summed together. *_cpu_seconds metrics are counters that reflect the total seconds of CPU time each process has used.\n(rate(storageos_control_process_cpu_seconds_total[3m]) + rate(storastorageos_dataplane_process_cpu_seconds_total[3m]) + rate(storastorageos_stats_process_cpu_seconds_total[3m])) * 100 To calculate the average number of seconds of CPU time used per second, a rate must be taken. The rate expresses the fraction of 1 second of CPU time that was used by the StorageOS process in one second. Therefore to express this as a percentage, multiply by 100.\nMemory Usage *_resident_memory_bytes metrics are gauges that show the current resident memory of a StorageOS process. Although metrics about virtual memory usage are also exposed, resident memory gives an overview of memory allocated to each process that is actively being used.\nstorageos_control_process_resident_memory_bytes storageos_director_process_resident_memory_bytes storageos_stats_process_resident_memory_bytes As with CPU usage the resident memory of each StorageOS process needs to be summed to calculate the memory footprint of StorageOS processes.\nVolumes per Node StorageOS has two volumes types; masters and replicas. A master volume is the device that a pod mounts and the replicas are hot stand-bys for the master volume.\nsum(storageos_node_volumes_total{alias=~\u0026#34;$node\u0026#34;}) by (alias, volume_type) By summing across the Prometheus alias and volume_type labels the number of master and replica volumes per node can be found. Changes in the relative numbers of master and replicas indicate that volumes have failed over, assuming that no new volumes or replicas have been created.\n","excerpt":"Ingesting StorageOS Metrics StorageOS metrics are exposed on each cluster node at …","ref":"/v1.x/docs/operations/monitoring/","title":"Monitoring StorageOS"},{"body":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on linux.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MS SQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1...spec:serviceAccountName:mssql...volumeMounts:- name:datamountPath:/var/opt/mssql...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MS SQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mssql Confirm MS SQL is up and running.\n$ kubectl get pods -w -l app=mssql NAME READY STATUS RESTARTS AGE mssql-0 1/1 Running 0 1m Connect to the MS SQL client pod and connect to the MS SQL server through the service\n$ kubectl exec -it mssql-0 -- /opt/mssql-tools/bin/sqlcmd -S mssql-0.mssql -U SA -P \u0026#39;Password15\u0026#39; 1\u0026gt; USE master; 2\u0026gt; GO Changed database context to \u0026#39;master\u0026#39;. 1\u0026gt; SELECT name, database_id, create_date FROM sys.databases ; 2\u0026gt; GO name database_id create_date --------------------------- ----------- ----------------------- master 1 2003-04-08 09:13:36.390 tempdb 2 2018-11-02 16:30:37.907 model 3 2003-04-08 09:13:36.390 msdb 4 2018-10-19 01:18:57.300 (4 rows affected) Configuration If you need custom startup options, you can edit the ConfigMap file 15-mssql-configmap.yaml with your desired MS SQL configuration settings.\n","excerpt":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on …","ref":"/v1.x/docs/usecases/mssql/","title":"MS SQL"},{"body":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With mTLS both StorageOS and etcd authenticate each other ensuring that communication only happens between mutually authenticated end points, and that all communication is encrypted.\nStorageOS uses environment variables to specify the location of the necessary certificates and keys. This allows Kubernetes secrets to be mounted as volumes, directly into pods.\nSetting up mTLS with Etcd Below is an example StorageOSCluster resource that can be used to setup StorageOS with etcd using mTLS.\napiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:storageos-clusternamespace:\u0026#34;default\u0026#34;spec:images:nodeContainer:\u0026#34;storageos/node:{{ site.latest_node_version }}\u0026#34;secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;default\u0026#34;namespace:\u0026#34;storageos\u0026#34;# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;etcd\u0026#34;# Namespace of the client certificates secretkvBackend:address:\u0026#34;https://storageos-etcd-cluster-client.etcd.svc:2379\u0026#34;# Etcd client service address.backend:\u0026#34;etcd\u0026#34;# Backend typetlsEtcdSecretRefName and tlsEtcdSecretRefNamespace are used to pass a reference to a secret that should contain the following key:values:\n etcd-client-ca.crt - containing the etcd Certificate Authority certificate etcd-client.crt - containing the etcd Client certificate etcd-client.key - cotaining the etcd Client key The StorageOS operator uses the etcd secret that contains the client certificates, to build a secret in the StorageOS installation namespace. This secret contains the certificate filenames and certificate file contents. The StorageOS daemonset that is created by the operator mounts the secret as a volume so that the certificate files are avaliable inside the pod. Environment variables containing the file paths are passed to the StorageOS process in order to use the files from the mounted path.\nA worked example of settings up StorageOS with external etcd using mTLS is avaliable here. For ease of use the example uses the CoreOS etcd operator and the CoreOS guide for Cluster TLS is followed.\n","excerpt":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With …","ref":"/v1.x/docs/operations/external-etcd/etcd-tls/","title":"Encrypting communication with Etcd"},{"body":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular web-based applications including WordPress.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MySQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiversion:apps/v1kind:statefulsetmetadata:name:mysqlspec:selector:matchlabels:app:mysqlenv:prodservicename:mysqlreplicas:1...spec:serviceaccountname:mysql...volumemounts:- name:datamountpath:/var/lib/mysqlsubpath:mysql- name:confmountpath:/etc/mysql/mysql.conf.d...volumeclaimtemplates:- metadata:name:datalabels:env:prodspec:accessmodes:[\u0026#34;readwriteonce\u0026#34;]storageclassname:\u0026#34;fast\u0026#34;# storageos storageclass resources:requests:storage:5giThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MySQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mysql Confirm MySQL is up and running.\n$ kubectl get pods -w -l app=mysql NAME READY STATUS RESTARTS AGE mysql-0 1/1 Running 0 1m Connect to the MySQL client pod and connect to the MySQL server through the service\n$ kubectl exec client -- mysql -h mysql-0.mysql -e \u0026#34;show databases;\u0026#34; Database information_schema mysql performance_schema Configuration If you need custom startup options, you can edit the ConfigMap file 15-mysqld-configmap.yaml with your desired MySQL configuration settings.\n","excerpt":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular …","ref":"/v1.x/docs/usecases/mysql/","title":"MySQL"},{"body":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a StorageOS cluster to be used by multiple teams across multiple projects.\nIt is not necessary to create StorageOS namespaces manually, as StorageOS maps Kubernetes namespaces on a one-to-one basis when PersistentVolumeClaims using the StorageOS StorageClass are created.\nAccess to Namespaces is controlled through user or group level policies.\n","excerpt":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a …","ref":"/v1.x/docs/concepts/namespaces/","title":"Namespaces"},{"body":"Namespaces help different projects or teams share a StorageOS cluster. No namespaces are created by default, and users can have any number of namespaces.\nNamespaces apply to volumes and rules.\n Note: Docker does not support namespaces, so you should avoid mixing volumes created by docker volume create (which does not allow namespaces) with volumes created by storageos volume create (which requires a namespace).\n Create a namespace To start creating rules and volumes, at least one namespace is required. To create a namespace, run:\n$ storageos namespace create legal --description compliance-volumes legal Add the --display-name flag to set a display-friendly name.\nList all namespaces To view namespaces, run:\n$ storageos namespace ls -q default legal performance Remove -q for full details\nInspect namespaces Check if a namespace has labels applied.\n$ storageos namespace inspect legal | grep labels \u0026#34;labels\u0026#34;: null, Removing a namespace Removing a namespace will remove all volumes and rules that belong to that namespace. An API call or CLI command to remove a namespace will fail if there are mounted volumes to prevent data loss.\nTo remove a namespace:\n$ storageos namespace rm legal legal To force remove, even if there are mounted volumes:\nstorageos namespace rm --force my-namespace ","excerpt":"Namespaces help different projects or teams share a StorageOS cluster. No namespaces are created by …","ref":"/v1.x/docs/operations/namespaces/","title":"Namespaces"},{"body":"$ storageos namespace Usage:\tstorageos namespace COMMAND Manage namespaces Aliases: namespace, ns Options: --help Print usage Commands: create Create a namespace inspect Display detailed information on one or more namespaces ls List namespaces rm Remove one or more namespaces update Update a namespace Run \u0026#39;storageos namespace COMMAND --help\u0026#39; for more information on a command. storageos namespace create To create a namespace:\n$ storageos namespace create legal --description compliance-volumes legal Add the --display-name flag to set a display-friendly name.\nstorageos namespace inspect Check if a namespace has labels applied.\n$ storageos namespace inspect legal | grep labels \u0026#34;labels\u0026#34;: null, storageos namespace ls To view namespaces:\n$ storageos namespace ls -q default legal performance Remove -q for full details\nstorageos namespace rm To remove a namespace (add \u0026ndash;force to remove namespaces with mounted volumes):\n$ storageos namespace rm legal legal ","excerpt":"$ storageos namespace Usage:\tstorageos namespace COMMAND Manage namespaces Aliases: namespace, ns …","ref":"/v1.x/docs/reference/cli/namespace/","title":"Namespaces"},{"body":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load balancer or even as a Kubernetes ingress controller.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Nginx on Kubernetes You can find the latest files in the StorageOS use cases repostiory\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet definition\napiVersion:apps/v1kind:StatefulSetmetadata:name:nginxspec:serviceName:nginxspec:serviceAccountName:nginxcontainers:- name:nginximage:nginxports:- containerPort:80volumeMounts:- name:nginx-datamountPath:/usr/share/nginx/htmlsubPath:html- name:nginx-configmountPath:/etc/nginx/conf.dvolumes:- name:nginx-configconfigMap:name:nginxvolumeClaimTemplates:- metadata:name:nginx-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Nginx examples folder and create the objects\ncd storageos-usecases kubectl create -f ./nginx Confirm Nginx is up and running.\n$ kubectl get pods -w -l app=nginx NAME READY STATUS RESTARTS AGE nginx-0 1/1 Running 0 1m Connect to the nginx pod and write a file to /usr/share/nginx/html that Nginx will serve.\n$ kubectl exec nginx-0 -it -- bash root@nginx-0:/# echo Hello world! \u0026gt; /usr/share/nginx/html/greetings.txt Connect to the Busybox pod and connect to the Nginx server through the service and retrieve the directory index from Nginx.\n$ kubectl exec -it busybox -- /bin/sh / # wget -q -O- nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;greetings.txt\u0026#34;\u0026gt;greetings.txt\u0026lt;/a\u0026gt; 27-Feb-2019 12:04 13 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; Retrieve and display the contents of the greetings.txt file\n/ # wget -q -O- nginx/greetings.txt Hello world! Configuration If you need custom startup options, you can edit the ConfigMap file 15-nginx-configmap.yaml with your desired Nginx configuration settings.\n","excerpt":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load …","ref":"/v1.x/docs/usecases/nginx/","title":"Nginx"},{"body":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS container. Under Kubernetes orchestration a StorageOS container runs as part of a Kubernetes pod on all Kubernetes worker node. StorageOS nodes aggregate host storage and present this storage as StorageOS pools.\n","excerpt":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS container. Under …","ref":"/v1.x/docs/concepts/nodes/","title":"Nodes"},{"body":"$ storageos node Usage: storageos node COMMAND Manage nodes Aliases: node, n Options: --help Print usage Commands: connectivity Display detailed connectivity information on one or more nodes cordon Put one or more nodes into an unschedulable state delete Remove an offline node from the cluster. drain Migrate volumes from one or more nodes. health Display detailed information on a given node inspect Display detailed information on one or more nodes ls List nodes uncordon Restore one or more nodes from an unschedulable state undrain Stop drain on one or more nodes. update Update a node Run \u0026#39;storageos node COMMAND --help\u0026#39; for more information on a command. storageos node connectivity Display the latency and status of the connection between different nodes of the cluster and the services attached to the network started by StorageOS. It shows whether any node or service can\u0026rsquo;t be connected.\nThe output can be for all permutations of nodes in the cluster or for one specific node.\n$ storageos node connectivity SOURCE NAME ADDRESS LATENCY STATUS MESSAGE ip-172-20-88-241 ip-172-20-100-61.api 172.20.100.61:5705 1.172108ms OK ip-172-20-88-241 ip-172-20-100-61.directfs 172.20.100.61:5703 1.238443ms OK ip-172-20-88-241 ip-172-20-100-61.nats 172.20.100.61:5708 1.088224ms OK ip-172-20-88-241 ip-172-20-32-38.api 172.20.32.38:5705 1.201392ms OK ip-172-20-88-241 ip-172-20-32-38.directfs 172.20.32.38:5703 1.225196ms OK ip-172-20-88-241 ip-172-20-32-38.nats 172.20.32.38:5708 1.097115ms OK ip-172-20-88-241 ip-172-20-39-233.api 172.20.39.233:5705 1.190318ms OK ip-172-20-88-241 ip-172-20-39-233.directfs 172.20.39.233:5703 1.222903ms OK ip-172-20-88-241 ip-172-20-39-233.nats 172.20.39.233:5708 1.286556ms OK ip-172-20-39-233 ip-172-20-100-61.api 172.20.100.61:5705 1.257497ms OK ip-172-20-39-233 ip-172-20-100-61.directfs 172.20.100.61:5703 1.102858ms OK ip-172-20-39-233 ip-172-20-100-61.nats 172.20.100.61:5708 1.240308ms OK ip-172-20-39-233 ip-172-20-32-38.api 172.20.32.38:5705 1.169309ms OK ip-172-20-39-233 ip-172-20-32-38.directfs 172.20.32.38:5703 1.238169ms OK ip-172-20-39-233 ip-172-20-32-38.nats 172.20.32.38:5708 1.120058ms OK ip-172-20-39-233 ip-172-20-88-241.api 172.20.88.241:5705 1.285212ms OK ip-172-20-39-233 ip-172-20-88-241.directfs 172.20.88.241:5703 1.274576ms OK ip-172-20-39-233 ip-172-20-88-241.nats 172.20.88.241:5708 1.257659ms OK ip-172-20-100-61 ip-172-20-32-38.api 172.20.32.38:5705 1.136496ms OK ip-172-20-100-61 ip-172-20-32-38.directfs 172.20.32.38:5703 1.200905ms OK ip-172-20-100-61 ip-172-20-32-38.nats 172.20.32.38:5708 1.227956ms OK ip-172-20-100-61 ip-172-20-39-233.api 172.20.39.233:5705 1.075072ms OK ip-172-20-100-61 ip-172-20-39-233.directfs 172.20.39.233:5703 1.279988ms OK ip-172-20-100-61 ip-172-20-39-233.nats 172.20.39.233:5708 1.239564ms OK ip-172-20-100-61 ip-172-20-88-241.api 172.20.88.241:5705 1.240107ms OK ip-172-20-100-61 ip-172-20-88-241.directfs 172.20.88.241:5703 1.219858ms OK ip-172-20-100-61 ip-172-20-88-241.nats 172.20.88.241:5708 1.309962ms OK ip-172-20-32-38 ip-172-20-100-61.api 172.20.100.61:5705 1.08306ms OK ip-172-20-32-38 ip-172-20-100-61.directfs 172.20.100.61:5703 1.186921ms OK ip-172-20-32-38 ip-172-20-100-61.nats 172.20.100.61:5708 1.233717ms OK ip-172-20-32-38 ip-172-20-39-233.api 172.20.39.233:5705 1.272816ms OK ip-172-20-32-38 ip-172-20-39-233.directfs 172.20.39.233:5703 1.215386ms OK ip-172-20-32-38 ip-172-20-39-233.nats 172.20.39.233:5708 1.272102ms OK ip-172-20-32-38 ip-172-20-88-241.api 172.20.88.241:5705 1.276441ms OK ip-172-20-32-38 ip-172-20-88-241.directfs 172.20.88.241:5703 1.248265ms OK ip-172-20-32-38 ip-172-20-88-241.nats 172.20.88.241:5708 1.328679ms OK $ storageos node connectivity ip-172-20-88-241 SOURCE NAME ADDRESS LATENCY STATUS MESSAGE ip-172-20-88-241 ip-172-20-100-61.api 172.20.100.61:5705 2.804153ms OK ip-172-20-88-241 ip-172-20-100-61.directfs 172.20.100.61:5703 2.77249ms OK ip-172-20-88-241 ip-172-20-100-61.nats 172.20.100.61:5708 2.791746ms OK ip-172-20-88-241 ip-172-20-32-38.api 172.20.32.38:5705 2.755615ms OK ip-172-20-88-241 ip-172-20-32-38.directfs 172.20.32.38:5703 2.815147ms OK ip-172-20-88-241 ip-172-20-32-38.nats 172.20.32.38:5708 2.526776ms OK ip-172-20-88-241 ip-172-20-39-233.api 172.20.39.233:5705 2.817432ms OK ip-172-20-88-241 ip-172-20-39-233.directfs 172.20.39.233:5703 2.839914ms OK ip-172-20-88-241 ip-172-20-39-233.nats 172.20.39.233:5708 2.894249ms OK storageos node cordon Puts one or more nodes into an unschedulable state, in preparation for upgrading or decommissioning a node.\n$ storageos node cordon storageos-1 storageos-1 storageos node delete Only applicable for installations with KV_BACKEND=etcd.\n Removes a StorageOS node from the cluster. It will only succeed if the node is Offline. All data in that node will be lost, hence it is recommended to drain the node before decommissioning it.\n$ storageos node delete storageos-1 storageos-1 storageos node drain Evicts all volumes from one or more nodes and puts them into an unschedulable state.\n$ storageos node drain storageos-1 storageos-1 storageos node inspect To view detailed information such as state, port configuration, health, version and capacity in JSON format, inspect the node:\n$ storageos node inspect storageos-1 [ { \u0026#34;id\u0026#34;: \u0026#34;dafdcc3b-7a7a-14e4-77f4-bcc2070543cf\u0026#34;, \u0026#34;hostname\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;address\u0026#34;: \u0026#34;192.168.50.100\u0026#34;, \u0026#34;kvAddr\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;apiPort\u0026#34;: 5705, \u0026#34;natsPort\u0026#34;: 5708, \u0026#34;natsClusterPort\u0026#34;: 5710, \u0026#34;serfPort\u0026#34;: 5711, \u0026#34;dfsPort\u0026#34;: 5703, \u0026#34;kvPeerPort\u0026#34;: 5707, \u0026#34;kvClientPort\u0026#34;: 5706, \u0026#34;labels\u0026#34;: null, \u0026#34;logLevel\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;logFormat\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;logFilter\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;bindAddr\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;deviceDir\u0026#34;: \u0026#34;/var/lib/storageos/volumes\u0026#34;, \u0026#34;join\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;kvBackend\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;debug\u0026#34;: false, \u0026#34;devices\u0026#34;: [ { \u0026#34;ID\u0026#34;: \u0026#34;e303b84f-b562-c1b0-8434-1ee89f5a9f8d\u0026#34;, \u0026#34;labels\u0026#34;: { \u0026#34;default\u0026#34;: \u0026#34;true\u0026#34; }, \u0026#34;status\u0026#34;: \u0026#34;active\u0026#34;, \u0026#34;identifier\u0026#34;: \u0026#34;/var/lib/storageos/data\u0026#34;, \u0026#34;class\u0026#34;: \u0026#34;filesystem\u0026#34;, \u0026#34;capacityStats\u0026#34;: { \u0026#34;totalCapacityBytes\u0026#34;: 40576331776, \u0026#34;availableCapacityBytes\u0026#34;: 38596853760, \u0026#34;provisionedCapacityBytes\u0026#34;: 0 }, \u0026#34;createdAt\u0026#34;: \u0026#34;2018-06-22T09:20:17.499457963Z\u0026#34;, \u0026#34;updatedAt\u0026#34;: \u0026#34;2018-06-22T09:35:17.87048772Z\u0026#34; } ], \u0026#34;hostID\u0026#34;: 53012, \u0026#34;name\u0026#34;: \u0026#34;storageos-1\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;createdAt\u0026#34;: \u0026#34;2018-06-20T09:22:17.491813738Z\u0026#34;, \u0026#34;updatedAt\u0026#34;: \u0026#34;2018-06-22T09:35:17.876400176Z\u0026#34;, \u0026#34;health\u0026#34;: \u0026#34;healthy\u0026#34;, \u0026#34;healthUpdatedAt\u0026#34;: \u0026#34;2018-06-22T09:20:22.868508782Z\u0026#34;, \u0026#34;versionInfo\u0026#34;: { \u0026#34;storageos\u0026#34;: { \u0026#34;name\u0026#34;: \u0026#34;storageos\u0026#34;, \u0026#34;buildDate\u0026#34;: \u0026#34;2018-05-25T190132Z\u0026#34;, \u0026#34;revision\u0026#34;: \u0026#34;f8915fa\u0026#34;, \u0026#34;version\u0026#34;: \u0026#34;1.0.0\u0026#34;, \u0026#34;apiVersion\u0026#34;: \u0026#34;1\u0026#34;, \u0026#34;goVersion\u0026#34;: \u0026#34;go1.9.1\u0026#34;, \u0026#34;os\u0026#34;: \u0026#34;linux\u0026#34;, \u0026#34;arch\u0026#34;: \u0026#34;amd64\u0026#34;, \u0026#34;kernelVersion\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;experimental\u0026#34;: false } }, \u0026#34;version\u0026#34;: \u0026#34;StorageOS 1.0.0 (f8915fa), built: 2018-05-25T190132Z\u0026#34;, \u0026#34;Revision\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;scheduler\u0026#34;: true, \u0026#34;cordon\u0026#34;: false, \u0026#34;drain\u0026#34;: false, \u0026#34;volumeStats\u0026#34;: { \u0026#34;masterVolumeCount\u0026#34;: 0, \u0026#34;replicaVolumeCount\u0026#34;: 2, \u0026#34;virtualVolumeCount\u0026#34;: 0 }, \u0026#34;capacityStats\u0026#34;: { \u0026#34;totalCapacityBytes\u0026#34;: 40576331776, \u0026#34;availableCapacityBytes\u0026#34;: 38596853760, \u0026#34;provisionedCapacityBytes\u0026#34;: 0 } } ] storageos node ls To view the state of your cluster, run:\n$ storageos node ls NAME ADDRESS HEALTH SCHEDULER VOLUMES TOTAL USED VERSION storageos-1 192.168.50.100 Healthy 18 minutes true M: 0, R: 2 40.58GB 4.88% 1.0.0 storageos-2 192.168.50.101 Healthy 17 minutes false M: 1, R: 0 40.58GB 4.88% 1.0.0 storageos-3 192.168.50.102 Healthy 17 minutes false M: 1, R: 1 40.58GB 4.88% 1.0.0 The output shows a StorageOS cluster with three nodes named storageos-1, storageos-2 and storageos-3.\n SCHEDULER: whether the node contains the scheduler, which is responsible for the placement of volumes, performing health checks and providing high availability to nodes. A cluster will have exactly one scheduler node. VOLUMES: the number of master or replica copies of volumes on this node. storageos node uncordon Restore one or more nodes after cordoning the node for upgrade.\n$ storageos node uncordon storageos-1 storageos-1 storageos node undrain Stops any draining procedure in place and restores the node to its normal functionality.\n$ storageos node undrain storageos-1 storageos-1 ","excerpt":"$ storageos node Usage: storageos node COMMAND Manage nodes Aliases: node, n Options: --help Print …","ref":"/v1.x/docs/reference/cli/node/","title":"Nodes"},{"body":"The following document lists the open source software attributions in the StorageOS Control Plane, Data Plane and CLI.\nCNats The MIT License (MIT) Copyright (c) 2015 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CppUTest Copyright (c) 2007, Michael Feathers, James Grenning and Bas Vodde All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the \u0026lt;organization\u0026gt; nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE EARLIER MENTIONED AUTHORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL \u0026lt;copyright holder\u0026gt; BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Hayai Copyright (c) 2011 - Nick Bruun. This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. If you meet (any of) the author(s), you're encouraged to buy them a beer, a drink or whatever is suited to the situation, given that you like the software. This notice may not be removed or altered from any source distribution. LZ4 LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LibreSSL LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- ==================================================================== Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit. (http://www.openssl.org/)\u0026quot; 4. The names \u0026quot;OpenSSL Toolkit\u0026quot; and \u0026quot;OpenSSL Project\u0026quot; must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact openssl-core@openssl.org. 5. Products derived from this software may not be called \u0026quot;OpenSSL\u0026quot; nor may \u0026quot;OpenSSL\u0026quot; appear in their names without prior written permission of the OpenSSL Project. 6. Redistributions of any form whatsoever must retain the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)\u0026quot; THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==================================================================== This product includes cryptographic software written by Eric Young (eay@cryptsoft.com). This product includes software written by Tim Hudson (tjh@cryptsoft.com). Original SSLeay License ----------------------- Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) All rights reserved. This package is an SSL implementation written by Eric Young (eay@cryptsoft.com). The implementation was written so as to conform with Netscapes SSL. This library is free for commercial and non-commercial use as long as the following conditions are aheared to. The following conditions apply to all code found in this distribution, be it the RC4, RSA, lhash, DES, etc., code; not just the SSL code. The SSL documentation included with this distribution is covered by the same copyright terms except that the holder is Tim Hudson (tjh@cryptsoft.com). Copyright remains Eric Young's, and as such any Copyright notices in the code are not to be removed. If this package is used in a product, Eric Young should be given attribution as the author of the parts of the library used. This can be in the form of a textual message at program startup or in documentation (online or textual) provided with the package. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: \u0026quot;This product includes cryptographic software written by Eric Young (eay@cryptsoft.com)\u0026quot; The word 'cryptographic' can be left out if the rouines from the library being used are not cryptographic related :-). 4. If you include any Windows specific code (or a derivative thereof) from the apps directory (application code) you must include an acknowledgement: \u0026quot;This product includes software written by Tim Hudson (tjh@cryptsoft.com)\u0026quot; THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The licence and distribution terms for any publically available version or derivative of this code cannot be changed. i.e. this code cannot simply be copied and put under another distribution licence [including the GNU Public Licence.] protobuf This license applies to all parts of Protocol Buffers except the following: - Atomicops support for generic gcc, located in src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Code generated by the Protocol Buffer compiler is owned by the owner of the input file used when generating it. This code is not standalone and requires a support library to be linked with it. This support library is itself covered by the above license. rocksdb BSD License For rocksdb software Copyright (c) 2011-present, Facebook, Inc. All rights reserved. --------------------------------------------------------------------- Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/armon/go-metrics The MIT License (MIT) Copyright (c) 2013 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/armon/go-radix The MIT License (MIT) Copyright (c) 2014 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/bgentry/speakeasy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \u0026quot;[]\u0026quot; replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \u0026quot;printed page\u0026quot; as the copyright notice for easier identification within third-party archives. Copyright [2013] [the CloudFoundry Authors] Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/dgrijalva/jwt-go Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgrijalva/jwt-go/request Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgryski/go-metro MetroHash This package is a mechanical translation of the reference C++ code for MetroHash, available at https://github.com/jandrewrogers/MetroHash I claim no additional copyright over the original implementation. The MIT License (MIT) Copyright (c) 2015 J. Andrew Rogers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/docker/docker/pkg/mount Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv/store Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/golang/protobuf/proto Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/context Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/handlers Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/mux Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/websocket Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/consul/api Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/errwrap Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-cleanhttp Mozilla Public License, version 2.0 1. Definitions 1.1. \u0026quot;Contributor\u0026quot; means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. \u0026quot;Contributor Version\u0026quot; means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. \u0026quot;Contribution\u0026quot; means Covered Software of a particular Contributor. 1.4. \u0026quot;Covered Software\u0026quot; means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. \u0026quot;Incompatible With Secondary Licenses\u0026quot; means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. \u0026quot;Executable Form\u0026quot; means any form of the work other than Source Code Form. 1.7. \u0026quot;Larger Work\u0026quot; means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. \u0026quot;License\u0026quot; means this document. 1.9. \u0026quot;Licensable\u0026quot; means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. \u0026quot;Modifications\u0026quot; means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. \u0026quot;Patent Claims\u0026quot; of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. \u0026quot;Secondary License\u0026quot; means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. \u0026quot;Source Code Form\u0026quot; means the form of the work preferred for making modifications. 1.14. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) means an individual or a legal entity exercising rights under this License. For legal entities, \u0026quot;You\u0026quot; includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, \u0026quot;control\u0026quot; means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an \u0026quot;as is\u0026quot; basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - \u0026quot;Incompatible With Secondary Licenses\u0026quot; Notice This Source Code Form is \u0026quot;Incompatible With Secondary Licenses\u0026quot;, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-msgpack/codec Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/go-multierror Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/memberlist Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/coordinate Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/serf Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/kelseyhightower/envconfig Copyright (c) 2013 Kelsey Hightower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/mattn/go-isatty Copyright (c) Yasuhiro MATSUMOTO \u0026lt;mattn.jp@gmail.com\u0026gt; MIT License (Expat) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/miekg/dns Extensions of the original work are copyright (c) 2011 Miek Gieben As this is fork of the official Go code the same license applies: Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/mitchellh/cli Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/conf The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/logger The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server/pse The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/util The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats/encoders/builtin The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nuid The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/pborman/uuid Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/Sirupsen/logrus The MIT License (MIT) Copyright (c) 2014 Simon Eskildsen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/thejerf/suture Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/urfave/negroni The MIT License (MIT) Copyright (c) 2014 Jeremy Saenz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang.org/x/crypto/bcrypt Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/crypto/blowfish Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/net/context Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/unix Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/registry Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/svc/eventlog Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/time/rate Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. gopkg.in/yaml.v2 Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ","excerpt":"The following document lists the open source software attributions in the StorageOS Control Plane, …","ref":"/v1.x/docs/reference/open_source_attribution/","title":"StorageOS Open Source Software Attribution Notice"},{"body":"OpenShift and StorageOS communicate with each other to perform actions such as creation, deletion and mounting of volumes. The standard communication procedure in OpenShift versions 3.x uses REST API calls. However, StorageOS also implements communication using CSI. When using CSI, OpenShift and StorageOS communicate via a Linux socket. This socket is created and handled by the Kubelet on the host.\nOpenShift v4 uses CSI by default.\nCSI (Container Storage Interface) Note CSI is the standard that enables storage drivers to release on their own schedule. This allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI is in \u0026ldquo;Technology Preview and not for production workloads\u0026rdquo; in OpenShift 3.11. StorageOS recommends the use of CSI for OpenShift v4, as CSI is generally available in Openshift v4\nStorageOS PersistentVolumeClaims The user can provide standard PVC definitions and StorageOS will dynamically provision them. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nInstallation StorageOS supports OpenShift 3.9, 3.10, 3.11, 4.1, 4.2 and 4.3.\nTo install StorageOS on OpenShift, please follow our installation instructions page.\n","excerpt":"OpenShift and StorageOS communicate with each other to perform actions such as creation, deletion …","ref":"/v1.x/docs/platforms/openshift/","title":"OpenShift"},{"body":"StorageOS has the capacity to influence Kubernetes Pod placement decisions to ensure that Pods are scheduled on the same nodes as their data. This functionality is known as Pod Locality.\nStorageOS grants access to data by presenting, on local or remote nodes, the devices used in a Pod\u0026rsquo;s VolumeMounts. However, it is often the case that it is required or preferred to place the Pod on the node where the StorageOS Primary Volume is located, because IO operations are fastest as a result of minimized network traffic and associated latency. Read operations are served locally and writes require fewer round trips to the replicas of the volume.\nStorageOS automatically enables the use of a custom scheduler for any Pod using StorageOS Volumes. Checkout the Admission Controller reference for more information.\nLocality modes There are two modes available to set pod locality for StorageOS Volumes.\nPreferred The Pod SHOULD be placed alongside its data, if possible. Otherwise, it will be placed alongside volume replicas. If neither scenario is possible, the Pod will start on another node and StorageOS will grant access to the data over the network.\nPreferred mode is the default behaviour when using the StorageOS scheduler.\nStrict The Pod MUST be placed alongside its data, i.e. on a node with the master volume or a replica. If that is not possible, the Pod will remain in pending state until the premise can be fulfilled.\nThe aim of strict mode is to provide the user with the capability to guarantee best performance for applications. Some applications are required to give a certain level of performance, and for such applications strict co-location of application and data is essential.\nFor instance, when running Kafka Pods under heavy load, it may be better to avoid scheduling a Pod using a remote volume rather than have clients direct traffic at a cluster member which exhibits degraded performance.\nTo see examples on how to set a mode for your Pods, check out the examples reference page.\nStorageos Kubernetes Scheduler StorageOS achieves Pod locality by implementing a Kubernetes scheduler extender. The Kubernetes standard scheduler interacts with the StorageOS scheduler when placement decisions need to be made.\nThe Kubernetes standard scheduler selects a set of nodes for a placement decision based on nodeSelectors, affinity rules, etc. This list of nodes is sent to the StorageOS scheduler which sends back the target node where the Pod shall be placed.\nThe StorageOS scheduler logic is provided by a Pod in the Namespace where StorageOS Pods are running.\nScheduling process When a Pod needs to be scheduled, the scheduler collects information about all available nodes and the requirements of the Pod. The collected data is then passed through the Filter phase, during which the scheduler predicates are applied to the node data to decide if the given nodes are compatible with the Pod requirements. The result of the filter consists of a list of nodes that are compatible for the given Pod and a list of nodes that aren\u0026rsquo;t compatible.\nThe list of compatible nodes is then passed to the Prioritize phase, in which the nodes are scored based on attributes such as the state. The result of the Prioritize phase is a list of nodes with their respective scores. The more favorable nodes get higher scores than less favorable nodes. The list is then used by the scheduler to decide the final node to schedule the Pod on.\nOnce a node has been selected, the third phase, Bind, handles the binding of the Pod to the Kubernetes apiserver. Once bound, the kubelet on the node provisions the Pod.\nThe StorageOS scheduler implement Filter and Prioritization phases and leaves binding to the default Kubernetes scheduler.\nAvailable +------------------+ +------------------+ NodeList \u0026amp; Pod | | Filtered NodeList | | Scored Information | | \u0026amp; Pod Information | | NodeList +--------------------\u0026gt;+ Filter +--------------------\u0026gt;+ Prioritize |---------------\u0026gt; | (Predicates) | | (Priorities) | | | | | +------------------+ +------------------+ Scheduling Rules The StorageOS scheduler filters nodes ensuring that the remaining subset fulfill the following prerequisites:\n The node is running StorageOS The node is healthy The node is not StorageOS Cordoned The node is not in a StorageOS Drained state The node is not a StorageOS compute-only node The scoring protocol once the nodes are filtered is as follows:\n Node with master volume - 15 points Node with replica volume - 10 points Node with no master or replica volume - 5 points Node with unhealthy volume or unsynced replica - 1 point Admission Controller StorageOS implements an admission controller that ensures any Pod using StorageOS Volumes are scheduled by the StorageOS Scheduler. This makes the use of the scheduler transparent to the user. Check the reference page to see how to alter this behaviour.\nThe Admission Controller is based on admission webhooks. Therefore, no custom admission plugins need to be enabled at bootstrap of your Kubernetes cluster. Admission webhooks are HTTP callbacks that receive admission requests and do something with them. The StorageOS Cluster Operator serves the admission webhook. So when a Pod is in the process of being created, the StorageOS Cluster Operator mutates the spec.schedulerName ensuring the storageos-scheduler is set.\n","excerpt":"StorageOS has the capacity to influence Kubernetes Pod placement decisions to ensure that Pods are …","ref":"/v1.x/docs/concepts/podlocality/","title":"Pod Placement"},{"body":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a user or group access to a namespace, a policy needs to be created mapping the user or group to the namespace.\n Note: Users always have access to the default namespace\n For more information on how to use policies, see the Policies operations page.\n","excerpt":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a …","ref":"/v1.x/docs/concepts/policies/","title":"Policies"},{"body":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user level so access can be controlled granularly.\nUsers can belong to one or more groups to control their Namespace permissions. Additionally user specific policies can be created to grant a user access to a namespace. Users can belong to any number of groups and have any number of user level policies configured.\n Note: Users are created with access to the default namespace. Policies cannot be applied to the default namespace.\n Create a policy To start creating policies, at least one custom namespace and user are required. To see more information on how to create namespaces see our Namespace guide, for users see our Users CLI reference.\n$ storageos namespace create testing --description quality-assurance --display-name QA $ storageos user create --user jim --groups qa $ storageos policy create --user jim --namespace testing The above commands created a namespace called testing, with a description and display name. A user jim was then created in the qa group and finally jim was given access rights to the testing namespace.\nList all policies To view policies, run:\n$ storageos policy ls ID USER GROUP NAMESPACE 6ad3c709-a16f-aa61-27d3-ec53526046d5 jim testing Inspect policies To inspect policies, run:\n$ storageos policy inspect 6ad3c709-a16f-aa61-27d3-ec53526046d5 [ { \u0026#34;spec\u0026#34;: { \u0026#34;user\u0026#34;: \u0026#34;jim\u0026#34;, \u0026#34;namespace\u0026#34;: \u0026#34;testing\u0026#34; } } ] Removing policies Removing a policy will remove access rights from users or groups that the policy affected.\nTo delete policies, run:\n$ storageos policy rm 6ad3c709-a16f-aa61-27d3-ec53526046d5 ","excerpt":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user …","ref":"/v1.x/docs/operations/policies/","title":"Policies"},{"body":"$ storageos policy Usage:\tstorageos policy COMMAND Manage policies Options: --help Print usage Commands: create Create a new policy, Either provide the set of policy files, set with options or write to stdin. E.g. \u0026#34;storageos policy create --user awesomeUser --namespace testing\u0026#34; E.g. \u0026#34;storageos policy create --policies=\u0026#39;rules1.jsonl,rules2.jsonl\u0026#39;\u0026#34; E.g. \u0026#34;echo \u0026#39;{\u0026#34;spec\u0026#34;: {\u0026#34;group\u0026#34;: \u0026#34;devs\u0026#34;, \u0026#34;namespace\u0026#34;: \u0026#34;develop\u0026#34;}}\u0026#39; | storageos policy create --stdin\u0026#34; inspect Display detailed information on one or more polic(y|ies) ls List policies rm Remove one or more polic(y|ies) Run \u0026#39;storageos policy COMMAND --help\u0026#39; for more information on a command. storageos policy create To create a new policy that allows users in the group \u0026ldquo;developers\u0026rdquo; to access the namespace \u0026ldquo;staging\u0026rdquo;, run:\n$ storageos policy create --group developers --namespace staging storageos policy inspect To display detailed information on one or more policies, run:\n$ storageos policy inspect 9fb096af-0a16-d1d2-5416-b637f0194b3f [ { \u0026#34;spec\u0026#34;: { \u0026#34;group\u0026#34;: \u0026#34;developers\u0026#34;, \u0026#34;namespace\u0026#34;: \u0026#34;staging\u0026#34; } } ] Note: The ID used is the policy ID as displayed in storageos policy ls\n storageos policy ls To list the policies on the system, run:\n$ storageos policy ls ID USER GROUP NAMESPACE 9fb096af-0a16-d1d2-5416-b637f0194b3f developers staging f1e33492-aa9b-3922-7824-383941d37a5d testers staging storageos policy rm To delete a policy from the system, run:\n$ storageos policy rm 9fb096af-0a16-d1d2-5416-b637f0194b3f 9fb096af-0a16-d1d2-5416-b637f0194b3f The policy is now deleted.\n$ storageos policy ls ID USER GROUP NAMESPACE f1e33492-aa9b-3922-7824-383941d37a5d testers staging Note: The ID used is the policy ID as displayed in storageos policy ls\n ","excerpt":"$ storageos policy Usage:\tstorageos policy COMMAND Manage policies Options: --help Print usage …","ref":"/v1.x/docs/reference/cli/policy/","title":"Policies"},{"body":"All volumes in StorageOS are thinly provisioned and support inline compression. Therefore volumes only occupy capacity in the pool based on the data actually written to the StorageOS volume. StorageOS allows pools to be overcommitted. This allows the creation of volumes whose size exceeds the actual storage presented by underlying nodes. It can be useful if you anticipate writing a lot of data to a volume eventually but you do not wish to pay for storage that is not currently being used.\nGiven a 120GB pool comprised of 3 nodes with 40GB disks, then the maximum volume size is 40GB. To ensure deterministic performance and replication of writes, StorageOS volumes must fit on a single node. Therefore if the pool is overcommitted by 20% then the new pool size is 144GB and the new maximum volume size is 48GB.\nWith hetrogenous node capacities it is important to keep in mind that a volume cannot be provisioned on a node without capacity that is equal to, or larger than the volume size.\nFor example given a 250GB pool, which is made up of a 200GB node and a 50GB node then StorageOS will not collocate volumes whose total size is larger than 50GB on the 50GB node. If the pool is overcommited by 10% then volumes up to 55GB could be scheduled on the 50GB node, and volumes up to 220GB on the 200GB node.\nAdding an Overcommit label to a pool To add a overcommit label to a pool you can use the CLI or GUI.\nWith the CLI:\n$ storageos pool update --label-add storageos.com/overcommit=20 default The command above would set overcommit to 20% on the default pool.\n$ storageos pool update --label-rm storageos.com/overcommit default The command above removes the overcommit label from the default pool.\nWith the GUI:\nNavigate to Pools and click on the labels row of the volume you would like to edit. In the popup menu you can add and remove labels. For more information on feature labels please see here.\n","excerpt":"All volumes in StorageOS are thinly provisioned and support inline compression. Therefore volumes …","ref":"/v1.x/docs/operations/overcommitment/","title":"Pool Overcommit"},{"body":"StorageOS aggregates host storage from all nodes where the StorageOS container runs into a storage pool. A pool is a collection of storage based on host attributes such as class of server, storage or location.\nNode storage can be allocated to a specific pool using Node selectors. Pool node selectors look for labels on host nodes and will aggregate storage from nodes whose labels match into the specific pool.\nPools can have feature labels applied to them such as storageos.com/overcommit which allows the pool to have its storage overcommited.\n","excerpt":"StorageOS aggregates host storage from all nodes where the StorageOS container runs into a storage …","ref":"/v1.x/docs/concepts/pools/","title":"Pools"},{"body":"$ storageos pool Usage:\tstorageos pool COMMAND Manage capacity pools Options: --help Print usage Commands: create Create a capacity pool inspect Display detailed information on one or more capacity pools ls List capacity pools rm Remove one or more capacity pools update Update a pool Run \u0026#39;storageos pool COMMAND --help\u0026#39; for more information on a command. storageos pool create To create a pool, consisting of nodes with a given label:\n$ storageos pool create no-ha --node-selector storage=fast no-ha To add a label at time of creation:\n$ storageos pool create production --label env=prod production storageos pool ls To list pools:\n$ storageos pool ls NAME DEFAULT NODE_SELECTOR DEVICE_SELECTOR NODES TOTAL USED default true 1 40.58GB 4.88% no-ha false storage=fast 1 40.58GB 4.88% production false 0 0B - storageos pool inspect To inspect the metadata for a given pool:\n$ storageos pool inspect no-ha [ { \u0026#34;id\u0026#34;: \u0026#34;e911c9fe-7d1c-8482-48b6-d0b1283476d4\u0026#34;, \u0026#34;name\u0026#34;: \u0026#34;no-ha\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;default\u0026#34;: false, \u0026#34;nodeSelector\u0026#34;: \u0026#34;storage=fast\u0026#34;, \u0026#34;deviceSelector\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;capacityStats\u0026#34;: { \u0026#34;totalCapacityBytes\u0026#34;: 40576331776, \u0026#34;availableCapacityBytes\u0026#34;: 38595940352, \u0026#34;provisionedCapacityBytes\u0026#34;: 0 }, \u0026#34;nodes\u0026#34;: [ { \u0026#34;id\u0026#34;: \u0026#34;ba9eca89-ea10-ce4e-26b9-85331f0b5ee2\u0026#34;, \u0026#34;hostname\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;address\u0026#34;: \u0026#34;192.168.50.100\u0026#34;, \u0026#34;kvAddr\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;apiPort\u0026#34;: 5705, \u0026#34;natsPort\u0026#34;: 5708, \u0026#34;natsClusterPort\u0026#34;: 5710, \u0026#34;serfPort\u0026#34;: 5711, \u0026#34;dfsPort\u0026#34;: 5703, \u0026#34;kvPeerPort\u0026#34;: 5707, \u0026#34;kvClientPort\u0026#34;: 5706, \u0026#34;labels\u0026#34;: { \u0026#34;storage\u0026#34;: \u0026#34;fast\u0026#34; }, \u0026#34;logLevel\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;logFormat\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;logFilter\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;bindAddr\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;deviceDir\u0026#34;: \u0026#34;/var/lib/storageos/volumes\u0026#34;, \u0026#34;join\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;kvBackend\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;debug\u0026#34;: false, \u0026#34;devices\u0026#34;: [ { \u0026#34;ID\u0026#34;: \u0026#34;59dadbcb-22c6-a01c-8334-1ae9f5ed72c6\u0026#34;, \u0026#34;labels\u0026#34;: { \u0026#34;default\u0026#34;: \u0026#34;true\u0026#34; }, \u0026#34;status\u0026#34;: \u0026#34;active\u0026#34;, \u0026#34;identifier\u0026#34;: \u0026#34;/var/lib/storageos/data\u0026#34;, \u0026#34;class\u0026#34;: \u0026#34;filesystem\u0026#34;, \u0026#34;capacityStats\u0026#34;: { \u0026#34;totalCapacityBytes\u0026#34;: 40576331776, \u0026#34;availableCapacityBytes\u0026#34;: 38595940352, \u0026#34;provisionedCapacityBytes\u0026#34;: 0 }, \u0026#34;createdAt\u0026#34;: \u0026#34;2018-06-22T10:18:28.528081144Z\u0026#34;, \u0026#34;updatedAt\u0026#34;: \u0026#34;2018-06-22T11:07:58.876658156Z\u0026#34; } ], \u0026#34;hostID\u0026#34;: 51087, \u0026#34;name\u0026#34;: \u0026#34;storageos-1\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;createdAt\u0026#34;: \u0026#34;2018-06-22T10:18:28.523996151Z\u0026#34;, \u0026#34;updatedAt\u0026#34;: \u0026#34;2018-06-22T11:07:58.894679039Z\u0026#34;, \u0026#34;health\u0026#34;: \u0026#34;healthy\u0026#34;, \u0026#34;healthUpdatedAt\u0026#34;: \u0026#34;2018-06-22T10:18:38.83674189Z\u0026#34;, \u0026#34;versionInfo\u0026#34;: { \u0026#34;storageos\u0026#34;: { \u0026#34;name\u0026#34;: \u0026#34;storageos\u0026#34;, \u0026#34;buildDate\u0026#34;: \u0026#34;2018-05-25T190132Z\u0026#34;, \u0026#34;revision\u0026#34;: \u0026#34;f8915fa\u0026#34;, \u0026#34;version\u0026#34;: \u0026#34;1.0.0\u0026#34;, \u0026#34;apiVersion\u0026#34;: \u0026#34;1\u0026#34;, \u0026#34;goVersion\u0026#34;: \u0026#34;go1.9.1\u0026#34;, \u0026#34;os\u0026#34;: \u0026#34;linux\u0026#34;, \u0026#34;arch\u0026#34;: \u0026#34;amd64\u0026#34;, \u0026#34;kernelVersion\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;experimental\u0026#34;: false } }, \u0026#34;version\u0026#34;: \u0026#34;StorageOS 1.0.0 (f8915fa), built: 2018-05-25T190132Z\u0026#34;, \u0026#34;Revision\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;scheduler\u0026#34;: true, \u0026#34;cordon\u0026#34;: false, \u0026#34;drain\u0026#34;: false, \u0026#34;volumeStats\u0026#34;: { \u0026#34;masterVolumeCount\u0026#34;: 1, \u0026#34;replicaVolumeCount\u0026#34;: 0, \u0026#34;virtualVolumeCount\u0026#34;: 0 }, \u0026#34;capacityStats\u0026#34;: { \u0026#34;totalCapacityBytes\u0026#34;: 40576331776, \u0026#34;availableCapacityBytes\u0026#34;: 38595940352, \u0026#34;provisionedCapacityBytes\u0026#34;: 0 } } ], \u0026#34;labels\u0026#34;: {} } ] storageos pool rm To delete a pool:\n$ storageos pool rm no-ha no-ha ","excerpt":"$ storageos pool Usage:\tstorageos pool COMMAND Manage capacity pools Options: --help Print usage …","ref":"/v1.x/docs/reference/cli/pool/","title":"Pools"},{"body":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational database management system (ORDBMS).\nPostgres is deployed across a wide variety of platforms with a mix of workloads ranging from small, single-node use cases to large Internet-facing clusters with many concurrent users.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. If you need to setup StorageOS on Kubernetes then please see our guide to [installing StorageOS on Kubernetes]({% link \u0026ldquo;docs/install/kubernetes.md\u0026rdquo; \u0026gt;}}).\nDeploying PostgreSQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases PersistentVolumeClaim and Pod definition excerpts\nkind:PersistentVolumeClaimmetadata:name:pg-dataannotations:volume.beta.kubernetes.io/storage-class:fast...kind:Podmetadata:name:postgressspec:securityContext:fsGroup:26containers:- name:pgimage:crunchydata/crunchy-postgres:centos7-10.4-1.8.3volumeMounts:- mountPath:/pgdataname:data...volumes:- name:datapersistentVolumeClaim:claimName:pg-dataThis excerpt is from the PersistentVolumeClaim and Pod definition. The pod definition references the pg-data VolumeClaim so storage is dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the PostgreSQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./postgres Confirm PostgreSQL is up and running.\n$ kubectl get pod postgres-0 -w NAME READY STATUS RESTARTS AGE postgres-0 1/1 Running 0 1m Connect to the PostgreSQL client pod and connect to the PostgreSQL server through the service.\n$ kubectl exec -it postgres-0 -- psql -h postgres-0.postgres -U primaryuser postgres -c \u0026#34;\\l\u0026#34; Password for user primaryuser: password List of Databases Name | Owner | Encoding | Collate | Ctype | Access privileges +=========================================================================+ postgres | postgres | SQL_ASCII | C | C | template0 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres template1 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres userdb | postgres | SQL_ASCII | C | C | =Tc/postgres + | | | | | postgres=CTc/postgres+ | | | | | testuser=CTc/postgres (4 rows) The password for the primary user is password. You can see this is set in the ConfigMap file.\n Configuration If you need custom startup options, you can edit the ConfigMap file 15-postgresd-configmap.yaml with your desired PostgreSQL configuration settings.\n","excerpt":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational …","ref":"/v1.x/docs/usecases/postgres/","title":"PostgreSQL"},{"body":"Our Prometheus endpoint exposes metrics about StorageOS artefacts (such as volumes), as well as internal StorageOS components.\nCustomers may scrape these metrics using Prometheus itself, or any compatible client, such as the popular Telegraf agent shipped with InfluxDB.\nArtefact Metrics Artefact metrics are those which instrument a specific StorageOS artefact. Typically these relate to volumes. These are useful for general purpose monitoring.\nVolume metrics make use of the following StorageOS logical concepts:\n Frontend - the layer serving a volume to an application. Always on the same host as the application. Network - the layer serving IO to remote volumes Backend - the layer representing IO to/from physical media. Not necessarily on the same host as an application. Name Explanation Additional Notes storageos_volume_backend_read_bytes_total Backend read operations Bandwidth/volume storageos_volume_backend_read_total Backend read operations Can be used to derive IOPS storageos_volume_backend_write_bytes_total Backend write bytes Bandwidth/volume storageos_volume_backend_write_total Backend write operations Can be used to derive IOPS storageos_volume_capacity_bytes Provisioned volume size bytes storageos_volume_frontend_read_bytes_total Frontend read bytes Bandwidth/volume storageos_volume_frontend_read_error_total Frontend read errors storageos_volume_frontend_read_total Frontend read operations Can be used to derive IOPS storageos_volume_frontend_write_bytes_total Frontend write bytes Bandwidth/volume storageos_volume_frontend_write_error_total Frontend write errors storageos_volume_frontend_write_total Frontend write operations Can be used to derive IOPS storageos_volume_network_read_bytes_total Network read bytes Bandwidth/volume storageos_volume_network_read_error_total Network read errors storageos_volume_network_read_retry_total Network read retries storageos_volume_network_read_total Network read operations Can be used to derive IOPS storageos_volume_network_read_wait_retry_total Network read delayed retries QOS related storageos_volume_network_write_bytes_total Network write bytes Bandwidth/volume storageos_volume_network_write_error_total Network write errors storageos_volume_network_write_retry_total Network write retries storageos_volume_network_write_total Network write operations Can be used to derive IOPS storageos_volume_network_write_wait_retry_total Network write delayed retries QOS related storageos_volume_utilisation_actual_bytes Backend non-zero device blocks A count of 1MiB chunks that have been written to since volume creation. The count is taken before compression, encryption etc. storageos_volume_utilisation_apparent_bytes Backend storage size Total blob file size on the host machine Node Metrics Our node metrics instrument various aspects of our container operation. These are illustrative of the health of your cluster, and we may ask you to provide them during a support engagement.\n Name Explanation Additional Notes exposer_bytes_transferred bytesTransferred to metrics services exposer_request_latencies Latencies of serving scrape requests, in microseconds exposer_request_latencies_count Total number of exposer_request_latencies exposer_request_latencies_sum Sum of all exposer_request_latencies exposer_total_scrapes Number of times metrics were scraped go_gc_duration_seconds A summary of the GC invocation durations. go_gc_duration_seconds_count Total number of go_gc_duration_seconds go_gc_duration_seconds_sum Sum of all go_gc_duration-seconds go_goroutines Number of goroutines that currently exist. go_info Information about the Go environment. go_memstats_alloc_bytes Number of bytes allocated and still in use. go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. go_memstats_frees_total Total number of frees. go_memstats_gc_cpu_fraction The fraction of this program\u0026rsquo;s available CPU time used by the GC since the program started. go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. go_memstats_heap_inuse_bytes Number of heap bytes that are in use. go_memstats_heap_objects Number of allocated objects. go_memstats_heap_released_bytes Number of heap bytes released to OS. go_memstats_heap_sys_bytes Number of heap bytes obtained from system. go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. go_memstats_lookups_total Total number of pointer lookups. go_memstats_mallocs_total Total number of mallocs. go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. go_memstats_other_sys_bytes Number of bytes used for other system allocations. go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. go_memstats_sys_bytes Number of bytes obtained from system. go_threads Number of OS threads created. storageos_control_process_cpu_seconds_total Total user and system CPU time spent in seconds (StorageOS control process) storageos_control_process_major_faults_total Total number of major page faults initiated by the process (StorageOS control process) storageos_control_process_max_fds Maximum number of open file descriptors (StorageOS control process) storageos_control_process_open_fds Number of open file descriptors (StorageOS control process) storageos_control_process_resident_memory_bytes Resident memory size in bytes (StorageOS control process) storageos_control_process_start_time_seconds Start time of the process since unix epoch in seconds (StorageOS control process) storageos_control_process_sys_cpu_seconds_total Total system CPU time spent in seconds (StorageOS control process) storageos_control_process_threads_total Number of currently spawned threads (StorageOS control process) storageos_control_process_user_cpu_seconds_total Total user CPU time spent in seconds (StorageOS control process) storageos_control_process_virtual_memory_bytes Virtual memory size in bytes (StorageOS control process) storageos_control_process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes (StorageOS control process) storageos_dataplane_process_cpu_seconds_total Total user and system CPU time spent in seconds (StorageOS dataplane process) storageos_dataplane_process_major_faults_total Total number of major page faults initiated by the process (StorageOS dataplane process) storageos_dataplane_process_max_fds Maximum number of open file descriptors (StorageOS dataplane process) storageos_dataplane_process_open_fds Number of open file descriptors (StorageOS dataplane process) storageos_dataplane_process_resident_memory_bytes Resident memory size in bytes (StorageOS dataplane process) storageos_dataplane_process_start_time_seconds Start time of the process since unix epoch in seconds (StorageOS dataplane process) storageos_dataplane_process_sys_cpu_seconds_total Total system CPU time spent in seconds (StorageOS dataplane process) storageos_dataplane_process_threads_total Number of currently spawned threads (StorageOS dataplane process) storageos_dataplane_process_user_cpu_seconds_total Total user CPU time spent in seconds (StorageOS dataplane process) storageos_dataplane_process_virtual_memory_bytes Virtual memory size in bytes (StorageOS dataplane process) storageos_dataplane_process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes (StorageOS dataplane process) storageos_local_leader_detected_node_failed_total Number of nodes the leader attempted to mark as failed storageos_local_leader_known_nodes_sync_seconds_bucket Time taken to synchronise known node list from data store storageos_local_leader_known_nodes_sync_seconds_count Total number of storageos_local_leader_known_nodes_sync_seconds_count storageos_local_leader_known_nodes_sync_seconds_sum Total of all storageos_local_leader_known_nodes_sync_seconds_count storageos_local_leader_known_nodes_total Number of nodes being actively monitored by this leader storageos_local_leader_volume_master_recover_total Number of master volumes the leader recovered storageos_node_device_capacity_bytes Total device capacity usable by StorageOS storageos_node_device_free_bytes Available device capacity usable by StorageOS storageos_node_dp_config_sync_seconds_bucket Time taken to compare desired dataplane config state versus actual and apply differences storageos_node_dp_config_sync_seconds_count Total number of storageos_node_dp_config_sync_seconds storageos_node_dp_config_sync_seconds_sum Total of storageos_node_dp_config_sync_seconds storageos_node_volumes_total Volumes on this node storageos_stats_process_cpu_seconds_total Total user and system CPU time spent in seconds (StorageOS stats process) storageos_stats_process_major_faults_total Total number of major page faults initiated by the process (StorageOS stats process) storageos_stats_process_max_fds Maximum number of open file descriptors (StorageOS stats process) storageos_stats_process_open_fds Number of open file descriptors (StorageOS stats process) storageos_stats_process_resident_memory_bytes Resident memory size in bytes (StorageOS stats process) storageos_stats_process_start_time_seconds Start time of the process since unix epoch in seconds (StorageOS stats process) storageos_stats_process_sys_cpu_seconds_total Total system CPU time spent in seconds (StorageOS stats process) storageos_stats_process_threads_total Number of currently spawned threads (StorageOS stats process) storageos_stats_process_user_cpu_seconds_total Total user CPU time spent in seconds (StorageOS stats process) storageos_stats_process_virtual_memory_bytes Virtual memory size in bytes (StorageOS stats process) storageos_stats_process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes (StorageOS stats process) storageos_store_query_seconds_bucket Data store query latency by operation type May reveal problems with external etcd storageos_store_query_seconds_count Total number of storageos_store_query_seconds storageos_store_query_seconds_sum Total of all storageos_store_query_seconds ","excerpt":"Our Prometheus endpoint exposes metrics about StorageOS artefacts (such as volumes), as well as …","ref":"/v1.x/docs/reference/prometheus/","title":"Prometheus"},{"body":"CoreOS has created a Kubernetes Operator for installing Prometheus. The operator uses ServiceMonitor custom resources (CRs) to scrape IP addresses defined in Kubernetes Endpoints. This article is intended to provide a quick guide to monitoring the StorageOS metrics endpoint and can be used with our example Grafana dashboard.\nScripted Prometheus Install There is a install-prometheus.sh script that will perform the installation of Prometheus using the Prometheus operator and a ServiceMonitor monitoring StorageOS. If you wish to use it then follow steps 1 and 2 before skipping to installing Grafana. If you do not wish to use the script then go to the Prometheus Operator installation.\n N.B. The standard installation of Prometheus and Grafana does not have persistent storage enabled. If you wish to use persistent storage then please see the README.md in the Prometheus directory from Step 1.\n Clone the StorageOS deploy repository and move into the prometheus-operator directory git clone https://github.com/storageos/deploy.git storageos cd storageos/k8s/examples/prometheus Run the install-prometheus.sh script. ./install-prometheus.sh Install the Prometheus Operator Clone the StorageOS deploy repository and move into the prometheus-operator directory git clone https://github.com/coreos/prometheus-operator.git prometheus-operator Deploy the quick start bundle.yaml kubectl create -f prometheus-operator/bundle.yaml Verify that the Prometheus operator is running. kubectl get pods -l apps.kubernetes.io/name=prometheus-operator Install Prometheus Now that the Prometheus Operator is installed, a Prometheus CR can be created which the Prometheus operator will act upon to configure a Prometheus StatefulSet.\n Clone the StorageOS deploy repo git clone https://github.com/storageos/deploy.git storageos cd storageos/k8s/examples/prometheus If your cluster uses RBAC then create the necessary Cluster role and service account for Prometheus. kubectl create -f prometheus-rbac.yaml Create a Prometheus CR that defines a Prometheus StatefulSet. kubectl create -f prometheus-cr.yaml Create a ServiceMonitor CR that directs Prometheus to scrape the Endpoints defined in the storageos Endpoints resource. Prometheus will scrape the /metrics URL of the Endpoints and collect the metrics. kubectl create -f storageos-serviceMonitor.yaml In order to view the Prometheus UI in the browser port forward the local port to the Prometheus pod port. kubectl port-forward prometheus-prometheus-storageos-0 9090 The Prometheus UI can now be seen in the browser at localhost:9090\n Now that the Prometheus UI is available StorageOS metrics can be queried from the Graph page. A complete list of StorageOS metrics can be found here Install Grafana Grafana is a popular solution for visualising metrics. At the time of writing (30/04/2019) there is no Grafana operator so instead a helm installation is used. If a helm installation will not work then the helm generated manifests can be used.\n Install Grafana\nhelm install stable/grafana Grafana can query the Prometheus pod for metrics, through a Service. The Prometheus operator automatically creates a service in any namespace that a Prometheus resource is created in. Setup a Grafana data source that points at the Prometheus service that was created. The URL to use will depend on the namespace that Grafana is installed into.\nIf the Grafana pod runs in the same namespace as the Prometheus pod then the URL is: http://prometheus-operated:9090 otherwise it\u0026rsquo;s http://prometheus-operated.$NAMESPACE.svc:9090\nWhen creating the data source make sure to set the scrape interval.\n Once the Prometheus data source has been created have a look at the example StorageOS dashboard for ideas about how to monitor your cluster.\n ","excerpt":"CoreOS has created a Kubernetes Operator for installing Prometheus. The operator uses ServiceMonitor …","ref":"/v1.x/docs/operations/monitoring/prometheus-setup/","title":"Setting up Prometheus to monitor StorageOS"},{"body":" Start by understanding the architecture and playing with a demo\n For production deployments, follow our installation guides for Kubernetes, OpenShift or Rancher.\n Dive into our Use Case documentation for Workloads deployed in Kubernetes.\n ","excerpt":" Start by understanding the architecture and playing with a demo\n For production deployments, …","ref":"/v1.x/docs/introduction/quickstart/","title":"Quickstart"},{"body":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS transparently supports Rancher deployments using CentOS, RHEL, Debian or RancherOS (CSI is not supported on RancherOS) and can support other Linux distributions detailed in the systems supported page if the appropriate kernel modules are present.\n","excerpt":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS …","ref":"/v1.x/docs/platforms/rancher/","title":"Rancher"},{"body":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional durability to disk.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Redis on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:redisspec:selector:matchLabels:app:redisenv:prodserviceName:redisreplicas:1...spec:serviceAccountName:redis...volumeMounts:- name:datamountPath:/bitnami/redis/data...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the Redis examples folder and create the objects\ncd storageos-usecases kubectl create -f ./redis Confirm Redis is up and running.\n$ kubectl get pods -w -l app=redis NAME READY STATUS RESTARTS AGE redis-0 1/1 Running 0 1m Connect to the Redis client pod and connect to the Redis server through the service\n$ kubectl exec -it redis-0 -- redis-cli -a password Warning: Using a password with \u0026#39;-a\u0026#39; option on the command line interface may not be safe. 127.0.0.1:6379\u0026gt; CONFIG GET maxmemory 1) \u0026#34;maxmemory\u0026#34; 2) \u0026#34;0\u0026#34; Configuration If you need custom startup options, you can edit the ConfigMap file 15-redis-configmap.yaml with your desired Redis configuration settings.\n","excerpt":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional …","ref":"/v1.x/docs/usecases/redis/","title":"Redis"},{"body":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than \u0026ldquo;latest\u0026rdquo;, and to perform upgrades only after reading the release notes.\nThe latest tagged release is 1.5.4, available from Docker Hub as storageos/node:1.5.4, or via the Helm Chart\nThe latest CLI release is 1.2.2, available from Github\nUpgrading See upgrades for upgrade strategies.\n1.5.4 - Released 2020/08/19 Fixed If a CSI volume create request expires before completing, the retry operation may not succeed. When detaching NFS volumes, return success if the volume has already been deleted. This avoids Kubernetes retrying the operation indefinitely. Don\u0026rsquo;t overlap /var/lib/kubelet mount. This could result in multiple entries for /var/lib/storageos in /etc/fstab. Set fsid_device to false so that NFS does not use local device major/minor numbers for filesystem descriptors. Run CSI containers as privileged. Pass cluster node selector to NFS pods. As an additional safeguard, check that the StorageOS device contains only zeros in the first 256KB before running mkfs. blkid is also still used to check for existing filesystems. Reduced writes to etcd for health checks. Fixed error \u0026ldquo;failed to detach volumes\u0026rdquo; \u0026hellip; \u0026ldquo;resource name may not be empty\u0026rdquo;. 1.5.3 - Released 2020/01/25 Fixed Resolved misrepresentation of internal metrics for licensed clusters. 1.5.2 - Released 2019/12/18 Adds support for Kubernetes 1.17 and CSI 1.2. Also reduces node container size by 70%.\nNew storageos/node container size reduced by removing extraneous contents. This reduces the amount of time required to install and boostrap a new cluster. Kubernetes 1.17.0 supported, resolving cannot get resource \u0026quot;leases\u0026quot; in API group \u0026quot;coordination.k8s.io\u0026quot; permission issue. Now supports version 1.2.0 of the CSI specification. Improved CSI helpers updated to latest versions. operator-sdk dependency updated to version 0.12. Downgraded some log messages for transient failures from error to info. Fixed Resolved documentation inconsistency between Openshift Marketplace and docs.storageos.com. Node label update now retries if the node is not yet known to StorageOS. 1.5.1 - Released 2019/11/26 Fixes a permission error on Kubernetes 1.16 and improves reporting of Pod scheduling failures.\nNew The [cluster-operator] now reports internal metrics suitable for scraping with Prometheus on http://0.0.0.0:8686/metrics. If Prometheus has been installed in the cluster using the [prometheus-operator], then ServiceMonitor resources will be created to register the endpoint automatically. See [prometheus-setup] for an example Prometheus installation. Currently only the default process metrics are reported but we intend to improve coverage over the next few releases. Improved Pod scheduling failures are now registered as Pod events to provide contextual error messages. Before, only HTTP status codes could be reported since that was the only information that could be passed via kube-scheduler.\n Leader election has been added to [cluster-operator] and \u0026ldquo;leader-for-life\u0026rdquo; enabled by default. In the future, this will allow multiple instances of the operator to be running simultaneously. For now, only a single instance is supported. See [leader-election] for more information.\n The [cluster-operator] now fully implements status phases. This means that Kubernetes and kubectl now report the following statuses for StorageOSCluster resources:\n Pending cluster status when the requirements are unmet. New clusters may wait in Pending state while there is an existing cluster running. Creating cluster status when the provisioning has started. Running cluster status for a provisioned cluster. Terminating cluster status when a cluster delete has been initiated. Updated the [cluster-operator] base image to latest ubi8/ubi-minimal.\n Fixed Fixed permission errors in Kubernetes 1.16:\n Failed to list *v1beta1.CSINode: csinodes.storage.k8s.io is forbidden 'events.events.k8s.io is forbidden Deployments using embedded etcd could fail to bootstrap when Kubernetes returns multiple addresses for a node. Previously, the first address in the list was assumed to be the address that the nodes should communicate on. Instead, we now prefer to use the InternalIP address, and if that is not found, the first address used as before.\n 1.5.0 - Released 2019/11/12 Along with a 10x improvement in sequential read performance, version 1.5.0 adds two much-requested features; Pod Locality and Shared Filesytems (RWX).\nNew Pod Locality improves application performance by placing workloads in close proximity to its data. In the default configuration, and when resources permit, StorageOS will instruct Kubernetes to prefer placing the workload on the node holding the master copy of the data, and then prefer the replicas. Where this is not possible, any other healthy node running StorageOS will be considered.\nPod Locality is enabled by default.\nSee [Pod locality]({{ ref \u0026ldquo;docs/concepts/podlocality.md\u0026rdquo; \u0026gt;}}) for more information.\n Shared Filesystem support allows StorageOS volumes to be provisioned for read \u0026amp; write access by multiple Pods concurrently. Using the same StorageOS StorageClass, PVC requests can now specify AccessMode ReadWriteMany (RWX) for shared filesystem, or ReadWriteOnce (RWO) for a standard volume.\nVolume features such as replication, encryption and caching can be applied to RWX volumes by using PVC labels and StorageClass parameters in the same way as with RWO volumes.\nEach shared filesystem is exposed via a dedicated service, named after the PVC name. The service also exposes Prometheus metrics for the shared volume on http://\u0026lt;service\u0026gt;:80/metrics. When Prometheus has been deployed using the Prometheus Operator, ServiceMonitor resources are automatically created for each endpoint.\nShared filesystem support is only available when using the CSI driver, which is the recommended deployment method from Kubernetes 1.12 onwards.\nAlthough Shared Filesystem support has been tested by customers in our Beta Programme and we are not aware of any critical issues, we are releasing the feature as a Technology Preview. We encourage all users to test Shared Filesystem support and let us know of any issues or how it can be improved before we recommend it for production workloads.\nSee Shared filesystems for more information.\n Improved Read performance for all workload types and block sizes have been improved, with sequential workloads now 10x faster than version 1.4.0. Remote read performance for large block sizes has also been substantially improved with optimizations to reduce round trips, increasing throughput and lowering overall latency. Remote writes for large block sizes have already been optimized. Block checksums are now stored with the volume metadata. While not exposed as features in 1.5.0, replica consistency can now be quickly proven and corruptions corrected. Existing volumes will not be affected, only newly created volumes. The dataplane log parser has been replaced and is now more efficient, and handles embedded quotes. Telemetry now uses gRPC over TLS. The existing Sentry-based telemetry will be phased out. Telemetry can be disabled by setting DISABLE_TELEMETRY=true. Fixed Encrypted volumes did not always respect the compression setting, resulting in capacity savings being lower than expected. Unlike most storage systems, StorageOS is able to compress data prior to encrypting it. Other The maximum provisioned capacity of an unregistered cluster (default install) has been reduced from 100GB to 50GB. We have made this change to encourage users to register for the free Developer licence, which grants 500GB of provisioned capacity.\nThrough cluster registrations we are able to get a better understanding of who is using StorageOS and which events or referrals are most effective. We would love to understand your challenges and any ideas you may have to improve StorageOS. You may unsubscribe from our mailing list at any time.\n 1.4.0 - Released 2019/09/06 Version 1.4.0 changes the way metadata keys are stored on disk resulting in improved key locality and thus an increase in IO performance.\nIMPORTANT: Due to the nature of this change, the upgrade process includes an automated conversion process. If upgrading from 1.3.0 or below, please ensure you follow the upgrade instructions. If the upgrade process is not followed, version 1.4.0 will refuse to start, but you may restart using the previous version. Once the upgrade process has been completed it is not possible to revert to an earlier version.\nNo new features will be added to the 1.4 series and we will instead release a 1.5.0-rc1 in the next few weeks.\nImproved Improved performance for sequential IO greater than 4k in size by changing the ordering of metadata keys so that the volume inode is in the most significant bits, followed by the offset. Volume deletes are now 5 - 20x faster, and for \u0026lt;10GB volumes should take less than a second. Fixed The Prometheus metrics endpoint no longer displays statistics for volumes that have been deleted. 1.3.0 - Released 2019/07/08 The 1.3.0 release contains further substantial performance improvements, particularly for workloads that can parallelise IO or when multiple volumes are performing high rates of IO.\nNew Switched internal device presentation from FUSE to TCMU when the kernel supports it, and when no conflicting TCMU-based devices already exist. TCMU improves performance in all use cases, but particularly when queue depths are greater than 1. TCMU can be disabled by setting disableTCMU: true, and it can be forced by setting forceTCMU: true. When forceTCMU: true has been set and either the kernel does not support TCMU or there are conflicting devices, then StorageOS will refuse to start. The operator now runs an additional scheduler Pod, which can be disabled by setting disableScheduler: true in the StorageOS CR. Improved The admin user\u0026rsquo;s username and password are no longer stored in the KV store and are now always read from the ADMIN_USERNAME and ADMIN_PASSWORD environment variables which are populated by the secret from secretRefName. Previously, the environment variables were only read on cluster bootstrap. This caused problems when re-installing a cluster as the previous credentials were cached in the KV store and the credentials set in the environment variables would not work. It is no longer required to clear /var/lib/storageos between re-installs. Replication ACK timeouts have be reduced from 15 to 3 seconds to allow retries to be performed sooner. The overall timeout remains the same. Telemetry now reports the Kubernetes version and the distribution name if set in the k8sDistro environment variable. This information helps us direct focus on the most relevant platforms. As always, telemetry reporting can be disabled by setting disableTelemetry: true. Low-level dataplane configuration is now included in the diagnostics bundle. The bundle can only be created by the cluster administrator and can be uploaded to assist with support. Logging for the Operator has been reworked to allow easier troubleshooting and consistency with other operators using Kubernetes\u0026rsquo; controller-runtime library. Fixed When creating volumes in stress test conditions, it was possible to be re-allocated the inode of a volume that had not yet finished the deletion process, causing the create to fail. When issuing an unmount request using the CSI driver, the operation would fail if the volume was already unmounted. The operation now returns success and allows a remount to succeed. 1.2.1 - Released 2019/05/15 1.2.1 is primarily a bug fix release, but it also includes some performance improvements.\nImproved Performance tuning on the metadata store, improving performance for sustained writes. Metadata checksums now use a pipelined SSE algorithm to improve performance. Set SCSI timeouts for StorageOS volumes to 120 seconds. More robust handling of devices on first initialisation. Volumes created with CSI are now formatted with lazy initialization enabled, reducing the time taken to provision the volume. Logging for volume detach operations when fencing a Pod. Cluster maintenance logging improved. Fixed Improved handling of in-memory cache during repeated failover events. When a volume was being deleted while there was IO pending to an offline remote node, the delete would wait until the IO timed out. IO is now abandoned when a volume is deleted. During shutdown, IO to offline remote nodes would be retried, causing shutdown to take longer than needed. Now, unsuccessful IO is marked as failed immediately if a shutdown has been requested. The network client is now shutdown before the block presentation layer so that threads with pending IO are immediately unblocked. This helps ensure shutdowns can finish gracefully within the 10 second window set by the orchestrator. Node inspect output did not show all configuration information. When CSI is enabled, devices are now created on all nodes instead of on demand. This mitigates a timing issue and allows faster failover. Replication logs could contain incorrect volume identifiers when multiple syncs are in progress. 1.2.0 - Released 2019/04/17 The 1.2.0 release contains significant performance and resource utilisation improvements, along with 3 new features:\n Pod fencing Volume Encryption etcd TLS New Pod fencing can improve application failover times, particularly for applications deployed as StatefulSets. When the label storageos.com/fenced=true is applied to a Pod, that mounts a replicated StorageOS volume, and StorageOS detects that the node running the Pod is offline, it will tell Kubernetes to re-schedule the Pod elsewhere.\nWithout this feature, Kubernetes will by default wait 5 minutes before marking the node offline and performing remedial actions. When applications are deployed as StatefulSets then manual intervention is also required before Pods will be re-scheduled. This behaviour guards against multiple nodes accessing the same volume concurrently as Kubernetes does not have visibility of storage presentations.\nSince StorageOS detects and remediates node failure more aggressively (in 30-40 seconds) and it protects against multiple nodes accessing volumes concurrently, the fencing option gives the ability to override the default Kubernetes behaviour.\nSee fencing for more information.\n Volume Encryption can now be enabled by setting the storageos.com/encryption=true label on a volume. When enabled, data will be encrypted prior to being persisted on disk.\nSee encrypted volumes for more information.\n Mutual TLS is now supported on external etcd KV backends.\nSee etcd TLS for more information.\n When deployed using the StorageOS Cluster Operator, node labels from Kubernetes are visible within StorageOS and available for use in rules. This is particularly useful to mark specific nodes as compute-only. See labels for more information.\n Improved Multiple significant performance improvements.\n Multiple data plane processes were merged into a single dataplane binary. This has multiple benefits:\n Improved performance by reducing context switches when passing data between processes. Easier to instrument and trace. Startup and shutdown logic simpler to manage. DirectFS, the network communication layer used by replication and remote volumes, has been largely re-written and now uses a connection pool to reduce memory overhead.\n Several log formatting and level improvements to provide more relevant and consistent messages.\n Gossip (serf) port added to the network connectivity heath check.\n Fixed The CLI command storageos cluster health now returns more useful information about the health of the cluster and does not require the cluster ID or join token to be supplied as a parameter. When changing the cluster log level at run-time using storageos logs --log-level debug, the new log level would be applied but an incorrect error would be returned. 1.1.5 - Released 2019/03/29 Version 1.1.5 is an interim bugfix release as we continue to test the upcoming 1.2 feature release internally.\nImproved Previously we would allocate 5% of free memory to use as cache. On larger systems that were being rebooted, we could use more memory than intended if we start before other applications do. We now set a hard upper-bound of 512MB. The cluster ID is now logged more prominently during startup. Labels prefixed with kubernetes.io/ are now accepted. This allows core Kubernetes labels to be inherited and used for scheduling decisions. A cache flush is now triggered when fsck is run on a volume. Removed noisy log messages from the dataplane stats module. Increased log level to info for volume create and delete with CSI. Fixed Mounting a volume while the volume was undergoing a series of configuration updates could cause the mount to hang. When increasing the number of replicas for a mounted volume, the new replica could get stuck in \u0026ldquo;provisioned\u0026rdquo; state. Removing the stuck replica and re-trying would normally clear the issue. The root cause has now been fixed. When a volume was being deleted and the data scrub finished, if there was an error removing the volume configuration from the kv store (e.g. kv store unavailable), it would log an error and not retry. Now, the operation is re-tried. The soft and hard mode failure detection was incorrectly using the count of failed replicas in the calculation, not the active replicas. When new replicas are being provisioned it\u0026rsquo;s normal to have more than the requested number of replicas with some of them pending removal. The logic was changed to count healthy replicas instead. Fixed excessive log messages when removing a device for a volume that still exists elsewhere in the cluster. The UI incorrectly counted the replica stuck in \u0026ldquo;provisioned\u0026rdquo; state as available when it did not yet have a consistent copy of the the data. When setting the cluster log level via the CLI, an error was returned even though the update suceeded. The UI was displaying storage capacity in GB rather than GiB. In the UI, the volume delete hover was missing, causing the button to appear disabled. Adding a label to a volume in the UI could insert the string \u0026ldquo;undefined\u0026rdquo;. 1.1.4 - Released 2019/03/06 Includes a single fix to increase the time we wait for the operating system to create a new block device. On some platforms (cloud providers in particular) the variation can be wider than expected.\nFixed Address device creation delays on certain platforms/distributions. 1.1.3 - Released 2019/03/01 Version 1.1.3 is a bugfix and usability release. Please upgrade if you are using external etcd.\nNew There are no new features. Look out for 1.2 coming soon! Improved When using an external etcd KV store and it becomes inaccessible, StorageOS is now more resilient. All volumes remain online and usable but configuration changes are restricted. Added a warning in the UI if cluster nodes are running different StorageOS versions. The warning includes a link to the upgrade documentation. The cluster ID is now more prominent in the UI. The cluster ID is required for licensing and support. UI login failure notifications are now automatically dismissed after a successful login. Improved error handling and logging when configuring volumes in the data plane. Strict dependency checking when configuring volumes. Fixed When refreshing the volumes page in the UI could return a \u0026ldquo;volume not found\u0026rdquo; error. When editing volume labels in the UI, there is now a confirmation check before removing a label. Fixed a bug that might prevent a volume delete request from being actioned. This would appear as a volume that is stuck in deleting state. 1.1.2 - Released 2019/02/03 1.1.2 improves stability when nodes lose network connectivity intermittenly by introducing a new grace period before performing node recovery operations.\nBreaking changes The environment variables introduced in version 1.1.1 to tune node failure detection have changed. Previously PROBE_INTERVAL_MS and PROBE_TIMEOUT_MS were set to a number of milliseconds. They have now been replaced with PROBE_INTERVAL and PROBE_TIMEOUT, which takes a time duration in string format, e.g. 500ms or 2s. We believe this change gives slightly more flexibility and is easier to use. Going forward, all duration-based configuration will use this format.\nNew When network connectivity to a node is confirmed lost, there is now a configurable grace period before recovery operations begin. The grace period allows time for the network or node to recover from short interruptions. The default is set to 30s. The minimum duration is 0s, and the maximum is 90s. In the UI, a button has been added to re-send the activation email when requesting a Developer licence. Improved Logs now use nanosecond precision for timestamps. This helps ensure the correct ordering when logging to an external source such as Elasticsearch.\n Upgraded Golang compiler to version 1.11.5.\n Support diagnostics can now be generated while nodes are waiting to join the cluster. This can help troubleshoot dependency or network connectivity issues that are blocking successful startup. This feature is currently only enabled in the API; in a future release it will be exposed in the UI and CLI. To generate the diagnostics bundle, run:\ncurl -o diagnostics.tar storageos:storageos@localhost:5705/v1/diagnostics/cluster Added more context to error logs when there is a failure to configure the dataplane.\n Added more logging at debug level when gossip detected a node\u0026rsquo;s health change.\n Fixed When CSI is enabled, if a volume was created in a namespace that did not already exist, then the create operation would fail. Now the namespace is created as part of the volume create operation. This matches the behaviour of the Kubernetes native driver. When importing Kubernetes ABAC policy files, the jsonl format was incorrectly detected and input was incorrectly parsed as json, which would not validate. 1.1.1 - Released 2019/01/21 1.1.1 contains a critical bugfix in the etcd client library that could cause the StorageOS control plane to become unresponsive when etcd is unavailable.\nNew Exposed tuning knobs for node failure detection via two new environment variables:\n PROBE_INTERVAL_MS: the interval in milliseconds between node probes. Setting this lower (more frequent) will cause the cluster to detect failed nodes more quickly at the expense of increased bandwidth usage. Defaults to 1000ms. PROBE_TIMEOUT_MS: the timeout to wait for an ack from a probed node before assuming it is unhealthy. This should be set to 99-percentile of RTT (round-trip time) on your network. Defaults to 3000ms. The defaults should be appropriate for most environments.\n Improvements If a node is determined to be offline by memberlist, we now verify by sending a TCP handshake from the scheduler to the node\u0026rsquo;s replication port. If the handshake is successful we abort the process to mark the node offline. If the StorageOS container was killed, we now log at error level instead of debug level. The UI now gives user feedback when a diagnostics bundle is being uploaded or downloaded. Volume details and dmesg output is now collected in the diagnostics bundle. This information helps our engineers diagnose issues more easily. Internal cleanup of API handlers and middleware. API errors now follow consistent formatting. Values for CSI parameters VolumeId and VolumePath are now validated to ensure they are not empty. This is internal to the interaction between the orchestrator and StorageOS and applies to the GetVolumeStats() CSI call. Removed spammy \u0026ldquo;failed to retrieve volume capacity stats\u0026rdquo; log message when a volume was in the process of being deleted. When a node is rebuilt and re-added to the cluster, manual steps must be taken to ensure that a previous (now invalid) config is not re-applied to the new node as it may not have the previous backend data. The log messages now better explain the issue and the steps required to resolve. Better error logging when data plane configuration was not applied. Fixed Updated the etcd client library to version 3.3.11, which includes a fix for https://github.com/etcd-io/etcd/issues/9578, a tight loop in the reconnect logic when a watch loses its connection to the etcd server. The loop would cause the control plane to become unresponsive, including not allowing it to respond to healthchecks, which could result in the node being taken offline. Fixed an issue in the etcd watcher when we specify an object version to use as a starting point to watch for change from, but that version no longer exists in etcd due to scheduled compaction of the etcd datastore. Now, the error is caught and retried from the latest version. The error was logged as: etcdserver: mvcc: required revision has been compacted. The diagnostics bundle was saving with the extension .tar.gz, even though it is not compressed. It now saves as .tar. When a node was heavily loaded, it was possible for volume delete requests to get stuck. The delete result processing is now done in a separate thread from the request so that a long request can\u0026rsquo;t block receiving responses. 1.1.0 - Released 2019/01/04 1.1.0 adds support for CSI 1.0 and includes some minor bug fixes.\nNew CSI 1.0 (available in Kubernetes 1.13 onwards) is now fully supported. Backwards compatibility with earlier versions of CSI and Kubernetes has been maintained and is the default. CSI 1.0 support is automatically enabled when using the StorageOS cluster-operator on Kubernetes 1.13 and above. If installing manually, set the CSI_VERSION environment variable to v1. Improvements In the UI, the namespaces in the dropdown list are now sorted in alphabetical order. Removed spammy log messages for IO size. When creating a volume using CSI, the namespace can be set by adding a volumenamespace label to the StorageClass. Fixed Cloud provider detection could cause a panic in non-AWS environments that respond on the same metadata ip address that AWS uses. Fixed in aws-sdk-go. Refreshing the UI caused the selected namespace to be reset. 1.0.2 - Released 2018/12/10 1.0.2 includes a fix required for Azure AKS and several other fixes and improvements, many to help with running on low-memory or heavily loaded servers.\nImprovements Cache allocation has been adjusted for lower memory systems. Allocation sizes are described in architecture. The cache size will never be set to more than 1/3rd of free memory.\n Increased the maximum number of replicas from 4 to 5.\n Increased the timeout for the device presentation shutdown to 9 seconds before sending a SIGKILL.\n Increased the wait timeout for dataplane startup from 10 to 30 seconds.\n Added colour-coded health labels in the UI.\n Rules can now be enabled and disabled in the UI.\n Improved input validation in the UI.\n IO error handling log messages are now more informative.\nNon-fatal (re-tryable) errors:\n Can't resolve replica inode configuration on local director Can't resolve master inode configuration on the remote director Can't resolve replica inode configuration on the remote director The client does not have the appropriate configuration No connection could be found Fatal errors:\n Generic fatal error (use this if there is no need to be specific) Can't resolve master inode configuration on the local director ENOSPC error Blobs corrupt/missing/not-sane etc A shm-pipe operation fatally failed Read only volume Sentinel value Corrected confusing log message: poll() returned error on network socket: Success. We now log: connection closed.\n Better logging when testing system device capabilities.\n Better logging for dataplane status errors.\n Fixed On Azure AKS, StorageOS could fail to restart. This was caused by an error in the system device capabilities verification incorrectly determining that the node didn\u0026rsquo;t meet prerequisites. During startup, it was possible to retrieve an uninitialized value from a dataplane status endpoint, which caused startup to fail. This was only seen on heavily-loaded systems. Potential deadlock in dynamic cache resize. This is not known to have occured. During a scheduler failover, it was possible to miss a node recovery gossip message and the node would never be marked as recovered. It was possible for a volume to get a volume stuck in syncing state if a KV watcher was interrupted. Now, full state from the KV store is re-evaluated every 10 seconds. When creating a volume using CSI, the pool field was ignored. 1.0.1 - Released 2018/11/23 This is primarily a bugfix release to better handle node recovery behaviour when network connectivity is re-established. There are also a few minor bugfixes and general improvements.\nImprovements Upgraded UI to use Vue CLI 3. It is no longer possible for a user to delete their own account, or for an administrator to downgrade their access to a normal user. This ensures that there is always at least one cluster administrator. When making volume placement decisions, the scheduler will now apply a lower ranking score to nodes that have come online less than 5 minutes ago. This allows the node to perform most recovery operations prior to accepting new volumes. This will not cause volume placement to be delayed, only that non-recovering nodes will be preferred. The cluster diagnostics bundle can now be downloaded via the Web UI in addition to uploading for StorageOS support. The volume list in the Web UI now shows more detailed volume health. Dataplane metrics now use less memory. Fixed If a node was partitioned as a result of a network failure, it would not always rejoin the cluster when the network was restored. Nodes now attempt to reconnect via gossip every 2 seconds. Dataplane process metrics (e.g. memory usage) were not being exposed via the Prometheus /metrics endpoint. storageos logs -f CLI command was incorrectly reporting all log entries at fatal level, overwriting the correct log level. Fixed errors in Web UI when API not yet available. 1.0.0 - Released 2018/11/15 1.0.0 is suitable for production workloads.\nNew Offline nodes can now be removed from management either by using the CLI node delete command, or in the UI. This is only available when external etcd is used as the KV store. See decommissioning nodes for more information. Nodes can now be placed into \u0026ldquo;Maintenance mode\u0026rdquo; to disable volume recovery on the node while it is offline for maintenance. This allows nodes to be upgraded without triggering potentially unwanted data migrations. Note that applications accessing StorageOS volumes on the node should also be shutdown. Alternatively, you may move applications off the node and use stoageos node drain to migrate data prior to taking the node offline. 5% of backend volume capacity is now reserved for indexes and recovery. This helps ensure that there is enough remaining capacity to perform recovery operations such as deleting or moving frontend volumes. Prometheus metrics for volume used capacity, and added to the API volume objects and the UI. A placeholder for actual used capacity (after compression) has also been added but is not yet provided. In the UI, storage pools and nodes now have an additional details page. Added group management to the UI. The overprovisioning ratio can now be configured on pools by applying the label storageos.com/overcommit to a positive integer representing the overcommit percentage. By default, overcommit is set to 0. At startup and every 24 hours, each cluster will make a DNS request to query the latest StorageOS version and log if a new version is available. The UI will also notify administrators. This check can be disabled by setting DISABLE_TELEMETRY=true. For more information see telemetry. Since data scrub operations can be intensive, we now limit to 5 concurrent volume scrubs. This does not affect the number of volumes that may be deleted in a batch as the scrubs occur asyncronously. Volumes will remain in deleted state until the data is scrubbed, then they will not be reported at all. Improved Updated Prometheus metrics names to follow best practices. Added process name to dataplane log messages. Removed excessive heartbeat messages from debug logging. Removed an unneeded mutex from the dataplane debug logging. Added protection against a potential hard hang while cleaning up devices by instructing the kernel to abort any IO on the device prior to removing. Add createdAt and updatedAt fields in volume replica API objects. All API errors now return JSON content type (content remains the same). UI improvements to remember state when validation fails. UI feedback and help text while requesting a developer licence is more intuitive. Better error message when manually applying an invalid licence. Volume detail page in the UI has numerous improvements to improve usability, including on smaller screen sizes. When the UI was accessed before the node was fully online it would show a blank page. It now shows the message Cluster API not yet available, waiting for nodes to join. The browser \u0026ldquo;back\u0026rdquo; button on the UI now works as expected. Label management in the UI now gives context-sensitive examples and improves validation. Added network diagnostics and Prometheus metrics to support diagnostics bundle. Updated container labels. Fixed When a replica is auto-promoted to master because the application container moved, a cache reset for the volume is now triggered prior to updating the presentation to ensure the cache can never serve stale data. Fixed a crash in the replication server during shutdown when a volume scrub operation was taking place on a deleted volume with a non-trivial amount of data. In a cluster with hundreds of volumes and nodes continuously rebooting, a race condition could cause a volume to be \u0026ldquo;stuck\u0026rdquo; in syncing state if the sync started before the dataplane was ready for operation. New replicas are now created with their health set to provisioned, and only enter the syncing state once the sync is underway. This allows the operation to be retried if required. After a restart, a write pointer was starting at the end of the current 64MB chunk. This would waste space if the chunk was not fully used. The pointer is now set to the end of the written data to ensure full utilisation. Check-and-set operations in the internal key-value store library were not honouring the TTL requested. This could cause CAS updates to fail when there was above average latency between nodes. If a non-admin user tried to cordon/drain a node in the UI it would fail silently. Now the action is disabled for non-admins and the API returns the correct error. In the UI, if a previous licensing operation failed, cached data could make it difficult to re-request a licence. If a volume was requested but failed due to validation (e.g. not enough capacity), it would be marked as failed and an attempt to re-create would fail with the message Volume with name 'foo' already exists. Now the second operation will replace the first and will be re-evaluated. After startup, pool capacity would only be calculated after the API was ready for use. This caused provisioning requests immediately after startup to fail. Capacity is now calculated prior to the API accepting provisioning requests. Slow/broken DNS responses could cause cloud provider detection to delay startup for up to 30 seconds while waiting for a timeout. Cluster health CLI and API endpoint no longer report an error when external etcd is used for the KV store. When creating new users, usernames are now validated to disallow names in the UUID format used internally. In the UI, non-admin users can now view and update their information, including changing their own password. Password resets now work correctly and can be used to create a licence for a portal account that was created with a social login. If there was an error deleting a rule, the API returned an Internal Server Error rather than a more helpful contextual error. 1.0.0-rc5 - Released 2018/10/01 1.0.0-rc5 is a major update, with multiple bug fixes, performance and usability improvements as we get closer to removing the RC label.\nBreaking changes Between rc4 and rc5 the location of the lock that governs configuration changes has moved. For maximum safety, shutdown StorageOS on all nodes prior to upgrading to rc5. If this is not feasible, contact support@storageos.com for further instructions. Prometheus metric names and types have changed to adhere to best practices. If you have existing dashboards they will need to be updated. New User, group and policy management added to the Web UI. A custom or self-hosted cluster discovery service can now be specified by setting the DISCOVERY_HOST environment variable to the hostname or ip of the running service. The source code for the discovery service is open and available on GitHub. Note that the cluster discovery service is optional and supplied as a convenience. The KV_ADDR environment variable now supports specifying multiple hostnames or addresses of external etcd servers. Hostnames and addresses should be comma-separated and can include the port number (required if not 2379, the etcd default). Passes CSI 0.3 (latest) conformance tests. RHEL/Centos has a limit of 256 devices per HBA. The API now returns cannot create new volume, active volumes at maximum when this limit has been reached. New network connectivity diagnostics to help diagnose potential firewall issues. With the CLI (storageos cluster connectivity) or API (GET /v1/diagnostics/network), all required connectivity will be verified. Without a licence, StorageOS has all features enabled but provisioned capacity is now limited to 100GB. Once registered (for free, via the Web UI), capacity increases to 500GB. Improved Increased overall performance by reducing context switches in the IO path, resulting in lower latency for all IO.\n Increased replication performance by optimising the parallel writes to multiple destinations. With \u0026gt;1 replicas this will roughly double replication performance, but it also reduces the overhead of adding additional replicas to a volume.\n Time taken to detect and recover from a node failure reduced from 45-70 seconds to \u0026lt;15 seconds.\n Improved the internal distributed lock mechanism to ensure correct and deterministic behaviour in the majority of failure scenarios.\n Implemented basic check-and-set (CAS) on control plane operations where consistency is required.\n Stop all control plane state evaluations if KV store is unavailable for more than the distributed lock TTL (5 seconds).\n Ensure node does not participate in state evaluations when it has been set as unschedulable with storageos node cordon or storageos node drain.\n Internal library change from Serf to Memberlist. This helps simplify node failure detection.\n Control plane state evaluations are now performed serially with an interval of once per second. This reduces the load on the cluster during bulk or recovery operations. Previously, multiple workers may try to apply the same changes, causing unnessessary validation to take place. A Prometheus histogram has been added to track duration of each state evaluation. Under normal operation it takes 10-50ms.\n Node capacity now includes capacity from all devices made available for use by StorageOS.\n Volume health management and reporting is improved. Health is now defined as:\n healthy: Only if the volume master and requested number of replicas are healthy. syncing: If any replicas are in the process of re-syncing. suspect: The master is healthy, but at least one replica is suspect or degraded. degraded: If the master is suspect/degraded or the master is healthy but at least one replica is dead. offline: If the volume is offline, typically because the master is not available and there were no healthy replicas to failover to. decommissioned: If a volume is being deleted the health will be marked as decommissioned. Prometheus metrics have been overhauled to provide more friendly and useful statistics.\n Internal library change from vue-resource (deprecation notice) to Axios for handling ajax requests in the Web UI.\n Warnings are now logged when volumes can not be provisioned due to licensing constraints.\n Internal logging improvements to ensure that runtime changes to log levels or filters maintain consistency across multiple consumers.\n Diagnostic requests now support JSON requests via the \u0026ldquo;Accepts\u0026rdquo; header, defaulting to the current tar archive response. Internally, collection from multiple nodes is now streamed with a timeout of 10 seconds to prevent an unresponsive node from blocking the response.\n Improved description and flow of cluster diagnostics upload.\n Web UI volume detail page re-designed.\n Web UI theme tweaks: list view, pagination, headers, in-place edit.\n Upgrade version of internal messaging library (nats) and restructure implementation.\n Ensure the API supports label selectors on all object listings and improve internal code consistency and tests.\n Ensure control plane workers and gRPC connection pool are shutdown cleanly prior to shutting down the data plane to ensure operations have completed and to reduce warnings in logs.\n Explicitly shutdown the data plane gRPC handlers in the replication client prior to shutting down the rest of the data plane. This protects against a potential issue that could lead to unclean shutdown.\n The lun ID is now computed dynamically. This allows support for devices across multiple HBAs in the future, which will remove the limitation of 256 volumes on RHEL7/Centos7 clusters and other systems with kernel 3.x.\n SCSI lun support now supports co-existence with other block storage providers\n Bulk volume creation could cause excessive validation warning messages in the logs do to concurrent configuration requests to the data plane. Now, if the volume already exists or a CAS update fails, a conflict error (rather than generic error) is returned to the control plane. This allows the control plane to only log a warning or error message if the intended action failed.\n Ongoing log message readability and tuning of log levels.\n Fixed Filesystems now behave correctly when the underlying data store is out of capacity. Previously the filesystem would appear to hang as it would retry the operation indefinitely. Now, the filesystem will receive a fatal error and will typically become read-only. Listing volumes bypassed policy evaluation so a user in one namespace was able to view volume details in another namespace. Create, update and delete were not affected and required correct authorization. Deletion of encrypted volumes did not work correctly. Product version details were not always displayed correctly. Licence upload could fail with error: licence and actual array UUIDs do not match. Erroneously reported an error when removing data for a volume that was already deleted. Web UI icons no longer load from the Internet and now display in disconnected environments. Web UI capacity now uses gigabytes to be consistent with CLI. API error when running storageos cluster health was suggesting to use storageos cluster health to diagnose the errror. API for retrieving the Cluster ID is no longer restricted to administrators only, but any authenticated user. Rules against namespaces with common prefixes no longer clash. It was possible to create more than one namespace with the same name. Fixed fork/exec /usr/sbin/symmetra: text file busy, caused by the operating system not having closed the file for writing before it is executed. Fixed docker volume rm \u0026lt;volname\u0026gt; returning 0 but not deleting the volume correctly. If a volume has a single replica and the master fails when the replica is in syncing state, the volume is now marked as offline until the master recovers. Previously, the volume remained in syncing state indefinitely. If a volume with no replicas goes offline and then recovers, the volume was not marked as healthy. The CLI was reporting replicas that were not active, creating inaccurate volume counts. On a new volume with no replicas with the master on node1, if the volume was mounted on a node that does not hold the volume master (node2) and then node1 drained, then the new volume created on another node will not be mountable. Prometheus /metrics endpoint always returns text format with correct text encoding, even if binary output was requested. This matches the Prometheus 2.0 policy to deprecate the binary format, and allows the endpoint to work with other collectors that support the text format, e.g. Telegraf. Fixed an unclean shutdown issue in the data plane volume presentation where configuration of a volume could still be attempted while a shutdown was in progress. A lock is now taken to ensure order and to refuse configuration while a shutdown is in progress. This only occured during shutdown and did not affect data consistency. Fixed an unclean shutdown issue in the replication client where references to the client configuration could be cleared while IO was still in progress. A reference counter has been implemented to ensure that all operations have completed prior to the server entry being removed. This only occured during shutdown and did not affect data consistency. Ensure all internal IO acknowledgements are received or a timeout reached prior to shutting down the replication server. This fixes an issue where a shutdown or fatal error could cause the server to crash as it was being shutdown cleanly. This only occured during shutdown and did not affect data consistency. Fixed an issue in the replication client connection handler that in some situations could allow the client to re-use a connection while it was still being prepared for re-use. This could lead to a replication error or volumes stuck in \u0026ldquo;syncing\u0026rdquo; state. No longer logs an error when deleting a volume with no data. 1.0.0-rc4 - Released 2018/07/18 Multiple bug fixes and improvements, and improved shutdown handling.\nNew Available capacity is now used in volume placement decisions. This should allow for a more even distribution of capacity across the cluster, especially in clusters with large volumes. Licenses can now be applied from the Web UI. Improved The shutdown process has been improved within the data plane, speeding up the removal of devices and reducing the risk that the container runtime or orchestrator will forcefully stop StorageOS. IO operations on volumes being deleted now return a fatal error so that the operation is not retried and can fail immediately. During node filtering for volume scheduling decisions, it was possible to return duplicate nodes as candidates for the volume placement. This resulted in slightly unbalanced placement across the cluster. Volume replica removal didn\u0026rsquo;t prioritise syncing volumes over healthy. This meant that a fully-synced replica might be removed instead of one that is not yet synced, in cases where the number of replicas was increased and then immediately reduced. Log verbosity has been reduced at info level. Set LOG_LEVEL=debug for more verbose logging. Fixed UI labels overlapped when window resized in Firefox. You can now provision the licenced amount, rather than up to the licenced amount fixing: \u0026ldquo;cannot provision volume with size 999 GB, currently provisioned: 1 GB, licenced: 1000 GB\u0026rdquo;. 1.0.0-rc3 - Released 2018/07/02 Multiple improvements based on customer feedback.\nNew StorageOS can now be installed alongside other storage products that make use of the Linux SCSI Target driver. The CLI now checks for version compatibility with the API. Improved Volume deletion now uses the scheduler\u0026rsquo;s desired state processing rather than the previous imperative operation. This fixes an issue where deletes could be stuck in pending state if the scheduler loses state (e.g. from a restart) while the operation is in progress. Now the operation is idempotent and will be retried until successful. Volume placement now distributes volumes across nodes more evenly. CSI version 0.3 (latest) is now fully supported. Additionally, improvements to CSI include how the default filesystem is determined, read-only mounts, and better checking for volume capabilities. Internal communication now times out after 5 seconds instead of 60. This allows retry or recovery steps to initiate much quicker than before. This timeout only affects inter-process communication on the same host, not over the network to remote hosts. Added a \u0026ldquo;degraded\u0026rdquo; state to the internal health monitoring. This allows a recovery period before marking a node offline, which then triggers a restart. This improves stability when the KV store (internal or external) is undergoing a leadership change. Minor improvements to the UI notifications and error messages. Upgraded controlplane compiler to Golang 10.0.3 (from 1.9.1). Fixed Online device resizing now works with SCSI devices. Note that resize2fs still needs to be run manually on the filesystem, and we are working on making this step automated. When deleting data from a volume, some metadata was not always being removed. This meant that volumes with frequently changing data could use more capacity than allocated. 1.0.0-rc2 - Released 2018/05/31 Single fix to address provisioning issue in Amazon AWS.\nFixed Increased time to wait for device to appear, causing volume creation to fail. This was encountered on Xen-based RHEL/Centos VMs running in Amazon AWS. 1.0.0-rc1 - Released 2018/05/25 The 1.0 release series is focussed on supporting enterprise workloads, with numerous new features and improvements to performance, stability and maintainability.\nBreaking changes Several internal data structures were changed and are not compatible with previous versions. For this reason upgrades from version 0.10.x and earlier are not supported. The API endpoint /v1/metrics has been replaced with /metrics to conform with Prometheus best practices. New Volume presentation has changed to use the SCSI subsystem via the Linux SCSI Target driver. Previous versions used NBD where available, or FUSE where it wasn\u0026rsquo;t. Using the new volume presentation improves performance on the RHEL platform where NBD is not available. This feature is available for all major distributions and is widely used. For more information, see system configuration Internally, the StorageOS scheduler has switched to using level-based state handling and the gRPC protocol. This allows the scheduler to make assertions about the current state, rather than relying on events that can be missed. The scheduler now behaves in the same way as Kubernetes controllers do; by evaluating the current state, calculating adjustments, sharing desired state and allowing individual components to apply differences. For more information on level triggered logic, see Edge vs Level triggerd logic Encyption at-rest is available on a per-volume level by setting the label storageos.com/encryption=true. In Kubernetes, volume encryption keys are stored in Secrets. Elsewhere, keys are stored in etcd, the key-value store used internally within StorageOS. No other configuration is required. Container Storage Interface (CSI) version 0.2 compatibility. CSI is a specification for storage providers that enables StorageOS to support any orchestrator that supports CSI. Currently this includes Kubernetes and derivatives, Mesos and Docker. Over time CSI will replace the native driver in Kubernetes, though the existing v1 API will remain. Multiple devices on a node can now be used, and StorageOS will shard data across them. Node maintenance command such as storageos node cordon and storageos node drain aid upgrades by live-migrating active volumes prior to node maintenance. Node labels can now be used for volume anti-affinity, allowing StorageOS to make sure a master volume and its replicas are in separate failure domains. Cloud provider failure domains are now auto-set as labels for nodes deployed in Azure, AWS or GCE. Administrators can download or upload to Storageos, support, cluster information and log files. This is currently only available via the Web interfce or the API. Kubernetes mount option support (requires Kubernetes 1.10). Improved Performance has improved significantly throughout and has been thoroughly tested on a variety of workloads. Specific areas include optimisations for larger block sizes and improvements to the caching engine. Volumes will go read-only shortly before the underlying device runs out of space. This allows the filesystem to handle errors gracefully and can protect against corruption. Pools have been overhauled to be dynamic, making use of label selectors. Increased number of volumes per node. Invalid labels now generate a validation error. storageos.com/ namespaced labels must validate against known labels. Fixed Mapping of compression and throttle labels. Remove hard connection limit on replication services. docker ps shows unhealthy when everything is fine. Throttle impact was too small to be noticed. Volume size \u0026lsquo;0\u0026rsquo; produced 10GB volume. Re-adding user to a group creates a duplicate entry. Node and pool capacity stats were sometimes wrong. Previous Releases Upgrades Due to breaking changes between 0.9.x and 0.10.x upgrading is not possible. Instead we recommend that you provision a new cluster and migrate data.\nPlease update to the latest CLI when installing 0.10.x.\n0.8.x -\u0026gt; 0.9.x Start-up scripts should be updated to use the new cluster discovery syntax\nDo not mix a cluster with 0.8.x and 0.9.x versions as port numbers have changed. This may cause cluster instability while nodes are being upgraded.\n0.7.x -\u0026gt; 0.8.x Due to the nature the KV Store change there is no upgrade method from 0.7.x to 0.8.x+. Our recommendation is to create a new cluster, paying attention to the new parameters (CLUSTER_ID and INITIAL_CLUSTER). Note that CLUSTER_ID and INITIAL_CLUSTER have been replaced by JOIN in 0.9.x onwards.\n0.10.0 - Released 2018/02/28 The 0.10.0 version focusses on stability and usability as we get closer to GA, but also adds a number of new features (UI, Prometheus metrics, log streaming).\nBreaking changes We took the decision to bundle as many known breaking changes into this release in order to reduce pain later.\n The API endpoint /v1/controllers has been replaced with /v1/nodes. This removes the concept of storage controllers being different objects from storage clients. Now both are simply nodes, with the storageos.com/deployment label used to configure the deployment type. Deployment types can be computeonly and mixed are supported, with mixed being the default.\nOlder versions of the CLI should be mostly backwards compatible but some features such as node and pool capacity will not work.\n The label format for configuring StorageOS features has changed from storageos.feature.xyz to storageos.com/xyz. This format is more familiar to Kubernetes users. Using the old format will still work until 0.11 when conversion will be removed. In 0.10 a deprecation notice will be issued.\n The environment variable used to add labels to nodes at startup time has changed from LABEL_ADD to LABELS. Multiple labels can be comma-separated: LABELS=storageos.com/deployment=computeonly,region=us-west-1\n New There is now a Web interface, available on all cluster nodes on the API port (5705). To access, point your web browser to http://ADVERTISE_IP:5705. where ADVERTISE_IP is the public ip address of a StorageOS node. Unless you have changed the default credentials, login with user storageos, password storageos. See: https://docs.storageos.com/v1.x/docs/reference/gui/\n Prometheus stats are exported on each cluster node at http://ADVERTISE_IP:5705/v1/metrics. Prometheus 2.x is required.\n The CLI can now stream realtime logs from the active StorageOS cluster with storageos logs -f. Logs are aggregated from all cluster nodes over a single connection to the API. Future releases will add support for filtering and controlling the log level. See: https://docs.storageos.com/v1.x/docs/reference/cli/logs/\n The location of the StorageOS volumes can now be configured on a per-node basis by setting the DEVICE_DIR environment variable on the StorageOS node container.\nThis is especially useful when running Kubernetes in a container and you are unable or unwilling to share the default /var/lib/storageos/volumes location into the kubelet container. Instead, share the kubelet plugin directory into the StorageOS container, and set DEVICE_DIR to use it. Hovever, Kubernetes 1.10 will be required in order for kubelet to use a directory other than /var/lib/storageos/volumes.\nFor example:\nJOIN=$(storageos cluster create) docker run --rm --name storageos \\ -e HOSTNAME \\ -e DEVICE_DIR=/var/lib/kubelet/plugins/kubernetes.io~storageos/devices \\ -e JOIN=${JOIN} \\ --pid host \\ --privileged --cap-add SYS_ADMIN \\ -v /var/lib/kubelet/plugins/kubernetes.io~storageos:/var/lib/kubelet/plugins/kubernetes.io~storageos:rshared \\ --device /dev/fuse \\ --net host \\ storageos server Improved soft volume failure mode will now tolerate the replica being offline (for example during a node reboot), if there is only one replica configured. To ensure there are always two copies of the data, use hard mode with a single replica, or use two replicas with soft mode. Ensure volumes can only be mounted with the correct underlying filesystem. You will now get a validation error if you try to mount an ext3 formatted filesystem as ext4. ext4 is the default if not specified when mounting an unformatted volume. Fixed Ensure cache is invalidated after mkfs. Fixes mount error 32 that could occur with Kubernetes or OpenShift on Centos/RHEL. Memory leak in replication client led to excessive use over time. Standardised all paths to 256 byte length to support user-configurable DEVICE_DIR. Fixed clean shutdown issue where filesystem could start new threads while shutting down. This was only observed in stress tests. Shutdown signal handling improved, fixing Transport endpoint is not connected on Centos / RHEL in certain restart situations. 0.9.2 The 0.9.2 fixes a memory leak, so upgrading is recommended. It also reduces the time taken to delete volumes.\nMore work has gone into logging, with API endpoints to control verbosity and log filtering of running nodes individually or cluster-wide. Version 0.9.4 of the CLI provides the storageos logs subcommand to control settings remotely.\nNew API endpoints for logging configuration Improved Remove cgo dependencies from controlplane Better error message when the KV store is unavailable Reduced memory footprint of controlplane by 67%. Fixed Memory leak introduced in 0.9.0 fixed 0.9.1 The 0.9.1 release includes a fix for the mount deadline exceeded error, where a volume mount fails after a long delay.\nThe logging susbsystem has also been overhauled to reduce noise and to allow filtering messages by log level and component. We expect to build on this in upcoming releases to further improve diagnostics.\nNew Failure modes can be specified to control behaviour in specific failure scenarios. Full documentation to be added to the docs site soon.\n hard will enforce the desired number of replicas and take a volume offline if the replica count is not met. In practice, the volume should only go offline if there isn\u0026rsquo;t a suitable node to place a new replica on. For example, if you have configured 2 replicas in a 3 node cluster and a node goes offline. alwayson will optimise for availability and will keep a volume online even if all replicas have failed. soft is set by default, and will only take a volume offline if the number of replicas falls below the Failure tolerance (default: number of replicas - 1). This will allow a volume with 2 replicas in a 3 node cluster to remain active while a node reboots. You can select the failure mode through the labels:\nstorageos volume create --label storageos.com/replicas=2 --label storageos.com/failure.mode=alwayson volume-name Log filtering allows finer-grained control of log messages so that debug-level messages can be enabled only on some components. Filtering is controlled by setting LOG_FILTER=cat1=level1,catN=LevelN. For example, to enable debugging on only the etcd category:\n Set LOG_LEVEL=debug to enable debug logs. This must be set to the lowest level requested in the filter. Set LOG_FILTER=cp=info,dp=info,etcd=debug to set controlplane and dataplane logs back to info level, then etcd to debug. Enabled profiling of the controlplane when DEBUG=true is set. The endpoint is availble at http://localhost:5705/debug/pprof.\n Debug endpoints to monitor and control KV store leadership (DEBUG=true must be set).\n HTTP GET on http://localhost:5705/debug/leader returns true/false if the node is the cluster leader. HTTP PUT on http://localhost:5705/debug/leader/resign will cause leader to resign, triggering a new election. HTTP PUT on http://localhost:5705/debug/leader/run will cause node to run for cluster leadership. These actions are used primarily for automated testing where we introduce instability into the cluster to ensure there is no service disruption.\n Improved More instructive log messages if unable to create filesystem. Improve check for whether a device is using NBD for presentation. Log whenever a dataplane process restarts. Updated etcd libraries to v3.3.0-rc.0. Etcd tuning (TickMs = 200, ElectionMs = 5000) Updated other dependencies to latest stable releases. Dataplane, etcd, gRPC and NATS now use same log format as the controlplane. More user-friendly error message when ADVERTISE_IP is invalid. Fixed Fixed a bug that caused mounts to fail with Failed to mount: exit status 32. This was caused by the volume being marked as ready before the device was fully initialised, and the mount starting before the initialization completed. Volume names are no longer lowercased and keep the requested case. This fixes an issue with Docker EE with mixed-case volume names. Volume delete on non-existent volume now returns HTTP 404 instead of 500. Do not reset node health if internal healthcheck returns invalid response. Instead, retry and wait for valid response message. 0.9.0 - Released 2017/11/17 This release focusses on usability and backend improvements. It builds on the embedded KV store from 0.8.x and improves the bootstrap process.\nEnhanced stress testing and performance benchmarking have also led to a number of improvements and better failure handling and recovery.\nDeprecation notices While in Beta there may be changes that break backwards compatibility. GA releases will strive to preserve compatibility between versions.\n The INITIAL_CLUSTER and CLUSTER_ID environment variables have been replaced with JOIN and they should no longer be used to bootstrap a new cluster. JOIN provides a more flexible mechanism for creating new clusters and expanding existing clusters. Errors or deprecation notices in the log at startup will help verify that the correct environment variables are used. The API object format for users has changed to make it consistent with other objects. Version 0.9.0+ of the StorageOS CLI must be used to manage users on a version 0.9.x cluster. New Cluster can now start as soon as the first node starts, and any number of nodes can join the cluster. The JOIN environment variable replaces CLUSTER_ID and INITIAL_CLUSTER and is more flexible, allowing administrators to combine methods for bootstrapping and discovering clusters. See https://docs.storageos.com/docs/install/reference/clusterdiscovery Online replica sync for new and replacement replicas. This allows for writes to complete on the required number of replicas immediately instead of waiting for a new replica or a replacement replica to be fully synced. Network ports now use a contiguous range (5701-5711/tcp, 5711/udp) State-tracker added to frontend filesystem. This allows greater flexibilty for backend failure recovery. Prometheus endpoint for exposing internal metrics. This version includes API usage metrics, but will be extended in an upcoming release to include detailed volume, pool and node metrics. Reports are sent to Sentry if processes within the StorageOS container are stopped unexpectantly. These reports help us improve the stability of the product, but they can be disabled by setting the setting the DISABLE_TELEMETRY environment variable to true. Improved Enforced ordering for some types of read/write operations to ensure deterministic behaviour Ensured all components shutdown cleanly in normal situations and recover properly when clean shutdown not done. Added gRPC to dataplane for synchronous internal messaging User API endpoint now has a format consistent with other endpoints. /users now returns a list of users, and a POST/PUT to /users/{id} accepts a user object. Document minimum server requirements Document node selector for volume create operations Document quality of service (QoS) and protection from noisy neighbors Document volume placement hints Separate usage metric endpoints for internal (Dev/QA) and release (Public) Health endpoints show changedAt and updatedAt to help detect flapping services Unmount volumes through CLI and API Removed false-postive errors and warning from logs Improve signal handling LOG_LEVEL passed to dataplane components Fixed Clear mount lock after ungraceful node shutdown etcd error: mvcc: database space exceeded NBD cleanup during ungraceful node shutdown No longer possible to have two default pools storageos cluster health did not return useful information while the cluster was bootstrapping Node master and replica statistics were not getting updated after a node failure Known issues It is currently not possible for a node to leave the cluster completely. If the StorageOS container is stopped and/or removed from a node then the node will be detected as failed and it will be marked offline, but there is no way to remove the node from the list. storageos node rm will be added before GA along with storageos node cordon to disable scheduling new volumes on the node, and storageos node drain to move volumes to other nodes prior to maintenance or removing the node from the cluster. storageos volume mount \u0026lt;vol\u0026gt; \u0026lt;mountpoint\u0026gt; does not work on Managed Plugin installs. Volumes mount into containers correctly using Docker. Docker can only access volumes created in the default namespace. Clients mounting volumes from RHEL7/CentOS 7 will experience degraded performance due to the absence of the NBD kernel module on those platforms. 0.8.1 Improved NBD device numbers now start at 1 instead of 0 to defend against default values 0.8.0 - Released 2017/08/29 This is our first feature release since launching our public beta, and it focusses on feedback from users. As always, please let us know how you are using StorageOS, what problems it is solving for you and how it can improve. Join our Slack channel!\nNew Embedded KV Store based on etcd to further simplify deployment and ongoing cluster management. Support for external Consul KV clusters has been deprecated, though external etcd clusters are now possible, but not yet documented and supported. Cluster discovery service to help bootstrap clusters. Allocate a new cluster with storageos cluster create and pass the cluster identifier to each StorageOS node in the CLUSTER_ID environment variable to allow nodes to discover the cluster without specifying hostnames or ip addresses. Access control policies can restrict access to volumes and rules created within a namespace. User and group management allows multiple users to be created and then used to apply access policy by group membership or named account. Anonymized usage metrics are collected and sent to StorageOS to help us better understand usage patterns so we can focus our efforts accordingly. Metrics can be disabled by setting the DISABLE_TELEMETRY environment variable to true. Location-based scheduling allows administrators to specify scheduling constraints on volumes at creation time. This provides a simple mechanism to influence data placement. (e.g. The volume\u0026rsquo;s data may only be stored on nodes which have their environment label set to production) Cluster health reporting with CLI (storageos cluster health) Improved Graceful behaviour when communication blocked by firewalls Docker integration now supports ext2/3/4, btrfs and xfs Environment variable validation Selectors for rules Use default namespace when not specified Internal volume performance counters API can report health while waiting for cluster to initialize Fixed Better behaviour when communication blocked by firewalls Ensure namespaces are unique when creating Volume create should give error when size=0 CLI/API filters not working as expected Can\u0026rsquo;t edit a pool with the CLI Excessive logging on network timeouts Known issues Once a 0.8.0 cluster has been established, it is currently not possible to add or remove members. We expect this functionality to come in 0.8.1, and welcome feedback on how this should behave. storageos volume mount \u0026lt;vol\u0026gt; \u0026lt;mountpoint\u0026gt; does not work on Managed Plugin installs. Volumes mount into containers correctly using Docker. Docker can only access volumes created in the default namespace. Clients mounting volumes from RHEL7/CentOS 7 will experience degraded performance due to the absence of the NBD kernel module on those platforms. 0.7.10 This release continues our focus on stability and better test coverage across a growing suite of platforms and usage scenarios.\nNew Keepalives for replication though firewalls Improved Graceful shutdown internal cleanup User authentication internals made extensible to other auth providers Internal changes to allow different log levels per module Fixed Removed log entries where events are raised and logged again Consolidated log messages when KV store is not available 0.7.9 New Maintenance mode to allow partial startup when the KV store is not ready Improved Better write parallelization performance Fixed Issue with volume deletion 0.7.8 New Node statistics reporting Compression (in-transit and at-rest) enabled by default Improved Refactored startup process Better volume distribution across cluster Relaxed volume naming requirements to support Docker dynamic volumes More instrumentation counters for volume stats Improved data plane error handling and retry logic Replication enhancements Ongoing log-level tuning to reduce noise Fixed Issue removing metadata on volume destroy Improved compression performance Default volume size now consistent across CLI/API/Docker/Kubernetes KV Store documentation (thanks @wcgcoder) 0.7.7 Fix issue with replication after node failure 0.7.6 Log formatting improvements Do not mark a recovered controller available immediately, wait to detect flapping Mount lock improvements Docker should use ext4 as default filesystem 0.7.5 Remove bash 0.7.4 Log formatting improvements Add support for ext2, ext3, xfs and btrfs filesystem types Don\u0026rsquo;t allow pool delete with active volumes 0.7.3 and earlier Unreleased private betas ","excerpt":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than …","ref":"/v1.x/docs/reference/release_notes/","title":"Release notes"},{"body":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous replication ensures strong consistency for applications such as databases and Elasticsearch, incurring one network round trip on writes.\nThe basic model for StorageOS replication is of a master volume with distributed replicas. Each volume can be replicated between 0 and 5 times, which are provisioned to 0 to 5 nodes, up to the number of remaining nodes in the cluster.\nIn this diagram, the master volume D was created on node 1, and two replicas, D2 and D3 on nodes 3 and 5.\nWrites that come into D (step 1) are written in parallel to D2 and D3 (step 2). When both replicas and the master acknowledge that the data has been written (step 3), the write operation return successfully to the application (step 4).\nFor most applications, one replica is sufficient (storageos.com/replicas=1).\nAll replication traffic on the wire is compressed using the lz4 algorithm, then streamed over tcp/ip to target port tcp/5703.\nIf the master volume is lost, a random replica is promoted to master (D2 or D3 above) and a new replica is created and synced on an available node (Node 2 or 4). This is transparent to the application and does not cause downtime.\nIf a replica volume is lost and there are enough remaining nodes, a new replica is created and synced on an available node. While a new replica is created and being synced, the volume\u0026rsquo;s health will be marked as degraded and the failure mode will determine whether the volume can be used during recovery:\n Failure mode On loss of replica Example Hard If the number of replicas falls below that requested, mark the volume as unavailable. Any reads or writes will fail. storageos.com/failure.mode=hard Always On As long as any copy of the volume is available, the volume will be usable. storageos.com/failure.mode=alwayson Soft Default mode. Specify how many failed replicas to tolerate, defaulting to 1 if there is only 1 replica, or replicas - 1 if there is more than 1 replica. storageos.com/failure.mode=soft storageos.com/failure.tolerance=2 While replica count and replication failure mode are controllable on a per-volume basis, some environments may prefer to set appropriate policies to ensure a desired level of data protection. This can be accomplished on a per-namespace basis using Rules.\n","excerpt":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous …","ref":"/v1.x/docs/concepts/replication/","title":"Replication"},{"body":"Rules are used for managing data policy and placement using StorageOS features such as replication, QoS and compression. They provide a way for operators to define default charactistics (such as number of replicas).\nRules are defined using labels and selectors. When a volume is created, the rules are evaluated to determine whether to apply the action.\nAn example business requirement might be that all production volumes are replicated twice. This would be defined with a selector env==prod, and the action would be to add the label storageos.com/replicas=2.\nRules can be created with the CLI or Web UI. See Rules for details.\n","excerpt":"Rules are used for managing data policy and placement using StorageOS features such as replication, …","ref":"/v1.x/docs/concepts/rules/","title":"Rules"},{"body":"Required parameters The minimum parameters to create a rule are --selector and --label.\nCreate a rule that configures 2 replicas for volumes with the label env=prod:\n$ storageos rule create --namespace default --selector \u0026#39;env==prod\u0026#39; --label storageos.com/replicas=2 replicator default/replicator Optional parameters Rules also accept the optional parameters --action and --weight.\n-a, --action string Rule action (add|remove) (default \u0026quot;add\u0026quot;)\nWhere multiple rules apply to the same label, a weight is used to determine the order of evaluation. Rules are evaluated starting at the lowest weight.\n-w, --weight int Rule weight determines processing order, any integer\nUsing rules To create a rule that configures 2 replicas for volumes with the label env=prod:\n$ storageos rule create --namespace default --selector \u0026#39;env==prod\u0026#39; --action add --label storageos.com/replicas=2 replicator default/replicator View rules:\n$ storageos rule ls NAMESPACE/NAME SELECTOR ACTION LABELS default/dev-marker !storageos.com/replicas add env=dev default/prod-marker storageos.com/replicas\u0026gt;1 add env=prod default/replicator env==prod add storageos.com/replicas=2 default/uat-marker storageos.com/replicas\u0026lt;2 add env=uat Inspect a rule:\n$ storageos rule inspect default/replicator [ { \u0026quot;id\u0026quot;: \u0026quot;9db3252a-bd14-885b-0d0a-b0da1dd2d4a1\u0026quot;, \u0026quot;name\u0026quot;: \u0026quot;replicator\u0026quot;, \u0026quot;namespace\u0026quot;: \u0026quot;default\u0026quot;, \u0026quot;description\u0026quot;: \u0026quot;\u0026quot;, \u0026quot;active\u0026quot;: true, \u0026quot;weight\u0026quot;: 5, \u0026quot;action\u0026quot;: \u0026quot;add\u0026quot;, \u0026quot;selector\u0026quot;: \u0026quot;env==prod\u0026quot;, \u0026quot;labels\u0026quot;: { \u0026quot;storageos.com/replicas\u0026quot;: \u0026quot;2\u0026quot; } } ] Then, create a volume:\nstorageos volume create -n default -s 1 --label env=prod prodVolume Once it\u0026rsquo;s created, inspect it:\nstorageos volume inspect default/prodvolume You should see that it has two replicas provisioned and additional labels attached:\n\u0026#34;labels\u0026#34;: { \u0026#34;env\u0026#34;: \u0026#34;prod\u0026#34;, \u0026#34;storageos.driver\u0026#34;: \u0026#34;filesystem\u0026#34;, \u0026#34;storageos.com/replicas\u0026#34;: \u0026#34;2\u0026#34; }, Delete a rule:\n$ storageos rule rm default/replicator default/replicator Using advanced selectors Let\u0026rsquo;s create several rules that instead of adding storageos.com/replicas feature label it would read it\u0026rsquo;s value and based on it would label volumes with dev/uat/prod env values.\nFirst, create a rule to label dev environments:\nstorageos rule create --namespace default --selector \u0026#39;!storageos.com/replicas\u0026#39; --action add --label env=dev dev-marker This rule will be matching volumes that do not have (!) label storageos.com/replicas and will add env=dev label. Now, create a second rule to select volumes that have 1 replica (\u0026lt; 2) and add uat env label to them:\nstorageos rule create --namespace default --selector \u0026#39;storageos.com/replicas\u0026lt;2\u0026#39; --action add --label env=uat uat-marker Create new volume with 1 replica:\nstorageos volume create --namespace default --label storageos.com/replicas=1 uat-volume Inspect it:\nstorageos volume inspect default/uat-volume Labels should look like:\n\u0026#34;labels\u0026#34;: { \u0026#34;env\u0026#34;: \u0026#34;uat\u0026#34;, \u0026#34;storageos.driver\u0026#34;: \u0026#34;filesystem\u0026#34;, \u0026#34;storageos.com/replicas\u0026#34;: \u0026#34;1\u0026#34; }, Finally, create a rule that will mark volumes as prod if they have 2 or more (gt) configured replicas:\nstorageos rule create --namespace default --selector \u0026#39;storageos.com/replicas\u0026gt;1\u0026#39; --label env=prod prod-marker default/prod-marker Volumes created with 2 or more replicas should get env=prod label.\n","excerpt":"Required parameters The minimum parameters to create a rule are --selector and --label.\nCreate a …","ref":"/v1.x/docs/operations/rules/","title":"Rules"},{"body":"$ storageos rule Usage:\tstorageos rule COMMAND Manage rules Options: --help Print usage Commands: create Creates a rule. To create a rule that configures 2 replicas for volumes with the label env=prod, run: storageos rule create --namespace default --selector env==prod --action add --label storageos.com/replicas=2 replicator inspect Display detailed information on one or more rules ls List rules rm Remove one or more rules update Update a rule Run \u0026#39;storageos rule COMMAND --help\u0026#39; for more information on a command. storageos rule create To create a rule that configures 2 replicas for volumes with the label env=prod:\n$ storageos rule create --namespace default --selector \u0026#39;env==prod\u0026#39; --action add --label storageos.com/replicas=2 replicator default/replicator storageos rule inspect To inspect a rule:\n$ storageos rule inspect default/replicator [ { \u0026quot;id\u0026quot;: \u0026quot;9db3252a-bd14-885b-0d0a-b0da1dd2d4a1\u0026quot;, \u0026quot;name\u0026quot;: \u0026quot;replicator\u0026quot;, \u0026quot;namespace\u0026quot;: \u0026quot;default\u0026quot;, \u0026quot;description\u0026quot;: \u0026quot;\u0026quot;, \u0026quot;active\u0026quot;: true, \u0026quot;weight\u0026quot;: 5, \u0026quot;action\u0026quot;: \u0026quot;add\u0026quot;, \u0026quot;selector\u0026quot;: \u0026quot;env==prod\u0026quot;, \u0026quot;labels\u0026quot;: { \u0026quot;storageos.com/replicas\u0026quot;: \u0026quot;2\u0026quot; } } ] storageos rule ls To list all rules:\n$ storageos rule ls NAMESPACE/NAME SELECTOR ACTION LABELS default/dev-marker !storageos.com/replicas add env=dev default/prod-marker storageos.com/replicas\u0026gt;1 add env=prod default/replicator env==prod add storageos.com/replicas=2 default/uat-marker storageos.com/replicas\u0026lt;2 add env=uat storageos rule rm To delete a rule:\n$ storageos rule rm default/replicator default/replicator ","excerpt":"$ storageos rule Usage:\tstorageos rule COMMAND Manage rules Options: --help Print usage Commands: …","ref":"/v1.x/docs/reference/cli/rule/","title":"Rules"},{"body":"StorageOS has the capacity to influence Kubernetes Pod placement decisions to ensure that Pods are scheduled on the same nodes as their data. This functionality is known as Pod Locality.\nStorageOS grants access to data by presenting, on local or remote nodes, the devices used in a Pod\u0026rsquo;s VolumeMounts. However, it is often the case that it is required or preferred to place the Pod on the node where the StorageOS Primary Volume is located, because IO operations are fastest as a result of minimized network traffic and associated latency. Read operations are served locally and writes require fewer round trips to the replicas of the volume.\nStorageOS automatically enables the use of a custom scheduler for any Pod using StorageOS Volumes. Checkout the Admission Controller reference for more information.\nStorageos Kubernetes Scheduler StorageOS achieves Pod locality by implementing a Kubernetes scheduler extender. The Kubernetes standard scheduler interacts with the StorageOS scheduler when placement decisions need to be made.\nThe Kubernetes standard scheduler selects a set of nodes for a placement decision based on nodeSelectors, affinity rules, etc. This list of nodes is sent to the StorageOS scheduler which sends back the target node where the Pod shall be placed.\nThe StorageOS scheduler logic is provided by a Pod in the Namespace where StorageOS Pods are running.\nScheduling process When a Pod needs to be scheduled, the scheduler collects information about all available nodes and the requirements of the Pod. The collected data is then passed through the Filter phase, during which the scheduler predicates are applied to the node data to decide if the given nodes are compatible with the Pod requirements. The result of the filter consists of a list of nodes that are compatible for the given Pod and a list of nodes that aren\u0026rsquo;t compatible.\nThe list of compatible nodes is then passed to the Prioritize phase, in which the nodes are scored based on attributes such as the state. The result of the Prioritize phase is a list of nodes with their respective scores. The more favorable nodes get higher scores than less favorable nodes. The list is then used by the scheduler to decide the final node to schedule the Pod on.\nOnce a node has been selected, the third phase, Bind, handles the binding of the Pod to the Kubernetes apiserver. Once bound, the kubelet on the node provisions the Pod.\nThe StorageOS scheduler implement Filter and Prioritization phases and leaves binding to the default Kubernetes scheduler.\nAvailable +------------------+ +------------------+ NodeList \u0026amp; Pod | | Filtered NodeList | | Scored Information | | \u0026amp; Pod Information | | NodeList +--------------------\u0026gt;+ Filter +--------------------\u0026gt;+ Prioritize |---------------\u0026gt; | (Predicates) | | (Priorities) | | | | | +------------------+ +------------------+ Scheduling Rules The StorageOS scheduler filters nodes ensuring that the remaining subset fulfill the following prerequisites:\n The node is running StorageOS The node is healthy The node is not StorageOS Cordoned The node is not in a StorageOS Drained state The node is not a StorageOS compute-only node The scoring protocol once the nodes are filtered is as follows:\n Node with master volume - 15 points Node with replica volume - 10 points Node with no master or replica volume - 5 points Node with unhealthy volume or unsynced replica - 1 point Locality modes There are two modes available to set pod locality for StorageOS Volumes.\nPreferred The Pod SHOULD be placed alongside its data, if possible. Otherwise, it will be placed alongside volume replicas. If neither scenario is possible, the Pod will start on another node and StorageOS will grant access to the data over the network.\nPreferred mode is the default behaviour when using the StorageOS scheduler.\nStrict The Pod MUST be placed alongside its data, i.e. on a node with the master volume or a replica. If that is not possible, the Pod will remain in pending state until the premise can be fulfilled.\nThe aim of strict mode is to provide the user with the capability to guarantee best performance for applications. Some applications are required to give a certain level of performance, and for such applications strict co-location of application and data is essential.\nFor instance, when running Kafka Pods under heavy load, it may be better to avoid scheduling a Pod using a remote volume rather than have clients direct traffic at a cluster member which exhibits degraded performance.\nEnable preferred/strict modes by setting labels for your Pods. Checkout the examples section for full details.\n Locality Mode Label Description Preferred (Default) storageos.com/locality: preferred Best effort placement of the Pod along its StorageOS Volume Strict storageos.com/locality: strict Enforce placement of the Pod alongside the Primary Volume or do not schedule ","excerpt":"StorageOS has the capacity to influence Kubernetes Pod placement decisions to ensure that Pods are …","ref":"/v1.x/docs/reference/scheduler/","title":"StorageOS Scheduler"},{"body":"","excerpt":"","ref":"/v1.x/search/","title":"Search Results"},{"body":" Shared filesystems in the 1.5.4 release are currently in Technology Preview. This experimental feature is not yet intended for production use.\n Shared Filesystem support allows volumes to be mounted for read \u0026amp; write access by multiple containers simultaneously, even from different nodes. In Kubernetes, shared filesystems are referred to as ReadWriteMany or RWX volumes.\nFollow the Operations page to learn how to deploy and use ReadWriteMany PersistentVolumeClaims (PVCs).\nArchitecture The default StorageOS StorageClass can create volumes with either ReadWriteOnce (RWO) or ReadWriteMany (RWX) AccessModes.\nWhen requesting a RWX PVC, StorageOS will provision a standard RWO volume and mount the volume into an NFS server instance dedicated to this volume.\nThe NFS server is provisioned as a StatefulSet, which is created by the StorageOS Cluster Operator using an NFSServer Custom Resource Definition. The configuration of the NFS server is stored in a ConfigMap. The ConfigMap defines the export location matching the RWO volume that StorageOS dynamically provisioned to back the NFS server.\nSeparate Services are defined for the NFS traffic and for HTTP-based health and metrics traffic.\nThe NFS traffic Service is on port 2049 and uses TCP only to allow simple exposure to clients outside the Kubernetes cluster. Clients must be capable of NFS version 4.2 which has numerous (primarily performance) advantages over previous revisions.\nThe StorageOS NFS server exposes a health endpoint on http://\u0026lt;PodIP\u0026gt;:80/healthz.\n HTTP 200/OK will be returned when the server is operational and sending heartbeat messages. HTTP 503/Service Unavailable will be returned if the server hasn\u0026rsquo;t sent a heartbeat message within 10 seconds. Prometheus metrics for the NFS server, clients and IO activity are available on http://\u0026lt;PodIP\u0026gt;:80/metrics.\nDetails of the StorageOS NFS Server implementation are available on Github.\n The NFS server Pod and the RWO StorageOS volume are placed in the same Namespace as the RWX PVC.\n The NFS server StatefulSet is named after the RWO Persistent Volume. Therefore the NFS Pod will be named in the form of pvc-${UID}-0.\nOnce the NFS server is healthy and sharing the RWO volume\u0026rsquo;s filesystem, the RWX volume is available for use. It takes slightly longer to provision a RWX volume for the first time as the NFS server image has to be pulled.\nThe StorageOS RWO backing volume will be updated with labels:\n storageos.com/nfs.server: NFS server endpoint. storageos.com/nfs.share: NFS share path. The RWX Volume is a Kubernetes construct backed by StorageOS Volumes, therefore the StorageOS API will not report on shared volumes.\n Labels StorageOS uses labels to apply behaviour regarding the Volumes. Feature labels can be passed to the RWX PVC to ensure that the underlying StorageOS volume backing the NFS server implements those features\nFor instance, to enable replication, set the label storageos.com/replicas: 1 in the RWX PVC metadata.\nFencing StorageOS implements fencing for StatefulSet based pods. When replication has been enabled, the NFS server will use the fencing feature to ensure rapid failover when the node fails.\nCSI Shared Filesystems / RWX volumes are only supported by StorageOS when using CSI (Container Storage Interface).\n","excerpt":"Shared filesystems in the 1.5.4 release are currently in Technology Preview. This experimental …","ref":"/v1.x/docs/concepts/sharedfs/","title":"Shared Filesystem"},{"body":" Shared filesystems in the 1.5.4 release are currently in Technology Preview, and are only available using the CSI driver. This experimental feature is not yet intended for production use.\n Shared Filesystem support allows volumes to be mounted for read \u0026amp; write access by multiple containers simultaneously, even from different nodes. In Kubernetes, shared filesystems are referred to as ReadWriteMany or RWX volumes.\nSee the architecture for more information about the Shared Filesystem components.\nHow to provision ReadWriteMany Volumes RWX Volumes are dynamically provisioned by StorageOS when using a StorageOS Kubernetes StorageClass. To create an RWX volume set the AccessMode for the PVC to ReadWriteMany.\nAn example of a RWX PersistentVolumeClaim is shown below:\nkind: PersistentVolumeClaim apiVersion: v1 metadata: name: shared-vol labels: storageos.com/replicas: \u0026#34;1\u0026#34; # Enable features using Labels # You can enable replication in the # StorageClass params to set defaults spec: storageClassName: \u0026#34;fast\u0026#34; # StorageOS StorageClass accessModes: - ReadWriteMany # AccessMode that triggers creation of an NFS based StorageOS Volume resources: requests: storage: 20Gi The following Deployment example shows how multiple Nginx Pods can mount the same shared filesystem PVC.\napiVersion: apps/v1 kind: Deployment metadata: name: nginx labels: app: nginx spec: replicas: 3 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx:1.7.9 ports: - containerPort: 80 volumeMounts: - name: docroot mountPath: /usr/share/nginx/html volumes: - name: docroot persistentVolumeClaim: claimName: shared-vol Only installations of StorageOS using CSI support RWX Volumes.\n Volume Deletion Shared volumes can be deleted through standard Kubernetes PVC deletion. When a RWX PVC is deleted, StorageOS acts differently according to the ReclaimPolicy of the PVC. For more information about the ReclaimPolicy field see the Kubernetes Documentation.\nReclaim Policy The default reclaim policy for a PVC, PV pair is defined in the StorageClass. All PVs created by a StorageClass inherit the reclaimPolicy set in the StorageClass.\napiVersion: storage.k8s.io/v1 kind: StorageClass metadata: labels: app: storageos name: storageos parameters: csi.storage.k8s.io/fstype: ext4 pool: default provisioner: storageos reclaimPolicy: Delete # \u0026lt;----- Default reclaim policy Delete (default)\nIf the RWX PVC is deleted then the RWX PV is deleted along with the NFS server StatefulSet and its underlying RWO volume. As such data written to the RWX volume is non-recoverable after deletion.\n Retain\nThe RWX PVC is deleted but the PV remains available. The NFS server StatefulSet remains untouched along with its associated RWO PVC. Hence, it is possible to reuse the volume and have access to data written to the RWX volume after the RWX PVC has been deleted.\nIn order to reuse the NFS server to serve a different RWX PVC, it is is necessary to \u0026ldquo;unlock\u0026rdquo; the PV, which only Kubernetes administrators with privileges can do. The PV is \u0026ldquo;unlocked\u0026rdquo; by deleteing the spec.claimRef field from the PV.\nFor instance given a RWX PVC,pvc-1 was bound to the PV pv-1, before pvc-1 was deleted. Then in order to reuse pv-1 edit the PV, i.e kubectl edit pv pv-1 and delete the spec.claimRef attribute.\nDeletion of the spec.claimRef field makes the PV available for any new PVC whose requirements are met by the PV. Therefore creating a new RWX volume with the previously used; StorageClass, capacity, access type and filesystem will make the PVC bind to the previously used RWX PV, even if the new PVC is provisioned in a different namespace.\n ","excerpt":"Shared filesystems in the 1.5.4 release are currently in Technology Preview, and are only available …","ref":"/v1.x/docs/operations/sharedfs/","title":"Shared Filesystem"},{"body":"","excerpt":"","ref":"/v1.x/","title":"StorageOS"},{"body":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for two different purposes.\nTelemetry Telemetry is made up of the DNS query and crash reports sent to sentry.io once per day. Error reporting is the sentry.io crash dump reporting. sentry.io StorageOS sends crash reports and information about the StorageOS cluster to sentry.io. This information helps our developers monitor and fix crashes. Information is sent to sentry.io once per day or when a process inside the StorageOS container crashes.\n The once per day report includes the cluster version, node counts and license information. The crash report contains the signal that triggered the shutdown (e.g. SIGSEGV), the exit code and whether or not the crash generated a core dump. All StorageOS clusters with a routable connection to the internet will send crash reports to sentry.io over tcp/443. StorageOS respects environment variables that ProxyFromEnvironment uses.\nAn exhaustive list of information included in the once per day report is below:\n API version Cluster ID CPU architechture Go version Healthy volume count Logging level Logging user License type Namespace count Node count Pool count OS type Rule count Server name Sentry version StorageOS build information (CI pipeline reference) StorageOS version Suspect volume count Total volume size Volume count Volume degraded count Volume offline count Volume syncing count An exhaustive list of information included in the crash report is below:\n API version Cluster ID CPU architechture Crashed component name Error level Error message Exceptions Exit code Datetime of crash Go version Kernel signal OS type License type Logging user Server name Stacktrace StorageOS build information (CI pipeline reference) StorageOS version Whether a core dump occured DNS Query StorageOS will also send anonymized node ids, cluster id and StorageOS version information to StorageOS using a DNS query. The information that we send in the query is encoded as well as being anonymized. This query allows us to inform Cluster admins when StorageOS upgrades are available in the StorageOS GUI and in the logs.\nThe DNS query includes:\n Anonymized StorageOS Cluster ID Anonymized StorageOS node ID StorageOS version number If k8sDistro is set then the Kubernetes version and Kubernetes distribution will also be reported. This information helps us direct focus onto the most relevant platforms.\nDisable Telemetry It is possible to disable telemetry using the CLI, API, environment variables or the StorageOS Cluster Spec.\nAPI The example below shows how you can use the /v1/telemetry API endpoint to disable or enable telemetry.\n$ curl -X \u0026#34;PUT\u0026#34; \u0026#34;http://127.0.0.1:5705/v1/telemetry\u0026#34; \\ -H \u0026#39;Content-Type: application/json; charset=utf-8\u0026#39; \\ -u \u0026#39;storageos:storageos\u0026#39; \\ -d $\u0026#39;{ \u0026#34;telemetryEnabled\u0026#34;: true, \u0026#34;reportErrors\u0026#34;: false }\u0026#39; StorageOS Cluster Spec Disable telemetry explicitly through the configurable spec parameters of the StorageOSCluster custom resource.\nEnvironment Variables You can use the following environmental variables to disable or enable telemetry.\nDISABLE_TELEMETRY # Disable the DNS query and once per day sentry.io reporting DISABLE_ERROR_REPORTING # Disable sentry.io crash reports You can find more information about StorageOS environment variables here.\n","excerpt":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for …","ref":"/v1.x/docs/reference/telemetry/","title":"StorageOS Telemetry"},{"body":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base directory for storing configuration and blob files. Recommended host filesystem types are ext4 or xfs. If you require a specific filesystem please contact StorageOS.\nPersistent Volume Filesystems StorageOS provides a block device on which a file system can be created. The creation of the filesystem is either handled by StorageOS or by Kubernetes which affects what filesystems can be created.\nCSI Driver When using StorageOS with the CSI driver, StorageOS is responsible for running mkfs against the block device that pods mount. StorageOS is able to create ext2, ext3, ext4 and xfs file systems.\nNative Driver When using StorageOS with the native driver, Kubernetes is responsible for running mkfs against the StorageOS block device. This means that any file system that Kubernetes can create can be used.\nHowever if the StorageOS CLI is used to mount a device then an ext4 filesystem will be created by default.\n","excerpt":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base …","ref":"/v1.x/docs/reference/filesystems/","title":"Supported File systems"},{"body":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 active volumes per node 4.x kernels have a limitation of 4096 active volumes per node We are distribution agnostic as long as our prerequisites are met Orchestrators Kubernetes 1.7+ OpenShift 3.7+ ","excerpt":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 …","ref":"/v1.x/docs/introduction/platforms/","title":"Supported Platforms and Orchestrators"},{"body":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to the StorageOS installation, integration with orchestrators or common misconfigurations.\nTools To be able to troubleshoot issues the StorageOS cli is required.\nPod in pending because of mount error Issue: The output of kubectl describe pod $POD_ID contains no such file or directory and references the StorageOS volume device file.\nroot@node1:~# kubectl -n kube-system describe $POD_ID (...) Events: (...) Normal Scheduled 11s default-scheduler Successfully assigned default/d1 to node3 Warning FailedMount 4s (x4 over 9s) kubelet, node3 MountVolume.SetUp failed for volume \u0026#34;pvc-f2a49198-c00c-11e8-ba01-0800278dc04d\u0026#34; : stat /var/lib/storageos/volumes/d9df3549-26c0-4cfc-62b4-724b443069a1: no such file or directory Reason: There are two main reasons this issue may arise:\n The StorageOS DEVICE_DIR location is wrongly configured when using Kubelet as a container Mount Propagation is not enabled (Option 1) Misconfiguration of the DeviceDir/SharedDir\nSome Kubernetes distributions such as Rancher, DockerEE or some installations of OpenShift deploy the Kubelet as a container, because of this, the device files that StorageOS creates to mount into the containers need to be visible to the kubelet. StorageOS can be configured to share the device directory.\nModern installations use CSI, which handles the complexity internally.\nAssert: root@node1:~# kubectl -n default describe stos | grep \u0026#34;Shared Dir\u0026#34; Shared Dir: # \u0026amp;lt;-- Shouldn\u0026#39;t be blank Solution: The Cluster Operator Custom Definition should specify the SharedDir option as follows.\nspec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container ... See example on how to configure the StorageOS Custom Resource.\n (Option 2) Mount propagation is not enabled.\n Applies only if Option 1 is configured properly.\n Assert: If not using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/storageos/volumes is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/storageos/volumes/ root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/storageos/volumes bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nIf using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/kubelet/plugins/kubernetes.io~storageos/devices is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/kubelet/plugins/kubernetes.io~storageos/devices root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/kubelet/plugins/kubernetes.io~storageos/devices bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nSolution: Older versions of Kubernetes need to enable mount propagation as it is not enabled by default. Most Kubernetes distributions allow MountPropagation to be enabled using FeatureGates. Rancher specifically, needs to enable it in the \u0026ldquo;View in API\u0026rdquo; section of your cluster. You need to edit the section \u0026ldquo;rancherKubernetesEngineConfig\u0026rdquo; to enable the Kubelet feature gate.\nPVC pending state - Failed to dial StorageOS A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: root@node1:~/# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE vol-1 Pending fast 7s kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 7s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: Get http://storageos-cluster/version: failed to dial all known cluster members, (10.233.59.206:5705) Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount volume can\u0026rsquo;t be transmitted to StorageOS, hence the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nIn this case, the Event message indicates that StorageOS API is not responding, implying that StorageOS is not running. For Kubernetes to define StorageOS pods ready, the health check must pass.\nAssert: Check the status of StorageOS pods.\nroot@node1:~/# kubectl -n kube-system get pod --selector app=storageos # for CSI add --selector kind=daemonset NAME READY STATUS RESTARTS AGE storageos-qrqkj 0/1 Running 0 1m storageos-s4bfv 0/1 Running 0 1m storageos-vcpfx 0/1 Running 0 1m storageos-w98f5 0/1 Running 0 1m If the pods are not READY, the service will not forward traffic to the API they serve hence PVC will remain in pending state until StorageOS pods are available.\n Kubernetes keeps trying to execute the action until it succeeds. If a PVC is created before StorageOS finish starting, the PVC will be created eventually.\n Solution: StorageOS health check takes 60 seconds of grace before reporting as READY. If StorageOS is starting properly after that period, the volume will be created when StorageOS finishes its bootstrap. If StorageOS is not running or is not starting properly, the solution would be to troubleshoot the installation. PVC pending state - Secret Missing A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 13s (x2 over 28s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: failed to get secret from [\u0026#34;storageos\u0026#34;/\u0026#34;storageos-api\u0026#34;] Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount a volume can\u0026rsquo;t be transmitted to StorageOS, and the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nThe StorageClass provisioned for StorageOS references a Secret from where it retrieves the API endpoint and the authentication parameters. If that secret is incorrect or missing, the connections won\u0026rsquo;t be established. It is common to see that the Secret has been deployed in a different namespace where the StorageClass expects it or that is has been deployed with a different name.\nAssert: Check the StorageClass parameters to know where the Secret is expected to be found.\nkubectl get storageclass fast -o yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: 2018-09-25T08:44:57Z labels: app: storageos name: fast resourceVersion: \u0026#34;108853\u0026#34; selfLink: /apis/storage.k8s.io/v1/storageclasses/fast uid: 48490a9b-c09f-11e8-ba01-0800278dc04d parameters: adminSecretName: storageos-api adminSecretNamespace: storageos description: Kubernetes volume fsType: ext4 pool: default provisioner: kubernetes.io/storageos reclaimPolicy: Delete Note that the parameters specify adminSecretName and adminSecretNamespace.\n Check if the secret exists according to those parameters\nkubectl -n kube-system get secret storageos-api No resources found. Error from server (NotFound): secrets \u0026#34;storageos-api\u0026#34; not found If no resources are found, it is clear that the Secret doesn\u0026rsquo;t exist or it is not deployed in the right location.\n Solution: Deploy StorageOS following the installation procedures. If you are using the manifests provided for Kubernetes to deploy StorageOS rather than using automated provisioners, make sure that the StorageClass parameters and the Secret reference match.\nPeer discovery - Pod allocation Issue: StorageOS nodes can\u0026rsquo;t join the cluster and show the following log entries.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover the nodes in the cluster. When StorageOS starts, one or more active nodes must be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nIf there are no active StorageOS nodes, the bootstrap process will elect the first node in the JOIN variable as master, and the rest will try to discover from it. In case of that node not starting, the whole cluster will remain unable to bootstrap.\nInstallations of StorageOS use a DaemonSet, and by default do not schedule StorageOS pods to master nodes, due to the presence of the node-role.kubernetes.io/master:NoSchedule taint that is typically present. In such cases the JOIN variable must not contain master nodes or the StorageOS cluster will remain unable to start.\nAssert: Check that the first node of the JOIN variable started properly.\nroot@node1:~/# kubectl -n kube-system describe ds/storageos | grep JOIN JOIN: 172.28.128.3,172.28.128.4,172.28.128.5 root@node1:~/# kubectl -n kube-system get pod -o wide | grep 172.28.128.3 storageos-8zqxl 1/1 Running 0 2m 172.28.128.3 node1 Solution: Make sure that the JOIN variable doesn\u0026rsquo;t specify the master nodes. In case you are using the discovery service, it is necessary to ensure that the DaemonSet won\u0026rsquo;t allocate Pods on the masters. This can be achieved with taints, node selectors or labels.\nFor installations with the StorageOS operator you can specify which nodes to deploy StorageOS on using nodeSelectors. See examples in the Cluster Operator Examples page.\nFor more advanced installations using compute-only and storage nodes, check the storageos.com/deployment=computeonly label that can be added to the nodes through Kubernetes node labels, or StorageOS in the Labels page.\nLIO Init:Error Issue: StorageOS pods not starting with Init:Error\nkubectl -n kube-system get pod NAME READY STATUS RESTARTS AGE storageos-2kwqx 0/3 Init:Err 0 6s storageos-cffcr 0/3 Init:Err 0 6s storageos-d4f69 0/3 Init:Err 0 6s storageos-nhq7m 0/3 Init:Err 0 6s Reason: This indicates that since the Linux open source SCSI drivers are not enabled, StorageOS cannot start. The StorageOS DaemonSet enables the required kernel modules on the host system. If you are seeing these errors it is because that container couldn\u0026rsquo;t load the modules.\nAssert Check the logs of the init container.\nkubectl -n kube-system logs $ANY_STORAGEOS_POD -c storageos-init In case of failure, it will show the following output, indicating which kernel modules couldn\u0026rsquo;t be loaded or that they are not properly configured:\nChecking configfs configfs mounted on sys/kernel/config Module target_core_mod is not running executing modprobe -b target_core_mod Module tcm_loop is not running executing modprobe -b tcm_loop modprobe: FATAL: Module tcm_loop not found. Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and delete StorageOS pods, allowing the DaemonSet to create the pods again.\nLIO not enabled Issue: StorageOS node can\u0026rsquo;t start and shows the following log entries.\ntime=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;liocheck returned error\u0026#34; category=liocheck error=\u0026#34;exit status 1\u0026#34; module=dataplane stderr=\u0026#34;Sysfs root \u0026#39;/sys/kernel/config/target\u0026#39; is missing, is kernel configfs present and target_core_mod loaded? category=fslio level=warn\\nRuntime error checking stage \u0026#39;target_core_mod\u0026#39;: SysFs root missing category=fslio level=warn\\nliocheck: FAIL (lio_capable_system() returns failure) category=fslio level=fatal\\n\u0026#34; stdout= time=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;failed to start dataplane services\u0026#34; error=\u0026#34;system dependency check failed: exit status 1\u0026#34; module=command Reason: This indicates that one or more kernel modules required for StorageOS are not loaded.\nAssert The following kernel modules must be enabled in the host.\nlsmod | egrep \u0026#34;^tcm_loop|^target_core_mod|^target_core_file|^configfs\u0026#34; Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and restart the container.\n(OpenShift) StorageOS pods missing \u0026ndash; DaemonSet error StorageOS DaemonSet doesn\u0026rsquo;t have any pod replicas. The DaemonSet couldn\u0026rsquo;t allocate any Pod due to security issues.\nIssue: [root@master02 standard]# oc get pod No resources found. [root@master02 standard]# oc describe daemonset storageos (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedCreate 0s (x12 over 10s) daemonset-controller Error creating: pods \u0026#34;storageos-\u0026#34; is forbidden: unable to validate against any security context constraint: [provider restricted: .spec.securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used provider restricted: .spec.securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.volumes[0]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[1]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[2]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[3]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.initContainers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.initContainers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.initContainers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.initContainers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.containers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.containers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.containers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used] Reason: The OpenShift cluster has security context constraint policies enabled that forbid any pod, without an explicitly set policy for the service account, to be allocated.\nAssert: Check if the StorageOS ServiceAccount can create pods with enough permissions\noc get scc privileged -o yaml # Or custom scc with enough privileges (...) users: - system:admin - system:serviceaccount:openshift-infra:build-controller - system:serviceaccount:management-infra:management-admin - system:serviceaccount:management-infra:inspector-admin - system:serviceaccount:storageos:storageos \u0026amp;lt;-- - system:serviceaccount:tiller:tiller If the StorageOS sa system:serviceaccount:storageos:storageos is in the privileged scc it will be able to create pods.\nSolution: Add the ServiceAccount system:serviceaccount:storageos:storageos to a scc with enough privileges.\noc adm policy add-scc-to-user privileged system:serviceaccount:storageos:storageos Getting Help If our troubleshooting guides do not help resolve your issue, please see our support section for details on how to get in touch with us.\n","excerpt":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to …","ref":"/v1.x/docs/operations/troubleshooting/","title":"Troubleshooting"},{"body":" Before upgrading, check the release notes to confirm whether there is a safe upgrade path between versions.\n StorageOS version upgrades must be planned and executed taking into consideration that volumes will be inaccessible during the process. It is recommended to schedule a maintenance window.\nCurrently there are two strategies to upgrade StorageOS and both maintain data integrity during the upgrade process. StorageOS keeps the data on the hosts where the StorageOS container is running. When the new version of StorageOS starts, the volumes from the previous version are available.\n Full stop of the cluster Rolling upgrade More upgrade procedures will be released that will automate the main part of the process and fulfil use cases not covered currently.\n The StorageOS CLI is required to perform an upgrade.\n Option 1. Full stop of the cluster This option consists of downscaling all applications using StorageOS volumes to 0, stopping all StorageOS pods, starting StorageOS with a new version and rescaling applications to previous size. Deployments that don\u0026rsquo;t use StorageOS volumes remain unaffected.\nThis option does not require moving data across nodes, therefore it is recommended for clusters with large data sets. However, it implies downtime for stateful applications.\nExecute the following instructions:\nPrepare upgrade:\n Prepare an etcd backup if using external etcd.\n Optionally make sure all nodes have the new StorageOS image pulled, so the new containers will start promptly.\ndocker pull storageos/node:$NEW_VERSION Downscale any applications using StorageOS volumes to 0.\n Any mount points will hang while StorageOS Pods are not present if the application Pods haven\u0026rsquo;t been stopped. A restart of the Pods mounting volumes will be necessary if they are not stopped before hand.\n Put the StorageOS cluster in maintenance mode.\nStorageOS implements a maintenance mode that freezes the cluster. When in maintenance mode, StorageOS operations are limited. Functionalities such as volume provisioning, failover of primary volumes or managing nodes are disabled.\n Execute the StorageOS Upgrade cluster helper\ncurl -Ls https://raw.githubusercontent.com/storageos/deploy/master/k8s/deploy-storageos/upgrade-helper/prepare-upgrade.sh -o prepare-upgrade.sh chmod +x ./prepare-upgrade.sh ./prepare-upgrade.sh The upgrade helper patches the StorageOS DaemonSet and sets the latest version of the containers among other tasks.\n The prepare-upgrade.sh script can be executed as many times as needed to verify that your cluster is ready to be upgraded.\n Delete StorageOS Pods.\nkubectl -n $NAMESPACE delete pods --selector app=storageos,kind=daemonset Check that StorageOS is starting and wait until the Pods are in ready state.\nkubectl -n $NAMESPACE get pods Take the StorageOS cluster out of maintenance mode.\n Scale up applications that were using StorageOS volumes, once StorageOS is up and running.\n Option 2. Manual rolling upgrade This option consists of moving stateful applications and StorageOS volumes from a node, applying the version upgrade and repeating this process for every node. Only Pods using StorageOS nodes will need to be evicted.\nThis option requires the promotion of replicas and data to be rebuilt on new nodes at least once per volume. It is possible to wait for volumes without replicas to be evicted from a node, however we recommend that a replica is created as per the steps below. Please note that it is recommended to create at least one replica per node for the purpose of the upgrade.\nClusters with large data sets or a large number nodes might take a long time to finish the procedure.\nThis procedure requires a restart of the stateful pods at least twice during the procedure.\nThe amount of nodes you can upgrade at the same time will depend on the amount of replicas the volumes have. With 2 replicas per volume, it is possible to upgrade 2 nodes at a time without causing unavailability of data apart from the application stop/start. It is recommended to upgrade one node at a time.\nExecute the following instructions:\nPrepare upgrade:\n Prepare an etcd backup if using external etcd.\n Make sure all nodes have the new StorageOS image pulled, so the new containers will start promptly (optional).\ndocker pull storageos/node:$NEW_VERSION Make sure that all volumes have at least one replica\n# Save what volumes had 0 replicas to restore to that state later $ storageos volume ls --format \u0026#34;table {{.Name}}\\t{{.Replicas}}\u0026#34; \\ | grep \u0026#39;0/0\u0026#39; \\ | awk \u0026#39;{ print $1 }\u0026#39; \\ \u0026gt; /var/tmp/volume-0-replicas.txt # Update volumes to enable 1 replica $ storageos volume ls --format \u0026#34;table {{.Name}}\\t{{.Replicas}}\u0026#34; \\ | grep \u0026#39;0/0\u0026#39; \\ | awk \u0026#39;{ print $1 }\u0026#39; \\ | xargs -I{} storageos volume update --label-add storageos.com/replicas=1 {} Wait until all replicas are synced (1/1)\n$ storageos volume ls --format \u0026#34;table {{.Name}}\\t{{.Replicas}}\u0026#34; NAMESPACE/NAME REPLICAS default/pvc-166ba271-e75c-11e8-8a20-0683b54ab438 0/1 mysql/pvc-2b38b2e2-e75c-11e8-8a20-0683b54ab438 0/1 postgress/pvc-3b39f530-e75c-11e8-8a20-0683b54ab438 0/1 redis/pvc-8b8b37eb-e75d-11e8-8a20-0683b54ab438 0/1 (...) $ storageos volume ls --format \u0026#34;table {{.Name}}\\t{{.Replicas}}\u0026#34; NAMESPACE/NAME REPLICAS default/pvc-166ba271-e75c-11e8-8a20-0683b54ab438 1/1 mysql/pvc-2b38b2e2-e75c-11e8-8a20-0683b54ab438 1/1 postgress/pvc-3b39f530-e75c-11e8-8a20-0683b54ab438 1/1 redis/pvc-8b8b37eb-e75d-11e8-8a20-0683b54ab438 1/1 Execute the upgrade:\n Execute the StorageOS Upgrade cluster helper\ncurl -Ls https://raw.githubusercontent.com/storageos/deploy/master/k8s/deploy-storageos/upgrade-helper/prepare-upgrade.sh -o prepare-upgrade.sh chmod +x ./prepare-upgrade.sh ./prepare-upgrade.sh The upgrade helper patches the StorageOS DaemonSet and sets the latest version of the containers among other tasks.\n The prepare-upgrade.sh script can be executed as many times as needed to verify that your cluster is ready to be upgraded.\n Cordon and drain node\n# Select what node is being manipulated export NODE=node01 # Define your node storageos node cordon $NODE storageos node drain $NODE kubectl cordon $NODE kubectl drain --ignore-daemonsets $NODE If there are many Pods running stateless applications that don\u0026rsquo;t need to be evicted, you can delete the stateful Pods after the kubectl cordon $NODE so they start in a different node and omit the kubectl drain $NODE\n Delete StorageOS Pod.\nkubectl -n $NAMESPACE get pod -owide --no-headers | grep \u0026#34;$NODE\u0026#34; | awk \u0026#39;{print $1}\u0026#39; | xargs -I{} kubectl -n $NAMESPACE delete pod {} Wait until StorageOS Pod with the new version is started and ready.\n$ kubectl -n $NAMESPACE get pod -owide --no-headers | grep \u0026#34;$NODE\u0026#34; | awk \u0026#39;{print $1}\u0026#39; | xargs -I{} kubectl -n $NAMESPACE get pod {} NAME READY STATUS RESTARTS AGE storageos-j7pqf 1/1 Running 0 3m Uncordon node\nstorageos node uncordon $NODE storageos node undrain $NODE kubectl uncordon $NODE Repeat from point 2 for every node of the cluster\n Once you are finished with all nodes, remove the replicas of the volumes that didn\u0026rsquo;t have them on the first place.\n while read vol; do storageos volume update --label-add storageos.com/replicas=0 $vol done \u0026lt;/var/tmp/volume-0-replicas.txt ","excerpt":"Before upgrading, check the release notes to confirm whether there is a safe upgrade path between …","ref":"/v1.x/docs/operations/upgrades/","title":"Upgrading StorageOS version"},{"body":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces using policies.\n Note: Users are created with access to the default namespace. This access is only revoked when a policy is created for the user or their group.\n Creating users To create a user with the CLI, run:\n$ storageos user create jim --groups qa The above command will create a user named jim and add them to the group qa. The command will also prompt you to enter a password for the newly created user.\nThe groups flag is optional and the group will be created if it does not already exist.\nList all users To view all users, run:\n$ storageos user ls ID USERNAME GROUPS ROLE a3b2948c-c5ef-116c-35c0-0cf4a42acf79 storageos admin 395f9e99-8f60-52e7-6a90-36096666fea3 test test user Inspect users To inspect a user, run:\n$ storageos user inspect jim [ { \u0026#34;id\u0026#34;: \u0026#34;7f27fa40-ffdf-c443-1e60-214378003b97\u0026#34;, \u0026#34;username\u0026#34;: \u0026#34;jim\u0026#34;, \u0026#34;groups\u0026#34;: \u0026#34;qa\u0026#34;, \u0026#34;role\u0026#34;: \u0026#34;user\u0026#34; } ] Update a user To update a users attributes, run:\n$ storageos user update jim --add-groups dev The above command would add jim to the dev group. To see all the options that update has use the command below:\n$ storageos user update --help Deleting users To delete a user, run:\n$ storageos user rm jim Altering the StorageOS API account When installing with the StorageOS Operator, the StorageOS API account is defined by the storageos-api secret.\nFor installations using the native driver, Kubernetes uses the account defined in the secret to authenticate against the StorageOS API. Therefore if the account details are changed, the Kubernetes storageos-api secret needs to be updated. In order to update the secret you need to base64 encode the new username/password and edit the storageos-api secret to reflect the new account details.\necho -n USERNAME | base64 echo -n PASSWORD | base64 kubectl edit secret storageos-api For installations using CSI the storageos-api secret is used to define the default account credentials. However as Kubernetes communicates with StorageOS via the CSI socket, the secret is not used after cluster bootstrapping.\n","excerpt":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces …","ref":"/v1.x/docs/operations/users/","title":"User Management"},{"body":"$ storageos user Usage:\tstorageos user COMMAND Manage users Options: --help Print usage Commands: create Create a new User, E.g. \u0026#34;storageos user create --password alice\u0026#34; (interactive password prompt) inspect Display detailed information on one or more user(s) ls List users rm Remove one or more user(s) update Update select fields in a user account Run \u0026#39;storageos user COMMAND --help\u0026#39; for more information on a command. storageos user create To create a new (admin) user \u0026ldquo;awesomeUser\u0026rdquo; with a interactively provided password that is a member of the group \u0026ldquo;dev\u0026rdquo;:\n$ storageos user create --role admin --groups dev awesomeUser Password: Confirm Password: For multiple groups, use comma separation. E.g. --groups dev,testing,deploy For non admin users use --role user\nstorageos user inspect To display detailed information on the user \u0026ldquo;awesomeUser\u0026rdquo;, run:\n$ storageos user inspect awesomeUser [ { \u0026#34;id\u0026#34;: \u0026#34;861511a8-2843-1031-724c-2cabaa2ca4e9\u0026#34;, \u0026#34;username\u0026#34;: \u0026#34;awesomeUser\u0026#34;, \u0026#34;groups\u0026#34;: \u0026#34;dev\u0026#34;, \u0026#34;role\u0026#34;: \u0026#34;admin\u0026#34; } ] Note: Either the username or the ID can be used to select the user.\n storageos user ls To list the users on the system, run:\n$ storageos user ls ID USERNAME GROUPS ROLE 861511a8-2843-1031-724c-2cabaa2ca4e9 awesomeUser dev admin 9d6f2ea9-3c7d-2358-7d81-b29c50e10cc9 storageos admin storageos user rm To delete a user from the system, run:\n$ storageos user rm 861511a8-2843-1031-724c-2cabaa2ca4e9 861511a8-2843-1031-724c-2cabaa2ca4e9 The user is now deleted.\n$ storageos user ls ID USERNAME GROUPS ROLE 9d6f2ea9-3c7d-2358-7d81-b29c50e10cc9 storageos admin Note: Either the username or the ID can be used to select the user.\n storageos user update Change the password of the user \u0026ldquo;awesomeUser\u0026rdquo;:\n$ storageos user update --password awesomeUser Password: Confirm Password: Change the username of the user \u0026ldquo;awesomeUser\u0026rdquo;:\n$ storageos user update --username moreAwesomeUser awesomeUser Add \u0026ldquo;awesomeUser\u0026rdquo; to the dev group:\n$ storageos user update --add-groups dev awesomeUser Remove \u0026ldquo;awesomeUser\u0026rdquo; from the dev group:\n$ storageos user update --remove-groups dev awesomeUser Revoke \u0026ldquo;awesomeUser\u0026rdquo;\u0026lsquo;s admin privileges:\n$ storageos user update --role user awesomeUser Restore \u0026ldquo;awesomeUser\u0026rdquo;\u0026lsquo;s admin privileges:\n$ storageos user update --role admin awesomeUser ","excerpt":"$ storageos user Usage:\tstorageos user COMMAND Manage users Options: --help Print usage Commands: …","ref":"/v1.x/docs/reference/cli/user/","title":"Users"},{"body":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard POSIX semantics. We present volumes as mounts into containers via the Linux LIO subsystem.\nConceptually, StorageOS volumes have a frontend presentation, which is the side the application sees, and a backend presentation, which is the actual on-disk format. Depending on the configuration, frontend and backend components may be on the same or different hosts.\nVolumes are formatted using the linux standard ext4 filesystem by default. Kubernetes users may change the default filesystem type to ext2, ext3, ext4, or xfs by setting the fsType parameter in their StorageClass (See Supported Filesystems for more information). Different filesystems may be supported in the future.\nStorageOS volumes are represented on disk in two parts.\nActual volume data is written to blob files in /var/lib/storageos/data/dev[\\d+]. Inside these directories, each StorageOS block device gets two blob files of the form vol.xxxxxx.y.blob, where x is the inode number for the device, and y is an index between 0 and 1. We provide two blob files in order to ensure that certain operations which require locking do not impede in-flight writes to the volume.\nIn systems which have multiple /var/lib/storageos/data/dev[\\d+] directories, we place two blob files per block device. This allows us to load-balance writes across multiple devices. In cases where dev directories are added after a period of run time, later directories are favoured for writes until the data is distributed evenly across the blob files.\nMetadata is kept in directories named /var/lib/storageos/data/db[\\d+]. We maintain an index of all blocks written to the blob file inside the metadata store, including checksums. These checksums allow us to detect bitrot, and return errors on reads, rather than serve bad data. In future versions we may implement recovery from replicas for volumes with one or more replicas defined.\nStorageOS metadata requires approximately 2.7GB of storage per 1TB of allocated blocks in the associated volume. This size is consistent irrespective of data compression defined on the volume.\nTo ensure deterministic performance, individual StorageOS volumes must fit on a single node. In situations where overcommit is applied, a scaling factor is applied when determining whether to place a volume on a node.\nWe present various metrics regarding StorageOS volumes, including used capacity and throughput, via our Prometheus Endpoint.\n","excerpt":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard …","ref":"/v1.x/docs/concepts/volumes/","title":"Volumes"},{"body":"$ storageos volume Usage:\tstorageos volume COMMAND Manage volumes Options: --help Print usage Commands: create Create a volume inspect Display detailed information on one or more volumes ls List volumes mount Mount specified volume rm Remove one or more volumes unmount Unmount specified volume update Update a volume Run \u0026#39;storageos volume COMMAND --help\u0026#39; for more information on a command. Note that the mount and unmount commands are not available on MacOS or Windows.\nstorageos volume create To create a 15GB volume in the default namespace:\n$ storageos volume create --namespace default --size 15 --fstype xfs volume-name default/volume-name storageos volume inspect To view volume details:\n$ storageos volume inspect default/volume-name [ { \u0026#34;id\u0026#34;: \u0026#34;8d17066e-07d9-3f5f-e0de-edfc28e13f8f\u0026#34;, \u0026#34;inode\u0026#34;: 0, \u0026#34;name\u0026#34;: \u0026#34;volume-name\u0026#34;, \u0026#34;size\u0026#34;: 15, \u0026#34;pool\u0026#34;: \u0026#34;default\u0026#34;, \u0026#34;fsType\u0026#34;: \u0026#34;xfs\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;labels\u0026#34;: {}, \u0026#34;namespace\u0026#34;: \u0026#34;default\u0026#34;, \u0026#34;nodeSelector\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;master\u0026#34;: { \u0026#34;id\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;inode\u0026#34;: 0, \u0026#34;node\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;nodeName\u0026#34;: \u0026#34;storageos-1\u0026#34;, \u0026#34;controller\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;controllerName\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;health\u0026#34;: \u0026#34;healthy\u0026#34;, \u0026#34;status\u0026#34;: \u0026#34;active\u0026#34;, \u0026#34;createdAt\u0026#34;: \u0026#34;0001-01-01T00:00:00Z\u0026#34; }, \u0026#34;mounted\u0026#34;: false, \u0026#34;mountDevice\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;mountpoint\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;mountedAt\u0026#34;: \u0026#34;0001-01-01T00:00:00Z\u0026#34;, \u0026#34;replicas\u0026#34;: [], \u0026#34;health\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;status\u0026#34;: \u0026#34;active\u0026#34;, \u0026#34;statusMessage\u0026#34;: \u0026#34;\u0026#34;, \u0026#34;mkfsDone\u0026#34;: false, \u0026#34;mkfsDoneAt\u0026#34;: \u0026#34;0001-01-01T00:00:00Z\u0026#34;, \u0026#34;createdAt\u0026#34;: \u0026#34;0001-01-01T00:00:00Z\u0026#34;, \u0026#34;createdBy\u0026#34;: \u0026#34;storageos\u0026#34; } ] storageos volume ls To view all volumes in all namespaces:\n$ storageos volume ls NAMESPACE/NAME SIZE MOUNT SELECTOR STATUS REPLICAS LOCATION default/volume-name 15GB active 0/0 storageos-1 (healthy) storageos volume mount To mount a volume on the current node into /mnt (note this requires root):\nsudo -E storageos volume mount default/volume-name /mnt (Important: use the sudo -E option to preserve the storageos environment credentials)\nstorageos volume rm To delete a volume (use --force to delete mounted volumes):\n$ storageos volume rm default/volume-name default/volume-name storageos volume unmount To unmount a volume on the current node (note this requires root):\nsudo -E storageos volume unmount default/volume-name (Important: use the sudo -E option to preserve the storageos environment credentials)\n","excerpt":"$ storageos volume Usage:\tstorageos volume COMMAND Manage volumes Options: --help Print usage …","ref":"/v1.x/docs/reference/cli/volume/","title":"Volumes"},{"body":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.\nUsing StorageOS persistent volumes with Apache Zookeeper means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Zookeeper pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nAs Zookeeper has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Zookeeper to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Zookeeper on Kubernetes Pre-requisites StorageOS is assumed to have been installed; please check for the latest available version here. Helm To simplify the deployment of Zookeeper, we\u0026rsquo;ve used this Zookeeper helm chart (incubator) (version 1.2.2, app version 3.4.10) and rendered it into the example deployment files you can find in our GitHub repo.\nDeployment Clone the use cases repo You can find the latest files in the StorageOS use cases repository in /zookeeper/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion ---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:zookeeper...spec:replicas:3# \u0026lt;--- number of zookeeper pods...containers:- name:zookeeperimage:\u0026#34;gcr.io/google_samples/k8szk:v3\u0026#34;imagePullPolicy:IfNotPresent...volumeMounts:- name:datamountPath:/var/lib/zookeepervolumeClaimTemplates:- metadata:name:dataspec:accessModes:- \u0026#34;ReadWriteOnce\u0026#34;storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useresources:requests:storage:\u0026#34;5Gi\u0026#34;# \u0026lt;--- storage requested per podThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nCreate the kubernetes objects cd storageos-usecases kubectl apply -f ./zookeeper/ Confirm Zookeeper is up and running $ kubectl get pods NAME READY STATUS RESTARTS AGE zookeeper-0 1/1 Running 0 2m30s zookeeper-1 1/1 Running 0 112s zookeeper-2 1/1 Running 0 56s zookeeper-test-client 1/1 Running 0 2m30s Connect to Zookeeper Connect to the zookeeper client pod and list existing topics using the service endpoint\nkubectl exec -it zookeeper-test-client /bin/bash and issue a command to the zookeeper service\nzkCli.sh -server zookeeper ls /zookeeper ","excerpt":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration …","ref":"/v1.x/docs/usecases/zookeeper/","title":"Zookeeper"}]
\ No newline at end of file
diff --git a/v1.x/search/index.html b/v1.x/search/index.html
index cbcc8ded0..dd1c794d0 100644
--- a/v1.x/search/index.html
+++ b/v1.x/search/index.html
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v1.x/offline-search-index.c1d515a7dc0bab317cda6afaa525c31088b39057ff9870195d2553520983a8c7c88e897f25a8010c9e99eb1f59b7c02f785b0414ce1d40425b75d87d6033e1b8.json"
+ data-offline-search-index-json-src="/v1.x/offline-search-index.38f7c88ab8e5a7bbd77887e57c5905a6424ecd5c37922a170440aaca92ce3e3716a5a227e0b2d058c4a024a82a21f64213b4bc7056aa57f94af2e6087f79ef12.json"
data-offline-search-base-href="/"
>
diff --git a/v1.x/yaml/bundle-configuration.yaml b/v1.x/yaml/bundle-configuration.yaml
index f1c1004a0..5b21e3ed3 100644
--- a/v1.x/yaml/bundle-configuration.yaml
+++ b/v1.x/yaml/bundle-configuration.yaml
@@ -40,11 +40,18 @@ spec:
# InternalAPIPort = 5710 # Grpc API
# GossipPort = 5711 # Gossip+Healthcheck
- echo \"Source node for the test:\";
+ date;
+ echo \"Source node:\";
hostname -f -I; echo;
+ echo \"TCP:\";
parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
- ::: 5703 5704 5705 5710 5711
+ ::: 5703 5704 5705 5710 5711;
+ echo;
+
+ echo \"UDP:\";
+ parallel -j2 nc -vunz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
+ ::: 5711;
"
timeout: 90s
- exec:
diff --git a/v2.0/404.html b/v2.0/404.html
index 83cacdb43..354d937c9 100644
--- a/v2.0/404.html
+++ b/v2.0/404.html
@@ -181,7 +181,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/best-practices/index.html b/v2.0/docs/best-practices/index.html
index 07c5aeda2..242b50baa 100644
--- a/v2.0/docs/best-practices/index.html
+++ b/v2.0/docs/best-practices/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/architecture/index.html b/v2.0/docs/concepts/architecture/index.html
index 95a0a4146..37331b11f 100644
--- a/v2.0/docs/concepts/architecture/index.html
+++ b/v2.0/docs/concepts/architecture/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/clusters/index.html b/v2.0/docs/concepts/clusters/index.html
index 17c01c1b3..36b351be2 100644
--- a/v2.0/docs/concepts/clusters/index.html
+++ b/v2.0/docs/concepts/clusters/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/compression/index.html b/v2.0/docs/concepts/compression/index.html
index 807ac4077..523161610 100644
--- a/v2.0/docs/concepts/compression/index.html
+++ b/v2.0/docs/concepts/compression/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/etcd/index.html b/v2.0/docs/concepts/etcd/index.html
index 81df1083d..d0cfb5de3 100644
--- a/v2.0/docs/concepts/etcd/index.html
+++ b/v2.0/docs/concepts/etcd/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/index.html b/v2.0/docs/concepts/index.html
index 1b476d248..709dbf7cd 100644
--- a/v2.0/docs/concepts/index.html
+++ b/v2.0/docs/concepts/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/namespaces/index.html b/v2.0/docs/concepts/namespaces/index.html
index 84038ac27..8878ec018 100644
--- a/v2.0/docs/concepts/namespaces/index.html
+++ b/v2.0/docs/concepts/namespaces/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/nodes/index.html b/v2.0/docs/concepts/nodes/index.html
index 0706b0779..44158cbf1 100644
--- a/v2.0/docs/concepts/nodes/index.html
+++ b/v2.0/docs/concepts/nodes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/policies/index.html b/v2.0/docs/concepts/policies/index.html
index 74e40a310..5dd582d9a 100644
--- a/v2.0/docs/concepts/policies/index.html
+++ b/v2.0/docs/concepts/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/replication/index.html b/v2.0/docs/concepts/replication/index.html
index 4a250c804..38a950eaf 100644
--- a/v2.0/docs/concepts/replication/index.html
+++ b/v2.0/docs/concepts/replication/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/concepts/volumes/index.html b/v2.0/docs/concepts/volumes/index.html
index 99ffce5f9..17f72f87b 100644
--- a/v2.0/docs/concepts/volumes/index.html
+++ b/v2.0/docs/concepts/volumes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/index.html b/v2.0/docs/index.html
index 017214ee4..cb94c5583 100644
--- a/v2.0/docs/index.html
+++ b/v2.0/docs/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/index.xml b/v2.0/docs/index.xml
index c6463e3ca..660b2b246 100644
--- a/v2.0/docs/index.xml
+++ b/v2.0/docs/index.xml
@@ -1620,6 +1620,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -1652,7 +1657,7 @@ the init container, perform the following steps:</p>
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -1765,6 +1770,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2.0/docs/install/index.html b/v2.0/docs/install/index.html
index e36c999ea..f7911aaf5 100644
--- a/v2.0/docs/install/index.html
+++ b/v2.0/docs/install/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/install/kubernetes/index.html b/v2.0/docs/install/kubernetes/index.html
index 39718fb73..4106066c2 100644
--- a/v2.0/docs/install/kubernetes/index.html
+++ b/v2.0/docs/install/kubernetes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/install/openshift/index.html b/v2.0/docs/install/openshift/index.html
index da885b191..bdcb34485 100644
--- a/v2.0/docs/install/openshift/index.html
+++ b/v2.0/docs/install/openshift/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/install/rancher/index.html b/v2.0/docs/install/rancher/index.html
index dd6be761c..f2e02516d 100644
--- a/v2.0/docs/install/rancher/index.html
+++ b/v2.0/docs/install/rancher/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/introduction/index.html b/v2.0/docs/introduction/index.html
index c5686f5a8..a37cd42cd 100644
--- a/v2.0/docs/introduction/index.html
+++ b/v2.0/docs/introduction/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/introduction/overview/index.html b/v2.0/docs/introduction/overview/index.html
index e346ebdf5..6903dd217 100644
--- a/v2.0/docs/introduction/overview/index.html
+++ b/v2.0/docs/introduction/overview/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/introduction/platforms/index.html b/v2.0/docs/introduction/platforms/index.html
index 4820f0244..c4a2c839a 100644
--- a/v2.0/docs/introduction/platforms/index.html
+++ b/v2.0/docs/introduction/platforms/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/introduction/quickstart/index.html b/v2.0/docs/introduction/quickstart/index.html
index 696b966d6..2df543740 100644
--- a/v2.0/docs/introduction/quickstart/index.html
+++ b/v2.0/docs/introduction/quickstart/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/external-etcd/etcd-tls/index.html b/v2.0/docs/operations/external-etcd/etcd-tls/index.html
index 3aa937df4..665aa20f2 100644
--- a/v2.0/docs/operations/external-etcd/etcd-tls/index.html
+++ b/v2.0/docs/operations/external-etcd/etcd-tls/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/external-etcd/index.html b/v2.0/docs/operations/external-etcd/index.html
index d566819f1..15b2d21c0 100644
--- a/v2.0/docs/operations/external-etcd/index.html
+++ b/v2.0/docs/operations/external-etcd/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/firstpvc/index.html b/v2.0/docs/operations/firstpvc/index.html
index 2a6215485..c735442e7 100644
--- a/v2.0/docs/operations/firstpvc/index.html
+++ b/v2.0/docs/operations/firstpvc/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/health/index.html b/v2.0/docs/operations/health/index.html
index 8d044f846..37d6e6dcc 100644
--- a/v2.0/docs/operations/health/index.html
+++ b/v2.0/docs/operations/health/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/index.html b/v2.0/docs/operations/index.html
index 90072e575..77de28923 100644
--- a/v2.0/docs/operations/index.html
+++ b/v2.0/docs/operations/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/licensing/index.html b/v2.0/docs/operations/licensing/index.html
index 92d10c9e7..2cb187179 100644
--- a/v2.0/docs/operations/licensing/index.html
+++ b/v2.0/docs/operations/licensing/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/managing-host-storage/index.html b/v2.0/docs/operations/managing-host-storage/index.html
index 4b1af66d3..80e758597 100644
--- a/v2.0/docs/operations/managing-host-storage/index.html
+++ b/v2.0/docs/operations/managing-host-storage/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/namespaces/index.html b/v2.0/docs/operations/namespaces/index.html
index 24a191c41..e3c00dd63 100644
--- a/v2.0/docs/operations/namespaces/index.html
+++ b/v2.0/docs/operations/namespaces/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/policies/index.html b/v2.0/docs/operations/policies/index.html
index f4136d799..4d0dccaab 100644
--- a/v2.0/docs/operations/policies/index.html
+++ b/v2.0/docs/operations/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/resize/index.html b/v2.0/docs/operations/resize/index.html
index 5cea47e8a..14116cd11 100644
--- a/v2.0/docs/operations/resize/index.html
+++ b/v2.0/docs/operations/resize/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/storageclasses/index.html b/v2.0/docs/operations/storageclasses/index.html
index 5ad6f2825..fc3c0ba06 100644
--- a/v2.0/docs/operations/storageclasses/index.html
+++ b/v2.0/docs/operations/storageclasses/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/troubleshooting/index.html b/v2.0/docs/operations/troubleshooting/index.html
index 0917ab0f8..155fc6a42 100644
--- a/v2.0/docs/operations/troubleshooting/index.html
+++ b/v2.0/docs/operations/troubleshooting/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/operations/users/index.html b/v2.0/docs/operations/users/index.html
index 204bb33a2..d4e646539 100644
--- a/v2.0/docs/operations/users/index.html
+++ b/v2.0/docs/operations/users/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/platforms/aws-eks/index.html b/v2.0/docs/platforms/aws-eks/index.html
index bd1ff3e0c..054aa6118 100644
--- a/v2.0/docs/platforms/aws-eks/index.html
+++ b/v2.0/docs/platforms/aws-eks/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/platforms/azure-aks/index.html b/v2.0/docs/platforms/azure-aks/index.html
index e0290373b..ba7d0cae9 100644
--- a/v2.0/docs/platforms/azure-aks/index.html
+++ b/v2.0/docs/platforms/azure-aks/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/platforms/dockeree/index.html b/v2.0/docs/platforms/dockeree/index.html
index 0ef4f735a..458c47bcc 100644
--- a/v2.0/docs/platforms/dockeree/index.html
+++ b/v2.0/docs/platforms/dockeree/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/platforms/index.html b/v2.0/docs/platforms/index.html
index c55930234..203b25358 100644
--- a/v2.0/docs/platforms/index.html
+++ b/v2.0/docs/platforms/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/platforms/openshift/index.html b/v2.0/docs/platforms/openshift/index.html
index 739ea56be..bc632161e 100644
--- a/v2.0/docs/platforms/openshift/index.html
+++ b/v2.0/docs/platforms/openshift/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/platforms/rancher/index.html b/v2.0/docs/platforms/rancher/index.html
index fb9d75cf4..913a1ec40 100644
--- a/v2.0/docs/platforms/rancher/index.html
+++ b/v2.0/docs/platforms/rancher/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/prerequisites/etcd/index.html b/v2.0/docs/prerequisites/etcd/index.html
index 6fb3c0dcd..9d8c3fa52 100644
--- a/v2.0/docs/prerequisites/etcd/index.html
+++ b/v2.0/docs/prerequisites/etcd/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/prerequisites/firewalls/index.html b/v2.0/docs/prerequisites/firewalls/index.html
index b175130d7..34b1e9958 100644
--- a/v2.0/docs/prerequisites/firewalls/index.html
+++ b/v2.0/docs/prerequisites/firewalls/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/prerequisites/index.html b/v2.0/docs/prerequisites/index.html
index 2d34043f6..089140663 100644
--- a/v2.0/docs/prerequisites/index.html
+++ b/v2.0/docs/prerequisites/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/prerequisites/mountpropagation/index.html b/v2.0/docs/prerequisites/mountpropagation/index.html
index b5e266cfc..2b504e870 100644
--- a/v2.0/docs/prerequisites/mountpropagation/index.html
+++ b/v2.0/docs/prerequisites/mountpropagation/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/prerequisites/systemconfiguration/index.html b/v2.0/docs/prerequisites/systemconfiguration/index.html
index 045547aef..11175138f 100644
--- a/v2.0/docs/prerequisites/systemconfiguration/index.html
+++ b/v2.0/docs/prerequisites/systemconfiguration/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/api/index.html b/v2.0/docs/reference/api/index.html
index 03d5e0046..cdf21d420 100644
--- a/v2.0/docs/reference/api/index.html
+++ b/v2.0/docs/reference/api/index.html
@@ -189,7 +189,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/bundle/index.html b/v2.0/docs/reference/bundle/index.html
index 0f879ed82..ddff8eb62 100644
--- a/v2.0/docs/reference/bundle/index.html
+++ b/v2.0/docs/reference/bundle/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/apply/index.html b/v2.0/docs/reference/cli/apply/index.html
index 282170b5f..26a33152b 100644
--- a/v2.0/docs/reference/cli/apply/index.html
+++ b/v2.0/docs/reference/cli/apply/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/attach/index.html b/v2.0/docs/reference/cli/attach/index.html
index f00dfbf21..c27e22879 100644
--- a/v2.0/docs/reference/cli/attach/index.html
+++ b/v2.0/docs/reference/cli/attach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/create/index.html b/v2.0/docs/reference/cli/create/index.html
index 8c81c7f44..34509b32a 100644
--- a/v2.0/docs/reference/cli/create/index.html
+++ b/v2.0/docs/reference/cli/create/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/delete/index.html b/v2.0/docs/reference/cli/delete/index.html
index ec6a3f602..4fa966ac1 100644
--- a/v2.0/docs/reference/cli/delete/index.html
+++ b/v2.0/docs/reference/cli/delete/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/describe/index.html b/v2.0/docs/reference/cli/describe/index.html
index ef15ab8d5..22f0e2b39 100644
--- a/v2.0/docs/reference/cli/describe/index.html
+++ b/v2.0/docs/reference/cli/describe/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/detach/index.html b/v2.0/docs/reference/cli/detach/index.html
index ad4df4fb3..aad9bb35b 100644
--- a/v2.0/docs/reference/cli/detach/index.html
+++ b/v2.0/docs/reference/cli/detach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/get/index.html b/v2.0/docs/reference/cli/get/index.html
index 5ca325be4..e33ccd2fb 100644
--- a/v2.0/docs/reference/cli/get/index.html
+++ b/v2.0/docs/reference/cli/get/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/help/index.html b/v2.0/docs/reference/cli/help/index.html
index b0111f469..936bcde87 100644
--- a/v2.0/docs/reference/cli/help/index.html
+++ b/v2.0/docs/reference/cli/help/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/index.html b/v2.0/docs/reference/cli/index.html
index 265b238c3..a8a91f243 100644
--- a/v2.0/docs/reference/cli/index.html
+++ b/v2.0/docs/reference/cli/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cli/version/index.html b/v2.0/docs/reference/cli/version/index.html
index 67997e823..ce495df99 100644
--- a/v2.0/docs/reference/cli/version/index.html
+++ b/v2.0/docs/reference/cli/version/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cluster-operator/configuration/index.html b/v2.0/docs/reference/cluster-operator/configuration/index.html
index 6a386b7c2..6b846366d 100644
--- a/v2.0/docs/reference/cluster-operator/configuration/index.html
+++ b/v2.0/docs/reference/cluster-operator/configuration/index.html
@@ -30,7 +30,7 @@
-
+
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -1115,6 +1115,11 @@ StorageOSCluster Resource Confi
false |
+images.apiManagerContainer |
+StorageOS API Manager container image |
+storageos/api-manager:v1.0.0 |
+
+
images.csiClusterDriverRegistrarContainer |
CSI Cluster Driver Registrar Container image |
quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 |
@@ -1147,7 +1152,7 @@ StorageOSCluster Resource Confi
images.initContainer |
StorageOS init container image |
-storageos/init:1.0.0 |
+storageos/init:2.1.0 |
images.kubeSchedulerContainer |
@@ -1260,6 +1265,11 @@ StorageOSCluster Resource Confi
/var/lib/kubelet/plugins/kubernetes.io~storageos |
+storageClassName |
+The name of the default StorageClass created for StorageOS volumes |
+fast |
+
+
tlsEtcdSecretRefName |
Secret containing etcd client certificates |
|
diff --git a/v2.0/docs/reference/cluster-operator/examples/index.html b/v2.0/docs/reference/cluster-operator/examples/index.html
index 4a69a47e9..285198fbd 100644
--- a/v2.0/docs/reference/cluster-operator/examples/index.html
+++ b/v2.0/docs/reference/cluster-operator/examples/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cluster-operator/index.html b/v2.0/docs/reference/cluster-operator/index.html
index ea000f02b..4d8c42e44 100644
--- a/v2.0/docs/reference/cluster-operator/index.html
+++ b/v2.0/docs/reference/cluster-operator/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cluster-operator/index.xml b/v2.0/docs/reference/cluster-operator/index.xml
index 1c7ba500f..6f72d5b25 100644
--- a/v2.0/docs/reference/cluster-operator/index.xml
+++ b/v2.0/docs/reference/cluster-operator/index.xml
@@ -108,6 +108,11 @@
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -140,7 +145,7 @@
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -253,6 +258,11 @@
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2.0/docs/reference/cluster-operator/install/index.html b/v2.0/docs/reference/cluster-operator/install/index.html
index 77414aabd..44ddf8995 100644
--- a/v2.0/docs/reference/cluster-operator/install/index.html
+++ b/v2.0/docs/reference/cluster-operator/install/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/cluster-operator/upgrade/index.html b/v2.0/docs/reference/cluster-operator/upgrade/index.html
index eb15b8c65..c206674fc 100644
--- a/v2.0/docs/reference/cluster-operator/upgrade/index.html
+++ b/v2.0/docs/reference/cluster-operator/upgrade/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/contributing/index.html b/v2.0/docs/reference/contributing/index.html
index 27fbc1df0..99449181d 100644
--- a/v2.0/docs/reference/contributing/index.html
+++ b/v2.0/docs/reference/contributing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/filesystems/index.html b/v2.0/docs/reference/filesystems/index.html
index 2898b7878..e87405e9a 100644
--- a/v2.0/docs/reference/filesystems/index.html
+++ b/v2.0/docs/reference/filesystems/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/gui/index.html b/v2.0/docs/reference/gui/index.html
index 4384a0110..cfaea2f64 100644
--- a/v2.0/docs/reference/gui/index.html
+++ b/v2.0/docs/reference/gui/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/index.html b/v2.0/docs/reference/index.html
index aff225e98..1f7369df8 100644
--- a/v2.0/docs/reference/index.html
+++ b/v2.0/docs/reference/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/init-container/index.html b/v2.0/docs/reference/init-container/index.html
index cbe1915df..c703b56d9 100644
--- a/v2.0/docs/reference/init-container/index.html
+++ b/v2.0/docs/reference/init-container/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/labels/index.html b/v2.0/docs/reference/labels/index.html
index 30a0b5a4c..987bd58c0 100644
--- a/v2.0/docs/reference/labels/index.html
+++ b/v2.0/docs/reference/labels/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/licence/index.html b/v2.0/docs/reference/licence/index.html
index e9be02106..b67bc31ae 100644
--- a/v2.0/docs/reference/licence/index.html
+++ b/v2.0/docs/reference/licence/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/open_source_attribution/index.html b/v2.0/docs/reference/open_source_attribution/index.html
index 0af8fe1a3..9c79dd4da 100644
--- a/v2.0/docs/reference/open_source_attribution/index.html
+++ b/v2.0/docs/reference/open_source_attribution/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/release_notes/index.html b/v2.0/docs/reference/release_notes/index.html
index 2bee90ed6..7ac904848 100644
--- a/v2.0/docs/reference/release_notes/index.html
+++ b/v2.0/docs/reference/release_notes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/telemetry/index.html b/v2.0/docs/reference/telemetry/index.html
index e84c4523e..cd522de7b 100644
--- a/v2.0/docs/reference/telemetry/index.html
+++ b/v2.0/docs/reference/telemetry/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/reference/tolerations/index.html b/v2.0/docs/reference/tolerations/index.html
index 32721a79b..0654201ed 100644
--- a/v2.0/docs/reference/tolerations/index.html
+++ b/v2.0/docs/reference/tolerations/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/self-eval/index.html b/v2.0/docs/self-eval/index.html
index 9317cbfd6..3b27e27e7 100644
--- a/v2.0/docs/self-eval/index.html
+++ b/v2.0/docs/self-eval/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/support/index.html b/v2.0/docs/support/index.html
index 5d67bab93..b48c076af 100644
--- a/v2.0/docs/support/index.html
+++ b/v2.0/docs/support/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/backups/index.html b/v2.0/docs/usecases/backups/index.html
index 9f2bde2d1..f3f206efb 100644
--- a/v2.0/docs/usecases/backups/index.html
+++ b/v2.0/docs/usecases/backups/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/cassandra/index.html b/v2.0/docs/usecases/cassandra/index.html
index a8da5a524..ef4446df9 100644
--- a/v2.0/docs/usecases/cassandra/index.html
+++ b/v2.0/docs/usecases/cassandra/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/elasticsearch/index.html b/v2.0/docs/usecases/elasticsearch/index.html
index 858634f1c..65f18eb4b 100644
--- a/v2.0/docs/usecases/elasticsearch/index.html
+++ b/v2.0/docs/usecases/elasticsearch/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/index.html b/v2.0/docs/usecases/index.html
index b7c191db6..afe951c88 100644
--- a/v2.0/docs/usecases/index.html
+++ b/v2.0/docs/usecases/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/jenkins/index.html b/v2.0/docs/usecases/jenkins/index.html
index 93288f3b2..73a2d25b5 100644
--- a/v2.0/docs/usecases/jenkins/index.html
+++ b/v2.0/docs/usecases/jenkins/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/kafka/index.html b/v2.0/docs/usecases/kafka/index.html
index 56b7f0e49..2b2c88045 100644
--- a/v2.0/docs/usecases/kafka/index.html
+++ b/v2.0/docs/usecases/kafka/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/kubevirt/index.html b/v2.0/docs/usecases/kubevirt/index.html
index 352925043..fc280a14c 100644
--- a/v2.0/docs/usecases/kubevirt/index.html
+++ b/v2.0/docs/usecases/kubevirt/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/mssql/index.html b/v2.0/docs/usecases/mssql/index.html
index c7b8a9152..93526e66e 100644
--- a/v2.0/docs/usecases/mssql/index.html
+++ b/v2.0/docs/usecases/mssql/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/mysql/index.html b/v2.0/docs/usecases/mysql/index.html
index 1bb5877a9..612caa712 100644
--- a/v2.0/docs/usecases/mysql/index.html
+++ b/v2.0/docs/usecases/mysql/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/nginx/index.html b/v2.0/docs/usecases/nginx/index.html
index daea09675..404dc1a92 100644
--- a/v2.0/docs/usecases/nginx/index.html
+++ b/v2.0/docs/usecases/nginx/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/postgres/index.html b/v2.0/docs/usecases/postgres/index.html
index 2896e246c..01fbd2a1a 100644
--- a/v2.0/docs/usecases/postgres/index.html
+++ b/v2.0/docs/usecases/postgres/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/prometheus/index.html b/v2.0/docs/usecases/prometheus/index.html
index 4a8004bed..8e933b98c 100644
--- a/v2.0/docs/usecases/prometheus/index.html
+++ b/v2.0/docs/usecases/prometheus/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/redis/index.html b/v2.0/docs/usecases/redis/index.html
index c03b0c1af..3a36ce148 100644
--- a/v2.0/docs/usecases/redis/index.html
+++ b/v2.0/docs/usecases/redis/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/docs/usecases/zookeeper/index.html b/v2.0/docs/usecases/zookeeper/index.html
index 44a531973..3034c7fa0 100644
--- a/v2.0/docs/usecases/zookeeper/index.html
+++ b/v2.0/docs/usecases/zookeeper/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/index.html b/v2.0/index.html
index 064228a96..67e925ab5 100644
--- a/v2.0/index.html
+++ b/v2.0/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/index.json b/v2.0/index.json
index d2aa6eb86..d5015ff8f 100644
--- a/v2.0/index.json
+++ b/v2.0/index.json
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json b/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json
new file mode 100644
index 000000000..98826c843
--- /dev/null
+++ b/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json
@@ -0,0 +1 @@
+[{"body":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher OpenShift ","excerpt":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher …","ref":"/v2.0/docs/reference/cluster-operator/install/","title":"Install"},{"body":"Current Version The current StorageOS version is v2.0.0\nSections Our documentation is arranged into sections, accessible from the navigation bar on the left.\nIntroduction - Quickstart and support information.\nConcepts - Architectural and deep technical information.\nPrerequisites - We require certain prerequisites to be met for the product to function correctly. Please read these carefully and ensure they are implemented.\nPlatforms - Due to differences in the various orchestrators that StorageOS can run under, we list install guides and other platform specific operations here.\nOperations - Platform agnostic operations.\nUse Cases - A set of examples to get up and running with StorageOS quickly.\nReference - Information on our GUI, CLI, and other important information.\n","excerpt":"Current Version The current StorageOS version is v2.0.0\nSections Our documentation is arranged into …","ref":"/v2.0/docs/introduction/","title":"Introduction"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed service such as EKS, AKS, GKE, DO or DockerEE platform can use the following Kubernetes guide to install StorageOS.\n StorageOS v2 uses CSI only. The native driver has been deprecated.\n Make sure to add a StorageOS licence after installing.\n 1.17 1.16 1.15 1.14 1.13 Install StorageOS on Kubernetes 1.17 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.16 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.15 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.14 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.13 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed …","ref":"/v2.0/docs/install/kubernetes/","title":"Kubernetes"},{"body":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some significant enhancements over our v1 product. We\u0026rsquo;ve built V2 based on our observations of trends in the industry, as well as our own experience.\nMany of our customers want to run big clusters - in the tens or hundreds of nodes. In these sorts of big environments, the challenges multiply. Not only do we need to scale well, but we also need to be more failure tolerant. Bigger environments typically suffer higher failure rates (more nodes = greater chance of something failing), but are also subject to all sorts of transient conditions such as network partitions.\nThe second trend we\u0026rsquo;ve seen become increasingly common is the desire to run multiple clusters, and consume storage between them in some way - sometimes to implement novel topologies such as a centralised storage cluster with satellites consuming the storage, and sometimes to replicate data between those clusters for HA or DR purposes.\nWe\u0026rsquo;ve built V2 with these architectures and design patterns in mind. Not only does it scale well, but it contains the foundations we need to implement a rich set of multi-cluster functionality.\nUpgraded Control Plane At the heart of the V2 release is an upgraded control plane. We\u0026rsquo;ve changed a lot here. Firstly, our usage of etcd is vastly improved. We\u0026rsquo;ve learnt a lot about the subtleties of distributed consensus in the last year, particularly in noisy or unpredictable environments. Not only is StorageOS V2 much lighter on your etcd cluster, but it\u0026rsquo;s a lot more tolerant of transient failure conditions that are often found in cloud environments, or clusters under heavy load.\nWe spent some time describing and testing our internal state machine using the TLA+ formal verification language. This allows us to have a much higher degree of confidence that our algorithms will behave correctly, particularly under hard-to-test edge cases and failure conditions.\nAdditionally, we\u0026rsquo;ve changed the way volumes behave with respect to centralised scheduling. Each volume group (consisting of a master and 0 or more replicas) now behaves as a strongly consistent unit allowing it to take action independent of the activities of the rest of the cluster. Other state can be distributed around the cluster using eventually consistent mechanisms. This approach inherently scales better and allows StorageOS V2 to effectively manage many more nodes and volumes than before.\nWe\u0026rsquo;ve implemented TLS on all endpoints. Not only does this give you encrypted traffic between nodes in your storage cluster, it also protects all endpoints with strong, public key based authentication. Today\u0026rsquo;s IT environments can\u0026rsquo;t rely on firewalls to keep bad actors out - they must implement security at all layers within the stack - defense in depth. While we recognise that this brings a welcome relief to many security conscious administrators, we also know that managing certificate authorities (CAs) can be an unwelcome source of complexity. For this reason, StorageOS V2 implements an internal CA by default, to manage this complexity for you. If you\u0026rsquo;d prefer to integrate your own CA, we support that too - it\u0026rsquo;s up to you.\nFinally - our logging has undergone a complete transformation in this edition. We know that systems engineers and operators don\u0026rsquo;t just value headline features, but that observability and diagnostics are equally important. All logs are now decorated with rich context to help you understand what is happening within your cluster, and we\u0026rsquo;ll output in json by default, for easy ingestion into log aggregators such as Elasticsearch.\nUpgraded Data Plane Not to be outdone, our data plane contains some significant improvements.\nFirstly, we\u0026rsquo;ve completely re-written our sync algorithm (see Delta Sync, used when seeding or catching up replicas that have been offline or partitioned. Our new algorithm uses a Hash List to sync only changed sections of a volume (similar in some ways to what rsync does). StorageOS maintains these hashes during normal operation, meaning that when resyncing a failed replica, for example after a node reboot, we can very quickly and efficiently catch this replica up, rather than needing to promote and build a new one from scratch. This improves resiliency within your cluster, and prevents using excessive network bandwidth during failover conditions - at a time when it might be needed the most.\nSecondly, a new threading model, with dynamic pool sizing, means that StorageOS is faster, a lot faster. In our tests we observed improvements across the board, with improvements in throughput of up to 135% for some scenarios.\n","excerpt":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some …","ref":"/v2.0/docs/introduction/overview/","title":"What is StorageOS V2"},{"body":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source implementation of the SCSI target.\nWe require the following modules to be loaded:\n target_core_mod tcm_loop target_core_file configfs target_core_user uio N.B. Other applications utilising TCMU cannot be run concurrently with StorageOS. Doing so may result in corruption of data. On startup, StorageOS will detect if other applications are using TCMU and fall back to FUSE. However if StorageOS is started first there is no mechanism for StorageOS to fallback to FUSE if another application begins to use TCMU. TCMU can be disabled using the DISABLE_TCMU StorageOSCluster spec parameter.\n Depending on the distribution, the modules are shipped as part of the base kernel package or as part of a kernel extras package which needs to be installed.\nDistribution Specifics The following distributions are supported by default:\n RHEL 7.5 CentOS 7 Debian 9 Ubuntu Azure RancherOS - Note CSI is not supported on RancherOS Ubuntu 16.04/18.04 requires the installation of additional packages.\n N.B. Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not provide the necessary linux-image-extra package - see below for more information\n Ubuntu Package Installation Ubuntu 16.04/18.04 Generic and Ubuntu 16.04 GCE require extra packages:\nUbuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Ubuntu With AWS Or GCE Kernels Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not yet provide the linux-image-extra package. As such you should either use Debian, CentOS or RHEL, or install the non-cloud-provider optimised Ubuntu kernel.\nInstalling the non-cloud-provider optimised Ubuntu kernel is something that should only be done with full understanding of potential ramifications.\nsudo apt -y update sudo apt install -y linux-virtual linux-image-extra-virtual sudo apt purge -y linux*aws # Reboot the machine sudo shutdown -r now Automatic Configuration Once required kernel modules are installed on the system, for convenience we provide a container which will ensure the appropriate modules are loaded and ready for use at runtime. On Docker installations, you will need to run the init container prior to starting StorageOS. Our installation guides for Kubernetes and OpenShift include this step.\n# Load the required kernel modules. The Kubernetes and OpenShift installations include this step. docker run --name enable_lio \\ --privileged \\ --rm \\ --cap-add=SYS_ADMIN \\ -v /lib/modules:/lib/modules \\ -v /sys:/sys:rshared \\ storageos/init:0.2 Manual Configuration For those wishing to manage their own kernel configuration, rather than using the init container, perform the following steps:\n Ensure kernel modules are all loaded per list above Ensure configfs is loaded and mounted at /sys/kernel/config ","excerpt":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source …","ref":"/v2.0/docs/prerequisites/systemconfiguration/","title":"System Configuration"},{"body":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters of the StorageOSCluster custom resource and their default values.\n Parameter Description Default csi.deploymentStrategy CSI helper deployment strategy (statefulset or deployment) statefulset csi.enable Enable CSI setup false csi.enableControllerPublishCreds Enable CSI controller publish credentials false csi.enableNodePublishCreds Enable CSI node publish credentials false csi.enableProvisionCreds Enable CSI provision credentials false debug Enable debug mode for all the cluster nodes false disableFencing Disable Pod fencing false disableScheduler Disable StorageOS scheduler false disableTCMU Disable TCMU to allow co-existence with other TCMU users. Disabling TCMU degrades performance false disableTelemetry Disable telemetry reports false forceTCMU Forces TCMU to be enabled or causes StorageOS to abort startup false images.apiManagerContainer StorageOS API Manager container image storageos/api-manager:v1.0.0 images.csiClusterDriverRegistrarContainer CSI Cluster Driver Registrar Container image quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 images.csiExternalAttacherContainer CSI External Attacher Container image quay.io/k8scsi/csi-attacher:v1.0.1 images.csiExternalProvisionerContainer CSI External Provisioner Container image storageos/csi-provisioner:v1.0.1 ìmages.csiLivenessProbeContainer CSI Liveness Probe Container Image quay.io/k8scsi/livenessprobe:v1.0.1 images.csiNodeDriverRegistrarContainer CSI Node Driver Registrar Container image quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 images.hyperkubeContainer Deprecated field - HyperKube Container image Default dependent on Scheduler version images.initContainer StorageOS init container image storageos/init:2.1.0 images.kubeSchedulerContainer Kube scheduler container image Default dependent on Scheduler version images.nfsContainer StorageOS nfs container image storageos/nfs:1.0.0 images.nodeContainer StorageOS node container image storageos/node:v2.0.0 ingress.annotations Annotations of the ingress used by the cluster ingress.enable Enable ingress for the cluster false ingress.hostname Hostname to be used in cluster ingress storageos.local ingress.tls Enable TLS for the ingress false k8sDistro The name of the Kubernetes distribution is use, e.g. rancher or eks kvBackend.address Comma-separated list of addresses of external key-value store. (1.2.3.4:2379,2.3.4.5:2379) kvBackend.backend (v2 deprecated) Name of the key-value store to use. Set to etcd for external key-value store. embedded namespace Namespace where storageos cluster resources are created kube-system nodeSelectorTerms Set node selector for storageos pod placement pause Pause the operator for cluster maintenance false resources Set resource requirements for the containers secretRefName Reference name of storageos secret secretRefNamespace Namespace of storageos secret service.annotations Annotations of the Service used by the cluster service.externalPort External port of the Service used by the cluster 5705 service.internalPort Internal port of the Service used by the cluster 5705 service.name Name of the Service used by the cluster storageos service.type Type of the Service used by the cluster ClusterIP sharedDir Path to be shared with kubelet container when deployed as a pod /var/lib/kubelet/plugins/kubernetes.io~storageos storageClassName The name of the default StorageClass created for StorageOS volumes fast tlsEtcdSecretRefName Secret containing etcd client certificates tlsEtcdSecretRefNamespace Namespace of the tlsEtcdSecretRefName tolerations Set pod tolerations for storageos pod placement ","excerpt":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters …","ref":"/v2.0/docs/reference/cluster-operator/configuration/","title":"Configuration"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment of an etcd cluster.\n If you have installed OpenShift in AWS ensure that the requisite ports are opened for the worker nodes\u0026rsquo; security group.\n Make sure to add a StorageOS licence after installing.\n StorageOS v2 supports OpenShift v4. For more information check the OpenShift platform page.\nThe recommended way to run StorageOS on OpenShift is to deploy the StorageOS Cluster Operator using the Operator Lifecycle Manager (OLM) from the Red Hat OperatorHub, and bootstrap StorageOS using a Custom Resource. If that option is not suited for your use case, you can choose the \u0026ldquo;Manual\u0026rdquo; installation.\n Manual install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\nFirst StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment …","ref":"/v2.0/docs/install/openshift/","title":"OpenShift"},{"body":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all nodes in the cluster:\n Port Number TCP/UDP Use 5701 TCP gRPC 5703 TCP DirectFS 5704 TCP Dataplane Supervisor 5705 TCP REST API 5711 TCP \u0026amp; UDP Gossip service StorageOS also uses ephemeral ports to dial-out to these ports on other StorageOS nodes. For this reason, outgoing traffic should be enabled.\nFirewalls and VPS providers Some VPS providers (such as Digital Ocean) ship default firewall rulesets which must be updated to allow StorageOS to run. Some example rules are shown below - modify to taste.\nUFW For distributions using UFW, such as RHEL and derivatives:\nufw default allow outgoing ufw allow 5701:5711/tcp ufw allow 5711/udp Firewalld For distributions that enable firewalld to control iptables such as some installations of OpenShift.\nfirewall-cmd --permanent --new-service=storageos firewall-cmd --permanent --service=storageos --add-port=5700-5800/tcp firewall-cmd --add-service=storageos --zone=public --permanent firewall-cmd --reload Iptables For those using plain iptables:\n# Inbound traffic iptables -I INPUT -i lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I INPUT -m state --state ESTABLISHED,RELATED -m comment --comment \u0026#39;Permit established traffic\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 5701:5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p udp --dport 5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT # Outbound traffic iptables -I OUTPUT -o lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I OUTPUT -d 0.0.0.0/0 -m comment --comment \u0026#39;Permit outbound traffic\u0026#39; -j ACCEPT Please ensure that the iptables rules you have added above come before any default DROP or REJECT rules.\n","excerpt":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all …","ref":"/v2.0/docs/prerequisites/firewalls/","title":"Firewalls"},{"body":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, OpenShift or Rancher.\n Dive into our Use Case documentation for Workloads deployed in Kubernetes.\n ","excerpt":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, …","ref":"/v2.0/docs/introduction/quickstart/","title":"Quickstart"},{"body":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and Password in base64 encoding.\nkubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;default\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z END This example contains a default password, for production installations, use a unique, strong password.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Create a cluster-config.yaml according to your needs from the examples below.\nkubectl create -f cluster-config.yaml Note that StorageOS will be deployed in spec.namespace (storageos by default), irrespective of what NameSpace the CR is defined in.\n Examples You can checkout all the parameters configurable in the configuration page.\n All examples must reference the storageos-api Secret.\nspec:secretRefName:\u0026#34;storageos-api\u0026#34;# Reference to the Secret created in the previous stepsecretRefNamespace:\u0026#34;storageos-operator\u0026#34;# Namespace of the SecretCheck out Cluster Definition examples for full CR files.\nInstalling with an external etcd spec:kvBackend:address:\u0026#39;10.43.93.95:2379\u0026#39;# IP of the SVC that exposes ETCD# address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can specify individual IPs of the etcd serversInstalling to a subset of nodes In this case we select nodes that are workers. To make sure that StorageOS doesn\u0026rsquo;t start in Master nodes.\nYou can see the labels in the nodes by kubectl get node --show-labels.\nspec:nodeSelectorTerms:- matchExpressions:- key:\u0026#34;node-role.kubernetes.io/worker\u0026#34;operator:Invalues:- \u0026#34;true\u0026#34;# OpenShift uses \u0026#34;node-role.kubernetes.io/compute=true\u0026#34;# Rancher uses \u0026#34;node-role.kubernetes.io/worker=true\u0026#34;# Kops uses \u0026#34;node-role.kubernetes.io/node=\u0026#34; Different provisioners and Kubernetes distributions use node labels differently to specify master vs workers. Node Taints are not enough to make sure StorageOS doesn\u0026rsquo;t start in a node. The JOIN variable is defined by the operator by selecting all the nodes that match the nodeSelectorTerms.\n Enabling CSI spec:csi:enable:truedeploymentStrategy:deploymentenableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:trueThe credentials must be defined in the storageos-api Secret\napiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# Add base64 encoded creds below for CSI credentials.csiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zSpecifying a shared directory for use with kubelet as a container spec:sharedDir:\u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39;Defining pod resource requests and reservations spec:resources:requests:memory:\u0026#34;512Mi\u0026#34;# cpu: \u0026#34;1\u0026#34;# limits:# memory: \u0026#34;4Gi\u0026#34;Limiting StorageOS can cause malfunction for IO to StorageOS volumes, therefore we do not currently recommend applying upper limits to resources for StorageOS pods.\nSpecifying custom Tolerations spec:tolerations:- key:\u0026#34;key1\u0026#34;operator:\u0026#34;Equal\u0026#34;value:\u0026#34;value1\u0026#34;effect:\u0026#34;EffectToTolerate\u0026#34;- key:\u0026#34;key2\u0026#34;operator:\u0026#34;Exists\u0026#34;Custom tolerations specified in the StorageOSCluster definition are added to all StorageOS components; the StorageOS daemonset, CSI helper and scheduler.\nIn the above example a toleration key1=value1:EffectToTolerate would be tolerated and key2 would be tolerated regardless of the value and effect. For more information about tolerations please see the Kubernetes documentation.\n","excerpt":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and …","ref":"/v2.0/docs/reference/cluster-operator/examples/","title":"Cluster Operator examples"},{"body":" Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by default.\n StorageOS requires mount propagation enabled to present devices as volumes for containers (see linux kernel documentation here).\nCertain versions of docker ship with a systemd manifest with MountFlags set to \u0026lsquo;slave\u0026rsquo;, thus preventing StorageOS from working. This can be removed or set to \u0026lsquo;shared\u0026rsquo; with a systemd drop in:\nmkdir -p /etc/systemd/system/docker.service.d/ cat \u0026lt;\u0026lt;EOF \u0026gt; /etc/systemd/system/docker.service.d/mount_propagation_flags.conf [Service] MountFlags=shared EOF # systemctl daemon-reload # systemctl restart docker.service To confirm behaviour, the following command should run without error.\ndocker run -it --rm -v /mnt:/mnt:shared busybox sh -c /bin/date Orchestrators such as Kubernetes or OpenShift have their own ways of exposing this setting. Kubernetes 1.10 and OpenShift 3.10 have mount propagation enabled by default. Previous versions require that feature gates are enabled on the Kubernetes master\u0026rsquo;s controller-manager and apiserver services and in the kubelet service on each node.\nInstallations of orchestrators using Docker require that mount propagation is enabled for both.\nRefer to our installation pages for the orchestrators to see details on how to check and enable mount propagation where appropriate.\n","excerpt":"Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by …","ref":"/v2.0/docs/prerequisites/mountpropagation/","title":"Mount Propagation"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Make sure to add a StorageOS licence after installing.\n StorageOS transparently supports Rancher deployments on CentOS, RHEL, Debian, Ubuntu or RancherOS (CSI is not supported on RancherOS) and can support other Linux distributions as detailed on the System Configuration page if the appropriate kernel modules are available.\n The StorageOS v2 installation using the Rancher Catalog will be available soon.\n Manual Manual Installation The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Make sure to add a …","ref":"/v2.0/docs/install/rancher/","title":"Rancher"},{"body":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following yaml manifest.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n Upgrade StorageOS Operator using Helm If you have installed the StorageOS Operator using the Helm Chart, then you can upgrade the operator using the following commands.\n$ helm list NAME REVISION STATUS CHART APP VERSION NAMESPACE storageos-v1 4 DEPLOYED storageos-operator-0.2.11 1.3.0 storageos-operator $ helm repo update $ helm upgrade $NAME storageos/storageos-operator When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n ","excerpt":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following …","ref":"/v2.0/docs/reference/cluster-operator/upgrade/","title":"Cluster Operator Upgrade"},{"body":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 active volumes per node 4.x kernels have a limitation of 4096 active volumes per node We are distribution agnostic as long as our prerequisites are met Orchestrators Kubernetes 1.13+ OpenShift 4.0+ ","excerpt":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 …","ref":"/v2.0/docs/introduction/platforms/","title":"Supported Platforms and Orchestrators"},{"body":"","excerpt":"","ref":"/v2.0/docs/concepts/","title":"Concepts"},{"body":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a 64-bit architecture. Kubernetes 1.9+ with CSI. Container Runtime Engine: CRI-O, Containerd or Docker 1.10+ with mount propagation enabled. The necessary ports should be open. See the ports and firewall settings. Etcd cluster for StorageOS A mechanism for device presentation. Recommended: At least three nodes for replication and high availability. Kubernetes 1.13+. Install the storageos CLI. If using Helm2, make sure the tiller ServiceAccount has enough privileges to create resources such as Namespaces, ClusterRoles, etc. For instance, following this installation procedure. System clocks synchronized using NTP or similar methods. While our distributed consensus algorithm does not require synchronised clocks, it does help to more easily correlate logs across multiple nodes. ","excerpt":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a …","ref":"/v2.0/docs/prerequisites/","title":"Prerequisites"},{"body":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different distributions such as OpenShift, Rancher, EKS, AKS, GKE, etc. The user can provide standard PVC definitions and StorageOS will dynamically provision matching volumes. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nKubernetes and StorageOS communicate with each other to perform actions such as creation, deletion or mounting of volumes. The CSI (Container Storage Interface) driver is the standard method of communication. Using CSI, Kubernetes and StorageOS communicate over a Unix domain socket.\nIt is recommended to use the CSI StorageOS installation from Kubernetes 1.13 onwards. Although the former communication procedure, the native driver, is still in use and StorageOS maintains support for it, its use is discouraged as it is a deprecated method, to be removed in newer versions of Kubernetes.\nCSI (Container Storage Interface) Note CSI is the standard method of communication that enables storage drivers for Kubernetes to release on their own schedule. The CSI standard allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI was released GA from Kubernetes 1.13. StorageOS v2 only supports the use of CSI as a storage driver. In addition, the StorageOS Cluster Operator handles the configuration of the CSI driver and its complexity by detecting the version of the Kubernetes installed.\nCheck out the status of the CSI release cycle in relation with Kubernetes in the CSI project page.\nCSI communication is fully supported by StorageOS if the cluster is deployed with any supported Linux Distribution.\nKubernetes Upgrades on Managed Services Managed services that support in place upgrades are fully supported. However, upgrading Kubernetes using green/blue deployments is not supported. This is because nodes are replaced rather than being upgraded. Any data stored on the nodes is lost when new nodes replace the previous ones.\nDocker Some managed Kubernetes platforms such as Azure AKS, enable the \u0026lsquo;Live-Restore\u0026rsquo; Docker feature, enabling containers to continue running while Docker is stopped or upgraded. This feature can cause nodes to hang while shutting-down or rebooting, as rather than going through an orderly shutdown, StorageOS (and other processes) are killed before the disks are synced and unmounted. Devices in this inaccessible state will log a warning similar to:\nTransport endpoint not connected To prevent this behaviour, we advise disabling this feature by setting\n{ ... \u0026quot;live-restore\u0026quot;: false } in /etc/docker/daemon.json.\nHere\u0026rsquo;s an example Ansible snippet that might be used to achieve this\n... - name: configure /etc/docker/daemon.json lineinfile: path: /etc/docker/daemon.json regexp: '^.*\u0026quot;live-restore\u0026quot;: true,$' line: ' \u0026quot;live-restore\u0026quot;: false,' backrefs: yes notify: restart docker ... Note: Use at own risk; you may need to adapt the example to work in your environment\n ","excerpt":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different …","ref":"/v2.0/docs/install/","title":"Install"},{"body":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to a specific platform.\nStorageOS can be used in any Kubernetes distribution when the prerequisites are met. That includes Kubernetes manged services.\n","excerpt":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to …","ref":"/v2.0/docs/platforms/","title":"Platforms"},{"body":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential cluster metadata and manage distributed configuration state. An embedded etcd instance is included in the StorageOS container, but for production environments and testing of production workloads, we recommend deploying an external etcd cluster. For more details about, and an example of, how to run etcd, see the External etcd Operations page.\nIt is highly recommended to use external etcd for cloud environments and place the etcd cluster on stable nodes. Placing the etcd on nodes that are recycled often might affect the normal operations of StorageOS.\nSetup of storage on the hosts We recommend creating a separate filesystem for StorageOS to mitigate the risk of filling the root filesystem on nodes. This has to be done for each node in the cluster.\nFollow the managing host storage best practices page for more details.\nResource reservations StorageOS resource consumption depends on the workloads and the StorageOS features in use.\nThe recommended minimum memory reservation for the StorageOS Pods is 512MB for non-production environments. However it is recommended to prepare nodes so StorageOS can operate with at least with 1-2GB of memory. StorageOS frees memory when possible.\nFor production environments, we recommend 4GB of Memory and 1 CPU as a minimum and to test StorageOS using realistic workloads and tune resources accordingly.\nStorageOS Pods resource allocation will impact directly on the availability of volumes in case of eviction or resource limit triggered restart. It is recommended to not limit StorageOS Pods.\nStorageOS implements a storage engine, therefore limiting CPU consumption might affect the I/O throughput of your volumes.\nStorageOS API username/password The API grants full access to StorageOS functionality, therefore we recommend that the default administrative password of \u0026lsquo;storageos\u0026rsquo; is reset to something unique and strong.\nYou can change the default parameters by encoding the apiUsername and apiPassword values (in base64) into the storageos-api secret.\nTo generate a unique password, a technique such as the following, which generates a pseudo-random 24 character string, may be used:\n# Generate strong password PASSWORD=$(cat -e /dev/urandom | tr -dc \u0026#39;a-zA-Z0-9-!@#$%^\u0026amp;*()_+~\u0026#39; | fold -w 24 | head -n 1) # Convert password to base64 representation for embedding in a K8S secret BASE64PASSWORD=$(echo -n $PASSWORD | base64) Note that the Kubernetes secret containing a strong password must be created before bootstrapping the cluster. Multiple installation procedures use this Secret to create a StorageOS account when the cluster first starts.\nStorageOS Pod placement StorageOS must run on all nodes that will contribute storage capacity to the cluster or that will host Pods which use StorageOS volumes. For production environments, it is recommended to avoid placing StorageOS Pods on Master nodes.\nStorageOS is deployed with a DaemonSet controller, and therefore tolerates the standard unschedulable (:NoSchedule) action. If that is the only taint placed on master or cordoned nodes StorageOS pods might start on them (see the Kubernetes docs for more details). To avoid scheduling StorageOS pods on master nodes, you can add an arbitrary taint to them for which the StorageOS DaemonSet won\u0026rsquo;t have a toleration.\nDedicated instance groups Cloud environments give users the ability to quickly scale the number of nodes in a cluster in response to their needs. Because of the ephemeral nature of the cloud, StorageOS recommends setting conservative downscaling policies.\nFor production clusters, it recommended to use dedicated instance groups for Stateful applications that allow the user to set different scaling policies and define StorageOS pools based on node selectors to collocate volumes.\nLosing a few nodes at the same time could cause the loss of data even when volume replicas are being used.\nPort blocking StorageOS exposes ports to operate. It is recommended that the ports are not accessible from outside the scope of your cluster.\nStorageOS in Docker EE StorageOS does not support running on Swarm nodes nor on mixed (Kubernetes and Swarm) nodes. StorageOS volumes have to be provisioned and used from Kubernetes nodes.\n","excerpt":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential …","ref":"/v2.0/docs/best-practices/","title":"StorageOS Best Practices"},{"body":"","excerpt":"","ref":"/v2.0/docs/operations/","title":"Operations"},{"body":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or other Orchestrators that are derived from Kubernetes such as OpenShift or Rancher. This is useful for running stateful applications, such as databases or CI/CD applications, under the control of Kubernetes as Kubernetes can make scheduling decisions without the application data being lost.\nWhat we have outlined in the cookbooks below are some quick deployments of stateful applications into a Kubernetes cluster. These examples are not production ready but have been provided to give you some insight into how to use StorageOS with stateful applications.\nStatefulSets The examples we have provided use StatefulSets as a way to deploy applications into Kubernetes. The reason for this is that the StatefulSet controller is designed to manage stateful applications and it \u0026ldquo;provides guarantees about the ordering and uniqueness\u0026rdquo; of sets of pods.\nPractically this means that when a StatefulSet scales, pods are created in order from 0-(N-1), if a StatefulSet scales down then pods are deleted in reverse order from (N-1)-0.\nSecondly, it means that the behaviour of the StatefulSet controller differs from that of Deployment and ReplicaSet controllers. For Deployment and ReplicaSet controllers \u0026ldquo;\u0026hellip; at many points in the lifetime of a replica set there will be 2 copies of a pod\u0026rsquo;s processes running concurrently\u0026rdquo;. Having two different pods mount a volume at the same time can cause corruption of data. Currently Kubernetes accessModes only apply restrictions to nodes mounting volumes rather than pods, so it is important that StatefulSets are used with StorageOS volumes so the necessary pod uniqueness guarantees are maintained.\nStatefulSet Manifests The StorageOS specific part of the Kubernetes manifests for these examples lies in the VolumeClaimTemplate that\u0026rsquo;s part of the statefulset definition.\nVolumeClaimTemplate\napiVersion:apps/v1kind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1template:metadata:labels:app:mssqlenv:prodspec:serviceAccountName:mssqlcontainers:- name:fooimage:barvolumeMounts:- name:bazmountPath:/var/opt/barenvFrom:- configMapRef:name:mssqlvolumeClaimTemplates:- metadata:name:bazlabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiIn the StatefulSet definition above the container has a volume mount defined called baz. The definition for this volume is found in the VolumeClaimTemplate where the fast storageClass will be used to dynamically provision storage if the persistent volume does not already exist.\n","excerpt":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or …","ref":"/v2.0/docs/usecases/","title":"Use Cases: Install applications on Kubernetes with StorageOS"},{"body":"","excerpt":"","ref":"/v2.0/docs/reference/","title":"Reference"},{"body":"There are several ways to reach us if you require support. The fastest way to get in touch is to join our public Slack channel. \nYou can file a support ticket via email to support@storageos.com, or use our Help Desk portal\nTo help us provide effective support, we request that you provide as much information as possible when contacting us. The list below is a suggested starting point. Additionally, please include anything specific, such as log entries, that may help us debug your issue.\nInformation about the cluster can be automatically sent to StorageOS engineers as mentioned in the section StorageOS Diagnostic Bundle.\nPlatform Cloud provider/Bare metal OS distribution and version Kernel version docker version and installation procedure (distro packages or docker install) StorageOS Version of StorageOS storageos get nodes storageos get volumes storageos describe volume VOL_ID # in case of issues with a specific volume Orchestrator related (Kubernetes, OpenShift, etc) Version and installation method Managed or self managed? kubectl -n kube-system get pod kubectl -n kube-system logs -lapp=storageos -c storageos kubectl -n kube-system get storageclass Specific for your namespaces: kubectl describe pvc PVC_NAME Specific for your namespaces: kubectl describe pod POD_NAME StorageOS Cluster Diagnostic Bundle StorageOS has a cluster diagnostic function that aggregates cluster information.\nThe cluster diagnostic bundle is created only when a user chooses to do so. For convenience the cluster diagnostic bundle can either be uploaded to StorageOS or downloaded to the machine running the browser or CLI. The cluster diagnostic bundle is uploaded from a StorageOS node to a StorageOS GCP encrypted bucket using a TLS encrypted connection. The upload takes place only after user confirmation.\nAs a general outline the following information is sent to us:\n StorageOS logs StorageOS cluster state - node config, volumes and namespaces output of storageos describe node output of lshw output of dmesg See Support Bundle for an exhaustive list of what is included in the cluster diagnostic bundle.\nStorageOS engineers might ask for a cluster diagnostic bundle to be generated during support cases.\nThe information given in the cluster cluster diagnostic bundle is only used for support purposes and it will be removed once the data is no longer needed for such purposes. In case the information is sensitive and can\u0026rsquo;t be given to StorageOS, please make sure that support engineers have as much information about your environment as possible.\nYou can generate a cluster diagnostic bundle through the StorageOS GUI by navigating to the Cluster menu and downloading the StorageOS support Bundle. The cluster cluster diagnostic bundle can also be generated using the CLI command below.\n$ storageos get diagnostics -o storageos-diagnostic-bundle ","excerpt":"There are several ways to reach us if you require support. The fastest way to get in touch is to …","ref":"/v2.0/docs/support/","title":"Support"},{"body":"If you have any specific or more complex requirements please contact StorageOS as we\u0026rsquo;d be happy to organise a POC in conjunction with our Engineering team. You can join our slack channel or email us at: info@storageos.com\nSupport for Self Evaluations Should you have questions or require support, there are several ways to get in touch with us. The fastest way to get in touch is to join our public Slack channel. You can also get in touch via email to info@storageos.com.\nFurthermore you can fill out the form below and we will get in touch.\n hbspt.forms.create({ portalId: \"3402546\", formId: \"a07fecd3-ce5b-4835-b136-51a94a35632b\", sfdcCampaignId: \"70158000000BAZzAAO\" }); Installation The first phase of the self-evaluation is to install StorageOS. This section of the document aims to layout what options are exposed to you during installation and why some options may be preferable to you over others.\nA standard StorageOS installations uses the StorageOS operator, so as much of the necessary configuration is handled for you. The StorageOS operator has been certified by Red Hat and is open source.\nStorageOS Operator The StorageOS operator is a Kubernetes native application that manages the StorageOS cluster lifecycle. It simplifies cluster installation, cluster removal and other operations.\nThe StorageOS operator watches for the creation of StorageOSCluster Custom Resources. A StorageOSCluster is a declarative representation of a StorageOS cluster. For example if CSI is enabled in the StorageOSCluster resource, a StorageOS cluster will be created that uses the CSI driver.\nStorageOS Operator features Specific configuration options for the StorageOS Operator that we believe to be important during a self-evaluation will be laid out in this guide.\nA set of example StorageOSCluster are listed here. For an exhaustive list of configuration settings for the StorageOS operator please see our documentation.\nExternal Etcd StorageOS highly recommends that an etcd cluster on standalone nodes outside the Kubernetes cluster is used for production deployments. In this configuration the etcd cluster would run on separate boxes from the rest of the Kubernetes and StorageOS cluster ensuring stability and resilience of the etcd cluster. However for the purposes of a self-evaluation it is acceptable to run etcd as a container inside Kubernetes.\nWe do not recommend running etcd on the same nodes as StorageOS when node failure will be tested, as if the majority of etcd nodes fail then the etcd cluster cannot be recovered automatically. Therefore it is better to run etcd on separate nodes.\nPrerequisites StorageOS has some prerequisites that must be met to complete a successful installation\n Machines intended to run StorageOS have at least 1 CPU core, 2GB RAM Docker 1.10 or later with mount propagation enabled TCP ports 5701-5710 and TCP \u0026amp; UDP 5711 open between all nodes in the cluster A 64bit supported operating system - By default StorageOS supports Debian 9, RancherOS, RHEL7.5 and CentOS7. Note Ubuntu 16.04 and 18.04 are supported but additional packages are required. Ubuntu 16.04/18.04 with the AWS kernel and Ubuntu 18.04 with the GCE kernel do not provide the required packages and are therefore NOT supported.\n To install the required kernel modules on Ubuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) To install the required kernel modules on Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Installing StorageOS Installation steps are as follows:\n Install etcd Install StorageOS Operator Create a Kubernetes secret detailing the default StorageOS administrator account Install StorageOS using a StorageOSCluster Custom Resource Install etcd In order to get a StorageOS cluster stood up quickly, a single node etcd cluster can be installed in Kubernetes, on a Kubernetes master. The reason for installing on a master is that master nodes generally have predictable lifetimes and low Pod scheduling churn. As such there is a lesser risk of the etcd pod being evicted ensuring a stable etcd cluster.\nNote that if the etcd pod is stopped for any reason the etcd cluster will cease to function pending manual intervention. Please take this into account during testing of failure scenarios.\n Download repo\n$ git clone https://github.com/coreos/etcd-operator.git Configure NS, Role and RoleBinding\n$ export ROLE_NAME=etcd-operator $ export ROLE_BINDING_NAME=etcd-operator $ export NAMESPACE=etcd Create Namespace\n$ kubectl create namespace $NAMESPACE Deploy Operator\n$ ./etcd-operator/example/rbac/create_role.sh $ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END The Kubernetes masters should then be labelled so a nodeSelector can be used in the EtcdCluster manifest\n$ kubectl label nodes \u0026lt;NODES\u0026gt; etcd-cluster=storageos-etcd Once the master node taints are known and the nodes have been labelled you can deploy an EtcdCluster manifest that contains tolerations for all taints on the master nodes and selects for the node label applied in the previous step. A sample manifest is below. Edit the size to match the number of masters you will deploy on and edit the tolerations to match all taints on the master nodes where etcd will be deployed.\n Create the EtcdCluster resource\n$ kubectl -n etcd create -f - \u0026lt;\u0026lt;END apiVersion: \u0026#34;etcd.database.coreos.com/v1beta2\u0026#34; kind: \u0026#34;EtcdCluster\u0026#34; metadata: name: \u0026#34;storageos-etcd\u0026#34; spec: size: 1 version: \u0026#34;3.4.7\u0026#34; pod: etcdEnv: - name: ETCD_QUOTA_BACKEND_BYTES value: \u0026#34;2147483648\u0026#34; # 2 GB - name: ETCD_AUTO_COMPACTION_RETENTION value: \u0026#34;100\u0026#34; # Keep 100 revisions - name: ETCD_AUTO_COMPACTION_MODE value: \u0026#34;revision\u0026#34; # Set the revision mode resources: requests: cpu: 200m memory: 300Mi securityContext: runAsNonRoot: true runAsUser: 9000 fsGroup: 9000 tolerations: - operator: \u0026#34;Exists\u0026#34; nodeSelector: etcd-cluster: storageos-etcd affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster operator: In values: - storageos-etcd topologyKey: kubernetes.io/hostname END Install StorageOS Operator In order to install the StorageOS operator download the requisite yaml manifests or apply them with kubectl.\n$ kubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml You can verify the operator is running using the following command\n$ kubectl get pods -n storageos-operator Install StorageOS Once the StorageOS operator has been installed a StorageOS cluster can be generated by creating a StorageOSCluster resource.\nA StorageOSCluster resource describes the state of the StorageOS cluster that is desired and the StorageOS operator will create the desired StorageOS cluster. For examples of StorageOSCluster resources please see our examples page here. For a full list of the configurable spec parameters of the StorageOSCluster resource please see here.\n Create a secret defining the API username and password\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zEND Create a StorageOSCluster resource\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:example-storageosclusternamespace:\u0026#34;storageos-operator\u0026#34;spec:secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;storageos-operator\u0026#34;k8sDistro:\u0026#34;upstream\u0026#34;# Set the Kubernetes distribution for your cluster (upstream, eks, aks, gke, rancher, dockeree)# storageClassName: fast # The storage class creates by the StorageOS operator is configurablecsi:enable:truedeploymentStrategy:\u0026#34;deployment\u0026#34;enableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:truekvBackend:address:\u0026#34;storageos-etcd-client.etcd.svc.cluster.local:2379\u0026#34;END Confirm that the cluster has been created and that StorageOS pods are running\n$ kubectl -n kube-system get pods StorageOS pods enter a ready state after a minimum of 65s has passed.\n Deploy the StorageOS CLI as a container\n$ kubectl -n kube-system run \\ --image storageos/cli:v2.0.0 \\ --restart=Never \\ --env STORAGEOS_ENDPOINTS=storageos:5705 \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sleep 999999 License cluster\n A newly installed StorageOS cluster does not include a licence. A cluster must be licensed within 24 hours of the installation. For more information, check the reference licence page.\n You can apply a Free Developer licence by following the operations licensing page, or purchase a licence by contacting sales@storageos.com.\n Confirm that StorageOS is working by creating a PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Verify that the CLI is working. pvc-1 should be listed in the CLI output\n$ kubectl -n kube-system exec -it cli -- storageos get volumes The StorageOS web UI can also be used to display information about the state of the cluster. The StorageOS UI can be accessed on any node that is running a StorageOS pod on port 5705. The username/password for the UI is defined by the storageos-api secret. For this self-evaluation the username/password is storageos:storageos\nhttp://\u0026lt;NODE_IP\u0026gt;:5705 Create a pod that consumes the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Check that the pod starts successfully. If the pod starts successfully then the StorageOS cluster is working correctly\n$ kubectl get pod d1 -w The pod mounts a StorageOS volume under /mnt so any files written there will persist the lifetime of the pod. This can be demonstrated using the following commands.\n Execute a shell inside the pod and write some data to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo Hello World! \u0026gt; /mnt/hello root@d1:/# cat /mnt/hello Hello World! Delete the pod\n$ kubectl delete pod d1 Recreate the pod\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Open a shell inside the pod and check the contents of /mnt/hello\n$ kubectl exec -it d1 -- cat /mnt/hello Hello World! Now that StorageOS has been successfully installed, the cluster has a standard license by default which allows for the creation of 50GB of persistent volumes. If you register the cluster then a developer license will be applied and 500GB of persistent volumes can be created. Replicas do not count towards the license total so a 500GB license could be used to created a 500GB volume with 5 replicas. For the purposes of this self-evaluation the standard license is sufficient.\n StorageOS Features Now that you have a correctly functioning StorageOS cluster we will explain some of our features that may be of use to you as you complete application and synthetic benchmarks.\nStorageOS features are all enabled/disabled by applying labels to volumes. These labels can be passed to StorageOS via persistent volume claims (PVCs) or can be applied to volumes using the StorageOS CLI or GUI.\nThe following is not an exhaustive feature list but outlines features which are commonly of use during a self-evaluation.\nVolume Replication StorageOS enables synchronous replication of volumes using the storageos.com/replicas label.\nThe volume that is active is referred to as the master volume. The master volume and its replicas are always placed on separate nodes. In fact if a replica cannot be placed on a node without a replica of the same volume, the volume will fail to be created. For example, in a three node StorageOS cluster a volume with 3 replicas cannot be created as the third replica cannot be placed on a node that doesn\u0026rsquo;t already contain a replica of the same volume.\nThe failure mode for a volume affects how many failed replicas can be tolerated before the volume is marked as offline. Replicas are also segregated according to the iaas/failure-domains node label. StorageOS will automatically place a master volume and its replicas in separate failure domains where possible.\nSee our replication documentation for more information on volume replication.\n To test volume replication create the following PersistentVolumeClaim\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-replicated labels: storageos.com/replicas: \u0026#34;1\u0026#34; spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Note that volume replication is enabled by setting the storageos.com/replicas label on the volume.\n Confirm that a replicated volume has been created by using the StorageOS CLI or UI\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that uses the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: replicated-pod spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-replicated END Write data to the volume\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# echo Hello World! \u0026gt; /mnt/hello root@replicated-pod:/# cat /mnt/hello Hello World! Find the location of the master volume and shutdown the. Shutting down a node causes all volumes, with online replicas, on the node to be evicted. For replicated volumes this immediately promotes a replica to become the new master.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-replicated Bound pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 5Gi RWO fast 1m $ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker (online) kind-worker2 1/1 26 seconds ago Check the location of the master volume and notice that it is on a new node. If the pod that mounted the volume was located on the same node that was shutdown then the pod will need to be recreated.\n$ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker2 (online) kind-worker2 1/1 46 seconds ago Check that the data is still accessible to the pod\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# cat /mnt/hello Hello World! Benchmarking As a rule the best performance is obtained by using unreplicated volumes that are co-located with the application or benchmarking tool writing to the volume. See Volume Placement below for more information.\nWhen running benchmarks in the cloud, benchmarks need to be run multiple times and nodes should be destroyed and recreated so that the underlying machine changes. This should be done to reduce the impact that noisy neighbours might have on benchmark results.\nConsiderations Application vs StorageOS replication Certain applications are able to natively replicate or shard data between application instances. When using these applications it is worth considering whether application replication, StorageOS replication or a mixture of both should be used.\nWhen StorageOS replication is enabled the time to recover in cases of node failure can be lessened. This is because StorageOS will promote a replica, Kubernetes will reschedule the application instance and the amount of data the application needs to catch up on is limited to whatever data was modified while the application was being rescheduled. Without StorageOS replication the application would have to rebuild an entire copy of data. Some applications have their performance greatly impacted by having to rebuild shards/replicas so this is also avoided.\nStatefulSets StatefulSets are the de facto controller for stateful applications. As such, when deploying applications that will use StorageOS volumes, StatefulSets should be used. You can find more information about StatefulSets here.\nVolume Placement StorageOS volumes give the best performance when the application pod and the master volume are co-located on the same node. When benchmarking applications, it is useful to take into account that using remote volumes and replicas impact the overall performance of a volume.\nGoing from 0 to 1 replica has the greatest performance impact for writes as now the latency of the operation is equal to the round trip time to the node with the replica over the network. Adding additional replicas poses less of a performance impact as writes to replicas are done in parallel, and the round trip time to each node is unlikely to greatly increase unless replicas land on nodes that are geographically distant to the master volumes\u0026rsquo; node.\nEven when volumes are replicated co-location of pod and master volume is still desirable because application writes are first sent to the master and then sent from the master volume to the replicas. Writing to a local master therefore saves network latency between the application and the master volume. As reads are always served from the master volume a remote master volume will add latency to reads as well as writes.\nWhen testing applications, such as databases, it is also necessary to run benchmarks for a sufficiently long time to account for caching, and cache flushing that databases do. We recommend running application benchmarks over a 20-30min period for this reason.\nHow to land a volume and a pod on the same node StorageOS has an automatic co-location feature on our development roadmap that we are calling pod locality. Until the feature is GA co-location of a master volume and a pod can be achieved by leveraging existing StorageOS and Kubernetes features.\nstorageos.com/hint.master is a volume label that influences the placement of a StorageOS master volume. By setting this label to the StorageOS nodeID that corresponds to the nodeSelector on a StatefulSet or Pod the master volume and the pod should co-locate on the same node. You can reference our FIO local volumes job for an example of how to do this. Synthetic Benchmarks Synthetic benchmarks using tools such as FIO are a useful way to begin measuring StorageOS performance. While not fully representative of application performance, they allow us to reason about the performance of storage devices without the added complexity of simulating real world workloads, and provide results easily comparable across platforms.\nAs with application benchmarks, when testing in public clouds multiple runs on newly created nodes should be considered to account for the impact of noisy neighbours.\nStorageOS has created a test suite for running FIO tests against StorageOS volumes that can be found here. The test suite can be deployed into a Kubernetes cluster using the instructions below.\n Clone the StorageOS use cases repo. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/FIO/local-volumes.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the FIO local-volumes folder\n$ cd storageos-usecases/FIO/local-volumes Get the name of the node that you wish the FIO pods and volumes to be created on. Make sure that the node name and the label kubernetes.io/hostname match and that the node has enough storage capacity to create 8Gi worth of volumes\n$ kubectl get node --show-labels Generate the FIO jobs by passing in the node name that the job should run on, and the StorageOS node id for that node. The number is the number of volumes that FIO will test concurrently. The StorageOS node ID can be found using the cli to describe node\n$ ./job-generator-per-volumecount.sh 4 $NODE_NAME $STORAGEOS_NODEID Upload the FIO profiles as ConfigMaps\n$ ./upload-fio-profiles.sh Run the FIO tests\n$ kubectl create -f ./jobs Check the PVCs have been provisioned\n$ kubectl get pvc Use the StorageOS CLI to check the location of the volumes\n$ kubectl -n kube-system exec cli -- storageos get volumes Verify that the Pod is running on the same node\n$ kubectl get pod -owide FIO has a number of parameters that can be adjusted to simulate a variety of workloads and configurations. Particularly the queue depth, block size and the number of volumes used affect the FIO results. To tune the FIO parameters the profiles file can be edited or the ConfigMap that is created from the profiles file can be edited directly.\nStorageOS configuration also affects the overall volume performance. For example adding a replica to a volume will increase the latency for writes and affect IOPS and bandwidth for the volume.\nTo see the effect a StorageOS replica has on performance rerun an FIO test but add the storageos.com/replicas: \u0026quot;1\u0026quot; label to the PersistentVolumeClaims in the jobs spec. The greatest performance impact from adding replicas comes when moving from 0 to 1 replica. Adding additional replicas does not incur a significant performance penalty.\nThe remote volumes folder contains a guide for performing the same FIO tests against remote volumes. Application Benchmarks While synthetic benchmarks are useful for examining the behaviour of StorageOS with very specific workloads, in order to get a realistic picture of StorageOS performance actual applications should be tested.\nMany applications come with test suites which provide standard workloads. For best results, test using your application of choice with a representative configuration and real world data.\nAs an example of benchmarking an application the following steps lay out how to benchmark a Postgres database backed by a StorageOS volume.\n Start by cloning the StorageOS use cases repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/pgbench.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the Postgres examples folder\n$ cd storageos-usecases/pgbench Decide which node you want the pgbench pod and volume to be located on. The node needs to be labelled app=postgres\n$ kubectl label node \u0026lt;NODE\u0026gt; app=postgres Then set the storageos.com/hint.master label in 20-postgres-statefulset.yaml file to match the StorageOS nodeID for the node you have chosen before creating all the files. The StorageOS nodeID can be obtained using the cli and doing a describe node\n$ kubectl create -f . Confirm that Postgres is up and running\n$ kubectl get pods -w -l app=postgres Use the StorageOS CLI or the GUI to check the master volume location and the mount location. They should match\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Exec into the pgbench container and run pgbench\n$ kubectl exec -it pgbench -- bash -c \u0026#39;/opt/cpm/bin/start.sh\u0026#39; Conclusion After completing these steps you will have benchmark scores for StorageOS. Please keep in mind that benchmarks are only part of the story and that there is no replacement for testing actual production or production like workloads.\nStorageOS invites you to provide feedback on your self-evaluation to the slack channel or by directly emailing us at info@storageos.com\n","excerpt":"If you have any specific or more complex requirements please contact StorageOS as we\u0026rsquo;d be …","ref":"/v2.0/docs/self-eval/","title":"Self Evaluation Guide"},{"body":"","excerpt":"","ref":"/v2.0/index.json","title":""},{"body":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: '#ohpen_swagger_ui', presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ] }) window.ui = ui } \t","excerpt":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: …","ref":"/v2.0/docs/reference/api/","title":"API Reference"},{"body":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply licence --from-file \u0026lt;path-to-licence-file\u0026gt; $ echo \u0026#34;\u0026lt;licence file contents\u0026gt;\u0026#34; | storageos apply licence --from-stdin Flags: --from-file string reads a StorageOS product licence from a specified file path --from-stdin reads a StorageOS product licence from the standard input -h, --help help for licence Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply …","ref":"/v2.0/docs/reference/cli/apply/","title":"Apply licence"},{"body":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes.\nFundamentally, StorageOS uses the storage attached to the nodes in the StorageOS cluster to create and present virtual volumes into containers. Space on the host is consumed from the mount point /var/lib/storageos/data, so it is therefore recommended that disk devices are used exclusively for StorageOS, as described in Managing Host Storage \nStorageOS is agnostic to the underlying storage and runs equally well on bare metal, in virtual machines or on cloud providers.\nRead about the cloud native storage principles behind StorageOS.\nStorageOS on Kubernetes StorageOS is architected as a series of containers that fulfil separate, discrete functions.\n StorageOS Controlplane\nResponsible for monitoring and maintaining the state of volumes and nodes in the cluster\n StorageOS Dataplane\nResponsible for all I/O path related tasks; reading, writing, compression and caching\n StorageOS Scheduler\nResponsible for scheduling applications on the same node as applications volume\n CSI helper\nResponsible for registering StorageOS with Kubernetes as a CSI driver\n StorageOS Operator\nResponsible for the creation and maintenance of the StorageOS cluster\n StorageOS is deployed by the StorageOS operator. In Kubernetes, the StorageOS Controlplane and Dataplane are deployed in a single pod managed by a daemonset. This daemonset runs on every node in the cluster that will consume or present storage. The scheduler, CSI helper and Operator run as separate pods and are controlled as deployments.\nStorageOS is designed to feel familiar to Kubernetes and Docker users. Storage is managed through standard StorageClasses and PersistentVolumeClaims, and features are controlled by Kubernetes-style labels and selectors, prefixed with storageos.com/. By default, volumes are cached to improve read performance and compressed to reduce network traffic.\nAny pod may mount a StorageOS virtual volume from any node that is also running StorageOS, regardless of whether the pod and volume are collocated on the same node. Therefore, applications may be started or restarted on any node and access volumes transparently.\n","excerpt":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes. …","ref":"/v2.0/docs/concepts/architecture/","title":"Architecture"},{"body":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos attach --namespace my-namespace-name my-volume my-node Flags: -h, --help help for attach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos …","ref":"/v2.0/docs/reference/cli/attach/","title":"Attach"},{"body":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our Kubernetes installation instructions page.\nAn EKS deployment of Kubernetes uses AWS Linux by default with an optimized kernel. As the requisite kernel modules are not available for StorageOS to use TCMU, FUSE will be used as a fallback. Using FUSE instead of TCMU has performance implications.\nFor more details about the OS Distributions check the System Configuration page.\n","excerpt":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our …","ref":"/v2.0/docs/platforms/aws-eks/","title":"AWS EKS"},{"body":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our Kubernetes installation instructions page.\nAn AKS deployment of Kubernetes uses Ubuntu by default with an optimized kernel. All versions of Ubuntu with a kernel version later than4.15.0-1029-azure meet the StorageOS prerequisites.\n","excerpt":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our …","ref":"/v2.0/docs/platforms/azure-aks/","title":"Azure AKS"},{"body":"In this example use case we provide three different strategies for accessing files that have been written to a StorageOS persistent volume.\nIn the following examples the \u0026ldquo;application\u0026rdquo; container is the container main, which has a rsync, Nginx or sftp sidecar container. The StorageOS volume that the application is writing to will be mounted into the sidecar container so files written by the application are available for export. Files can be exported using Nginx as a web file server, transferred using rsync or accessed via SFTP.\nThe files create a stateful set that can be used AFTER a StorageOS cluster has been created. See our guide on how to install StorageOS on Kubernetes for more information.\nClone Repository In order to deploy the examples, clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/backup Before deploying the backup-example stateful set we recommend looking through the examples to understand how the different containers are configured\n Exfiltrating files through HTTP Deploy the Nginx example $ kubectl create -f nginx/ service/backup-example created configmap/nginx-config created statefulset.apps/backup-example created pod/busybox created Check that a backup-example pod is running $ kubectl get pods -w -l app=backup-example-nginx NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file $ kubectl exec -it backup-example-nginx-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-nginx ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 80/TCP 46s Use wget to access the files served by Nginx. Nginx is sharing files from the same volume that the main application container is writing to. The connection to the Nginx container is made via the backup-example service. $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;lost%2Bfound/\u0026#34;\u0026gt;lost+found/\u0026lt;/a\u0026gt; 12-Feb-2019 12:32 - \u0026lt;a href=\u0026#34;date.txt\u0026#34;\u0026gt;date.txt\u0026lt;/a\u0026gt; 12-Feb-2019 12:49 29 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx/date.txt Tue Feb 12 12:49:15 UTC 2019 Depending on what files have been written to the StorageOS volume the output of the index file will be different. In the example the date.txt file we created in Step 2 is present on the volume.\nExfiltrating files through Rsync Deploy the rsync example $ kubectl create -f rsync/ service/backup-example created configmap/rsync-config created secret/rsync-credentials created statefulset.apps/backup-example created pod/rsync created Check that a backup-example pod is running $ kubectl get pods -w -l app=backup-example-rsync NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file $ kubectl exec -it backup-example-rsync-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-rsync NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-rsync ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 873/TCP 46s Use rsync to access the files shared by the rsync daemon. rsync is sharing files from the same volume that the main container is writing to. A username and password that are set in the rsync-credentials secret. The secret supplied in the example has the username and password set to username and password. $ kubectl exec -it rsync sh / # rsync --list-only rsync://username@backup-example-rsync/share Password: drwxr-xr-x 4,096 2019/02/12 12:49:15 . -rw-r--r-- 29 2019/02/12 12:49:15 date.txt drwx------ 16,384 2019/02/12 12:32:40 lost+found / # rsync -chavzP rsync://username@backup-example-rsync/share/date.txt . Password: receiving incremental file list date.txt 29 100% 28.32kB/s 0:00:00 (xfr#1, to-chk=0/1) sent 43 bytes received 135 bytes 50.86 bytes/sec total size is 29 speedup is 0.16 / # cat date.txt Tue Feb 12 12:49:15 UTC 2019 In the example above the list of available files was shown and a file called date.txt was synchronized to the rsync container.\nExfiltrating files through SFTP Deploy the sftp example $ kubectl create -f sftp/ Exec into the main container and write some data to a file $ kubectl exec -it backup-example-sftp-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-sftp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-sftp ClusterIP 100.70.50.56 \u0026lt;none\u0026gt; 22/TCP 2h Use SFTP to access the files shared by the SFTP container. If you have made no changes to the sftp-config secret the password is password. $ kubectl exec -it sftp -- bash root@sftp:/# sftp alex@backup-example-sftp alex@backup-example-sftp\u0026#39;s password: Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found sftp\u0026gt; get date.txt Fetching /date.txt to date.txt /date.txt 100% 29 15.9KB/s 00:00 sftp\u0026gt; bye root@sftp:/# cat date.txt Tue Feb 12 17:51:32 UTC 2019 In order to do this a SFTP user needs to be configured. The details for the user are stored in the sftp-config secret (see sftp/17-secret.yaml). The secret consists of base64 encoded username:password:uid:guid and the user is chroot\u0026rsquo;ed inside their home directory so the mount point for the StorageOS volume in the SFTP container in sftp/20-backup-pod.yaml needs to be configured.\nUsing custom SSH Keys The ConfigMap ssh-key-pub (see sftp/15-configmap.yaml) needs to be populated with a public key. The corresponding private key needs to be base64 encoded and put into the ssh-key-private secret (see sftp/17-secret.yaml). The user to connect as is determined by the user that is configured in the sftp-config configMap. To restrict logins to the SSH key edit the sftp-config secret so it contains no password (user::uid:guid).\n Connect to the sftp pod and connect through the service to the SFTP container running inside the backup-example pod. $ kubectl exec -it sftp -- bash root@sftp:/# sftp -i /home/alex/.ssh/id_rsa alex@backup-example-sftp Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found ","excerpt":"In this example use case we provide three different strategies for accessing files that have been …","ref":"/v2.0/docs/usecases/backups/","title":"Backing up files from StorageOS volumes"},{"body":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing StorageOS persistent volumes with Cassandra means that if a Cassandra pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up the pod data is immediately available. Should Kubernetes schedule the Cassandra pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not a StorageOS master is located on the same node.\nAs Cassandra has features to allow it to handle replication careful consideration of whether to allow StorageOS or Cassandra to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Cassandra on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiVersion:apps/v1kind:StatefulSetmetadata:name:cassandraspec:selector:matchLabels:app:cassandraserviceName:cassandrareplicas:3...spec:...volumeMounts:- name:cassandra-datamountPath:/var/lib/cassandra...volumeClaimTemplates:- metadata:name:cassandra-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Move into the Cassandra examples folder and create the objects\ncd storageos-usecases kubectl create -f ./cassandra Confirm Cassandra is up and running.\n$ kubectl get pods -w -l app=cassandra NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 8m32s cassandra-1 1/1 Running 0 7m51s cassandra-2 1/1 Running 0 6m36s Connect to the Cassandra client pod and connect to the Cassandra server through the service\n$ kubectl exec -it cassandra-0 -- cqlsh cassandra-0.cassandra Connected to K8Demo at cassandra-0.cassandra:9042. [cqlsh 5.0.1 | Cassandra 3.11.3 | CQL spec 3.4.4 | Native protocol v4] Use HELP for help. cqlsh\u0026gt; SELECT cluster_name, listen_address FROM system.local; cluster_name | listen_address --------------+---------------- K8Demo | 100.96.7.124 (1 rows) ","excerpt":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing …","ref":"/v2.0/docs/usecases/cassandra/","title":"Cassandra"},{"body":"The storageos command line interface (CLI) is used to manage cluster-wide configuration.\nInstallation # linux/amd64 curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.0.0/storageos_linux_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ # MacOS curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.0.0/storageos_darwin_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ You will need to provide the correct credentials to connect to the API. The default installation uses the storageos-api Secret to generate the first admin user. By default, it creates a single user with username storageos and password storageos:\nexport STORAGEOS_USERNAME=storageos export STORAGEOS_PASSWORD=storageos export STORAGEOS_ENDPOINTS=10.1.5.249:5705 Usage $ storageos Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Formatting CLI Output StorageOS CLI output can be formatted using the --output option. The strings that are passed to --output are \u0026lsquo;json\u0026rsquo;, \u0026lsquo;yaml\u0026rsquo; or \u0026lsquo;text\u0026rsquo;.\nCheatsheet Command Subcommand Description apply licence Apply licence to cluster. attach Attach Volume to a node. create user, volume Create resources such as users or volumes. delete volume Delete resources. describe node, volume Show detailed view of resources. detach volume Detach volume from a node. get cluster, diagnostics, namespace, node, user, volume List resources. help Help version Show CLI version. Source is available on Github.\n","excerpt":"The storageos command line interface (CLI) is used to manage cluster-wide configuration. …","ref":"/v2.0/docs/reference/cli/","title":"Command line interface"},{"body":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator acts as a Kubernetes controller that watches the StorageOSCluster CR (Custom Resource). Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\nYou can find the source code in the cluster-operator repository.\nInstall the operator following orchestrator specific procedure.\nTo deploy a StorageOS cluster you will need to fulfil the following steps.\n Install Etcd Install operator Create storageos-api Secret Create a CR (Custom Resource) to deploy StorageOS ","excerpt":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS …","ref":"/v2.0/docs/reference/cluster-operator/","title":"Cluster Operator"},{"body":"StorageOS clusters represent groups of nodes which run a common distributed control plane.\nTypically, a StorageOS cluster maps one-to-one to a Kubernetes (or similar orchestrator) cluster, and we expect our daemonset to run on all worker nodes within the cluster that will consume or present storage.\nClusters use etcd to maintain state and manage distributed consensus between nodes.\n","excerpt":"StorageOS clusters represent groups of nodes which run a common distributed control plane. …","ref":"/v2.0/docs/concepts/clusters/","title":"Clusters"},{"body":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is generally increased when compression is enabled due to fewer read/write operations taking place on the backend store (the volumes\u0026rsquo; blob files). Compression can be disabled by setting the label storageos.com/nocompress=true on a volume.\nStorageOS utilises the lz4 compression algorithm when writing to the backend store and when compressing replication traffic before it is sent across the network. Compression is granular per 4k block and data will remain compressed/uncompressed once written to a volume. Therefore, compression can be dynamically enabled and disabled by setting the storageos.com/nocompress label on a volume.\nStorageOS detects whether a block can be compressed or not by creating a heuristic that predicts the size of a compressed block. If the heuristic indicates that the compressed block is likely to be larger than the original block then the uncompressed block is stored. Block size increases post compression if the compression dictionary is added to a block that cannot be compressed. By verifying whether blocks can be compressed, disk efficiency is increased and CPU resources are not wasted on attempts to compress uncompressible blocks. StorageOS\u0026rsquo; patented on disk format is used to tell whether individual blocks are compressed without overhead. As such volume compression can be dynamically enabled/disabled even while a volume is in use.\nWhen compression and encryption are both enabled for a volume, blocks are compressed then encrypted.\n","excerpt":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is …","ref":"/v2.0/docs/concepts/compression/","title":"Compression"},{"body":"We are always looking to improve our documentation. If you like to help people and can write, read on for the process for submitting your contributions. If your guide is published, you\u0026rsquo;ll receive $250 per article by PayPal.\nContent guidelines A guide contains step by step instructions for how to accomplish a specific task using StorageOS. To be accepted, guides must be:\n Written in English. Relevant, accurate and complete. Technically correct and thoroughly tested. Follow the writing style guide. Guides should avoid:\n Duplicating an existing guide or other sources, such as blogs or forum posts. Including irrelevant material. Submission and review You should submit your guide as a pull request to the GitHub repo.\nYour submission will be left open for community review for two weeks. After this, your submission will be reviewed internally for about another week.\nIf accepted, your pull request will be approved and you will have 36 hours to send your submission title and PayPal account information. Non-response will be taken as a go-ahead to publish.\nLegal COPYRIGHT OWNERSHIP. The StorageOS Guides \u0026amp; Tutorials repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license.\nCREDIT. Nothing contained in this Agreement shall be deeded to require StorageOS to use the Work, or any part thereof, in connection with StorageOS Guides \u0026amp; Tutorials or otherwise. Credit for the Work shall read, “Contributed by writer’s name.”\nPAYMENT. Upon publication of a submission to the StorageOS Guides \u0026amp; Tutorials Repository, the writer will be paid the sum of USD $250.00 as an electronic payment.\n","excerpt":"We are always looking to improve our documentation. If you like to help people and can write, read …","ref":"/v2.0/docs/reference/contributing/","title":"Contributing to the docs"},{"body":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: user Create a new user account volume Provision a new volume Flags: -h, --help help for create Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos create [command] --help\u0026#34; for more information about a command. create volume $ storageos create volume --help Provision a new volume Usage: storageos create volume [flags] Examples: $ storageos create volume --description \u0026#34;This volume contains the data for my app\u0026#34; --fs-type \u0026#34;ext4\u0026#34; --labels env=prod,rack=db-1 --size 10GiB --namespace my-namespace-name my-app $ storageos create volume --replicas 1 --namespace my-namespace-name my-replicated-app Flags: --async perform the operation asynchronously, using the configured timeout duration --cache caches volume data (default true) --compress compress data stored by the volume at rest and during transit (default true) -d, --description string a human-friendly description to give the volume -f, --fs-type string the filesystem to format the new volume with once provisioned (default \u0026#34;ext4\u0026#34;) -h, --help help for volume -l, --labels strings an optional set of labels to assign to the new volume, provided as a comma-separated list of key=value pairs -r, --replicas uint the number of replicated copies of the volume to maintain -s, --size string the capacity to provision the volume with (default \u0026#34;5GiB\u0026#34;) --throttle deprioritises the volume\u0026#39;s traffic by reducing the rate of disk I/O Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) create user $ storageos create user --help Create a new user account Usage: storageos create user [flags] Examples: $ storageos create user --with-username=alice --with-admin=true Flags: -h, --help help for user --with-admin control whether the user is given administrative priviledges --with-groups stringArray the list of policy groups to assign to the user --with-password string the password to assign to the user. If not specified, this will be prompted for. --with-username string the username to assign Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: …","ref":"/v2.0/docs/reference/cli/create/","title":"Create"},{"body":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available Commands: volume Delete a volume Flags: -h, --help help for delete Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos delete [command] --help\u0026#34; for more information about a command. delete volume $ storageos delete volume --help Delete a volume Usage: storageos delete volume [volume name] [flags] Examples: $ storageos delete volume my-test-volume my-unneeded-volume $ storagoes delete volume --namespace my-namespace my-old-volume Flags: --async perform the operation asynchronously, using the configured timeout duration --cas string make changes to a resource conditional upon matching a provided version -h, --help help for volume Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available …","ref":"/v2.0/docs/reference/cli/delete/","title":"Delete"},{"body":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] Available Commands: node Retrieve detailed information for nodes in the cluster volume Show detailed information for volumes Flags: -h, --help help for describe Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos describe [command] --help\u0026#34; for more information about a command. describe node $ storageos describe node --help Retrieve detailed information for nodes in the cluster Usage: storageos describe node [node names...] [flags] Aliases: node, nodes Examples: $ storageos describe node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) describe volume $ storageos describe volume --help Show detailed information for volumes Usage: storageos describe volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos describe volumes $ storageos describe volume --namespace my-namespace-name my-volume-name Flags: -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] …","ref":"/v2.0/docs/reference/cli/describe/","title":"Describe"},{"body":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] Examples: $ storageos detach volume --namespace my-namespace-name my-volume Flags: --cas string make changes to a resource conditional upon matching a provided version -h, --help help for detach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] …","ref":"/v2.0/docs/reference/cli/detach/","title":"Detach Volume"},{"body":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page.\nDocker EE and the Universal Control Plane can run on different Linux distributions. StorageOS supports RHEL, CentOS, Debian, and selected Ubuntu images. For more details, check out the supported OSs in the prerequisites page.\nStorageOS only supports Kubernetes nodes managed by Docker Enterprise Edition, not those nodes running Swarm. Mixed nodes (those running Kubernetes and Swarm workloads) are not supported. As a consequence, StorageOS volumes can only be provisioned on Kubernetes nodes, and only these nodes should be used for stateful workloads.\n","excerpt":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page. …","ref":"/v2.0/docs/platforms/dockeree/","title":"Docker Enterprise Edition"},{"body":"","excerpt":"","ref":"/v2.0/docs/","title":"Documentation"},{"body":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, most popularly used to aggregate logs, but also to serve as a search backend to a number of different applications.\nUsing StorageOS persistent volumes with ElasticSearch (ES) means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Elasticsearch pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nElasticsearch has features to allow it to handle data replication, and as such careful consideration of whether to allow StorageOS or Elasticsearch to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Elasticsearch on Kubernetes Prerequisites Some OS tuning is required, which is done automatically when using our example from the use cases repository.\nElasticsearch requires vm.max_map_count to be increased to a minimum of 262144, which is a system wide setting. One way to achieve this is to run sysctl -w vm.max_map_count=262144 and update /etc/sysctl.conf to ensure it persists over a reboot. See ElasicSearch reference here.\nAdministrators should be aware that this impacts the behaviour of nodes and that there may be collisions with other application settings. Administrators are advised to centrally collate sysctl settings using the tooling of their choice.\nDeployment of the application StatefulSet defintion ---apiVersion:apps/v1kind:StatefulSetmetadata:name:esdata[...]spec:serviceAccountName:elasticsearchcontainers:- name:dataimage:elasticsearch:6.7.0imagePullPolicy:IfNotPresent[...]volumeMounts:- name:datamountPath:/usr/share/elasticsearch/data/data[...]volumeClaimTemplates:- metadata:name:\u0026#34;data\u0026#34;spec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- default StorageOS storage class nameresources:requests:storage:10Gi# \u0026lt;--- change this to the appropriate valueThis excerpt is from the StatefulSet definition (/elasticsearch/10-es-data.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nInstallation Clone the use cases repo You can find the latest files in the StorageOS use cases repostiory in /elasticsearch/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases Create the kubernetes objects\nThis will install an ES cluster with 3 master, 3 data and 3 coordinator nodes. Combined they will require ~ 14 GiB of available memory in your cluster, however, more may be used as the application is being used\nkubectl apply -f ./elasticsearch/ Once completed, an internal service object will have been created making the cluster available as http://elasticsearch:9200/ which is the default Kibana (when installed via Helm) will be using.\n Confirm Elasticsearch is up and running\nkubectl get pods -l component=elasticsearch NAME READY STATUS RESTARTS AGE elasticsearch-exporter-d86ffd94-zw45l 1/1 Running 0 5m44s es-coordinator-b7b984dd4-7wlz5 1/1 Running 0 5m44s es-coordinator-b7b984dd4-89w26 1/1 Running 0 5m44s es-coordinator-b7b984dd4-b4t6j 1/1 Running 0 5m44s es-master-78dfd5b49f-9gf5c 1/1 Running 0 5m44s es-master-78dfd5b49f-smsbw 1/1 Running 0 5m44s es-master-78dfd5b49f-z4qpj 1/1 Running 0 5m44s esdata-0 1/1 Running 0 5m44s esdata-1 1/1 Running 0 4m34s esdata-2 1/1 Running 0 3m22s Connect to ElasticSearch\nTo connect to ES directly, you can use the following port-forward command\nkubectl port-forward svc/elasticsearch 9200 and then access it via http://localhost:9200\n Kibana (optional) One of the most popular uses of ES is to use it for log aggregation and indexing, Kibana helps us visualize the data in these indices and can be easily used when installed via its Helm chart\n Install the helm chart.\nhelm install stable/kibana Once installed, use a port-foward to Kibana instead of directly to ES\nkubectl port-forward --namespace default $(kubectl get pods --namespace default -l \u0026#34;app=kibana\u0026#34; -o jsonpath=\u0026#34;{.items[0].metadata.name}\u0026#34;) 5601 and then access it via http://localhost:5601\n Monitoring (optional) As part of the example deployment, ES metrics are exposed and can be scraped by Prometheus on port 9108 (see 77-es-exporter.yaml). This is enabled by default, and should work with the default Prometheus install via Helm. If you\u0026rsquo;re using the Prometheus service monitors, you can monitor this installation by creating a monitor for the es-exporter service. For an example of how this is done to monitor StorageOS, please see prometheus-setup.\n","excerpt":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, …","ref":"/v2.0/docs/usecases/elasticsearch/","title":"Elasticsearch"},{"body":"etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to durably persist the StorageOS cluster state. As the backing store for Kubernetes, StorageOS uses etcd for many of the same reasons.\nStorageOS uses etcd as the single source of truth for all StorageOS objects. Whenever a request is made to create, update or delete an object the result is written to etcd before the request is completed. Using etcd as a configuration store allows nodes to retrieve the current cluster state after being offlined, allowing offlined nodes to rejoin the cluster.\n N.B. StorageOS v2.0 does not provide an embedded etcd server as previous versions did. You will need to setup an etcd server for StorageOS to use prior to installation of StorageOS. Please see our etcd Operations page for more information on how to install and configure etcd.\n ","excerpt":"etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to …","ref":"/v2.0/docs/concepts/etcd/","title":"Etcd"},{"body":"StorageOS uses etcd to store cluster metadata. Because of the strong consistency model that etcd enforces, StorageOS metadata operations are guaranteed to be atomic and consistent.\nInstallation options Before installing StorageOS, an etcd cluster needs to be prepared. There are different topologies that fulfil this prerequisite.\n External etcd (Production) etcd as Pods (Testing) Production Testing External Etcd The production topology is designed to provide the highest stability for the etcd cluster. It is necessary for normal StorageOS function to have a reliable metadata cluster. Otherwise, central operations such as provisioning, attachment or failover of volumes cannot be performed. In the event that etcd becomes unavailable, StorageOS clusters become read only, allowing access to data but preventing metadata changes.\nIt is recommended to install etcd out of the scope of the orchestrator wherever possible. Following CoreOS best practices, a minimum of 3 independent nodes should be dedicated to etcd. StorageOS doesn\u0026rsquo;t require a high performance etcd cluster as the throughput of metadata to the cluster is low. Depending on the level of redundancy you feel comfortable with you can install etcd on the Kubernetes Master nodes. Take extreme care to avoid collisions of the StorageOS etcd installation with the Kubernetes etcd when using the Kubernetes Master nodes. Precautions such as changing the default configuration for the client and peer ports, and ensuring the etcd data directory is modified. The ansible playbook below will default the etcd installation directory to /var/lib/storageos-etcd.\nInstallation If you are familiar with etcd, you can proceed with the CoreOS instructions to install etcd, otherwise this section lays out out an example installation using Ansible.\n Clone StorageOS Helper repository\ngit clone https://github.com/storageos/deploy.git cd k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd Edit the inventory file\n Target the nodes that install etcd, where the file hosts.example serves as an example. The ip parameter is needed for each node.\n $ cat hosts.example [nodes] centos-1 ip=172.28.128.14 centos-2 ip=172.28.128.15 centos-3 ip=172.28.128.16 # Edit the inventory file $ vi hosts.example # Or your own inventory file Edit the etcd configuration\n If targeting Kubernetes Master nodes, you must change etcd_port_client, etcd_port_peers\n $ cat group_vars/all etcd_version: \u0026#34;3.3.18\u0026#34; etcd_port_client: \u0026#34;2379\u0026#34; etcd_port_peers: \u0026#34;2380\u0026#34; etcd_quota_bytes: 8589934592 # 8 GB etcd_auto_compaction_mode: \u0026#34;revision\u0026#34; etcd_auto_compaction_retention: \u0026#34;100\u0026#34; members: \u0026#34;{{ groups[\u0026#39;nodes\u0026#39;] }}\u0026#34; installation_dir: \u0026#34;/var/lib/storageos-etcd\u0026#34; $ vi group_vars/all Install\nansible-playbook -i hosts.example site.yml Verify installation\n The playbook installs the etcdctl binary on the nodes, at /usr/local/bin.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 member list 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Managed Services When running StorageOS on Managed Kubernetes services it may not be possible to deploy with the Production etcd topology described above. It is therefore recommended to deploy etcd on its own as much as possible, even if that means deploying 3 independent VMs for etcd to run on.\nAs managed services treat nodes as ephemeral resources, if the orchestration deletes the 3 nodes hosting etcd, the result will be catastrophic and a restore from a backup will be needed.\nIf it is not possible to deploy independent VMs for etcd, etcd can be deployed as pods, inside the cluster. This configuration requires an awareness of the stability that etcd requires. You can use the etcd-as-pods installation option, but be aware of the precautions that need to be taken.\nWhy External Etcd etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. In the case that quorum is lost, an etcd node stops and marks its contents as read-only. It cannot guarantee that the data being held is valid. Another peer might have a newer version that has not been delivered. Quorum is fundamental for etcd operations.\nIn a Kubernetes environment, applications are scheduled across and in some scenarios such as \u0026ldquo;DiskPressure\u0026rdquo; they may need to be evicted from a node, and be scheduled onto a different node. With an application such as etcd, the scenario described can result in quorum being lost, making the cluster unable to recover automatically. Usually a 3 node etcd cluster can survive losing one node and recover. However, losing a second node at the same time or even having a network partition between them will result in quorum lost.\nBind Etcd IPs to Kubernetes Service Kubernetes external services use a DNS name to reference external endpoints. You can use the example from the helper github repository to deploy the external Service. That might be of use when monitoring etcd from Prometheus.\n Etcd as Pods etcd can be deployed in Kubernetes using the official etcd-operator.\nDeploying etcd in Kubernetes makes the etcd installation very easy, however be aware that even though the official etcd-operator is maintained by RedHat, it hasn\u0026rsquo;t been under active development since 2019. As such it may be considered an archived project. For an actively maintained etcd Operator you might want to check the Improbable etcd Operator.\nExamples of deploying etcd clusters using the etcd-operator on Kubernetes and OpenShift are available.\nSince Kubernetes 1.16 the deployment api uses \u0026ldquo;apps/v1\u0026rdquo;. Once you have cloned the coreos etcd operator repository, you will need to change the apiVersion of the file \u0026ldquo;examples/deployment.yaml\u0026rdquo; from extensions/v1beta1 to apps/v1.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. Make sure you take frequent backups of the etcd cluster as it holds all the StorageOS cluster metadata.\nKnown etcd-operator issues This topology is only recommended for deployments where isolated nodes cannot be used.\netcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd in pods it is therefore important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster is not possible unless a backup is restored.\n StorageOS and Etcd When installing StorageOS, the etcd endpoints are passed in a StorageOSCluster Custom Resource.\nFor instance:\napiVersion: \u0026quot;storageos.com/v1\u0026quot; kind: StorageOSCluster metadata: name: \u0026quot;storageos\u0026quot; spec: secretRefName: \u0026quot;storageos-api\u0026quot; # Reference from the Secret created in the previous step secretRefNamespace: \u0026quot;default\u0026quot; # Namespace of the Secret (...) kvBackend: address: 'storageos-etcd-client.etcd:2379' # Example address, change for your etcd endpoint #address: '10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379' # You can set etcd server ips backend: 'etcd' Note the kvBackend.address section.\n For full Custom Resource documentation check StorageOSCluster resource definition.\nBest practices StorageOS uses etcd as a service, whether it is deployed following the above instructions or as a custom installation. It is expected that the user maintains the availability and integrity of the etcd cluster.\nIt is highly recommended to keep the cluster backed up and ensure high availability of its data. It is also important to keep the latency between StorageOS nodes and the etcd replicas low. Deploying an etcd cluster in a different data center or region can make StorageOS detect etcd nodes as unavailable due to latency. A 10ms latency between StorageOS and etcd would be the maximum threshold for proper functioning of the system.\nMonitoring It is highly recommended to add monitoring to the etcd cluster. etcd serves Prometheus metrics on the client port http://etc-url:2379/metrics.\nYou can use StorageOS developed Grafana Dashboards for etcd. When using etcd for production, you can use the etcd-cluster-as-service, while the etcd-cluster-as-pod can be used when using etcd from the operator.\nDefragmentation etcd uses revisions to store multiple versions of keys. Compaction removes all key revision prior to a certain revision from etcd. Typically the etcd configuration enables the automatic compaction of keys to prevent performance degradation and limit the storage required. Compaction of revisions can create fragmentation that means space on disk is available for use by etcd but is unavailable for use by the file system. In order to reclaim this space, etcd can be defragmented.\nReclaiming space is important because when the etcd database file grows over the \u0026ldquo;DB_BACKEND_BYTES\u0026rdquo; parameter, the cluster triggers an alarm and sets itself read only and only allows reads and deletes. To avoid hitting the db backend bytes limit, compaction and defragmentation are required. How often defragmentation is required depends on the churn of key revisions in etcd.\nThe Grafana Dashboards mentioned above indicate when nodes require defragmentation. Be aware that defragmentation is a blocking operation that is performed per node, hence the etcd node will be locked for the duration of the defragmentation. Defragmentation usually takes a few milliseconds to complete.\n","excerpt":"StorageOS uses etcd to store cluster metadata. Because of the strong consistency model that etcd …","ref":"/v2.0/docs/operations/external-etcd/","title":"Etcd"},{"body":"StorageOS requires an etcd cluster in order to function.\nFor more information on why etcd is required please see our etcd concepts page and for production recommendations for etcd installations please see our Etcd Operations page.\n N.B. StorageOS does not recommend using the Kubernetes etcd cluster for StorageOS installations\n Using Etcd with StorageOS During installation of StorageOS the kvBackend parameters of the StorageOS operator are used to specify the address of the etcd cluster. See the StorageOS cluster operator configuration examples for more information.\n","excerpt":"StorageOS requires an etcd cluster in order to function.\nFor more information on why etcd is …","ref":"/v2.0/docs/prerequisites/etcd/","title":"Etcd"},{"body":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. StorageOS implements dynamic provisioning, so the creation of a PVC will automatically provision a PV (PersistentVolume) that can be used to persist data written by a Pod.\nCreate the PersistentVolumeClaim You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic PVC definition\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;fast\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC will dynamically provision a 5GB volume using the fast StorageClass. This StorageClass was created during the StorageOS install and triggers creation of a PeristentVolume by StorageOS.\nFor installations with CSI, you can create multiple StorageClasses in order to specify default labels.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels. For simplicities sake this example will use unreplicated volumes.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# Reference to the StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1labels:storageos.com/replicas:\u0026#34;1\u0026#34;spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC has the storageos.com/replicas label set. This label tells StorageOS to add a replica for the volume that is created. For the sake of keeping this example simple an unreplicated volume will be used.\n Move into the examples folder and create a PVC using the PVC definition above.\n$ # from storageos-usecases/00-basic $ kubectl create -f ./pvc-basic.yaml You can view the PVC that you have created with the command below\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-1 Bound pvc-f8ffa027-e821-11e8-bc0b-0ac77ccc61fa 5Gi RWO fast 1m Create a pod that mounts the PVC created in step 2.\n$ kubectl create -f ./pod.yaml The command above creates a Pod that uses the PVC that was created in step 1.\napiVersion:v1kind:Podmetadata:name:d1spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mntname:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-1In the Pod definition above volume v1 references the PVC created in step 2, and is mounted in the pod at /mnt. In this example a debian image is used for the container but any container image with a shell would work for this example.\n Confirm that the pod is up and running\n$ kubectl get pods NAME READY STATUS RESTARTS AGE d1 1/1 Running 0 1m Execute a shell inside the container and write some contents to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo \u0026#34;Hello World!\u0026#34; \u0026gt; /mnt/helloworld root@d1:/# cat /mnt/helloworld Hello World! By writing to /mnt inside the container, the StorageOS volume created by the PVC is being written to. If you were to kill the pod and start it again on a new node, the helloworld file would still be avaliable.\nIf you wish to see more use cases with actual applications please see our Use Cases documentation.\n ","excerpt":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. …","ref":"/v2.0/docs/operations/firstpvc/","title":"StorageOS Volume Guide"},{"body":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available Commands: cluster Fetch cluster-wide configuration details diagnostics Fetch a cluster diagnostic bundle namespace Retrieve basic details of cluster namespaces node Retrieve basic details of nodes in the cluster user Fetch user details volume Retrieve basic details of volumes Flags: -h, --help help for get Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos get [command] --help\u0026#34; for more information about a command. get cluster $ storageos get cluster --help Fetch cluster-wide configuration details Usage: storageos get cluster [flags] Examples: $ storageos get cluster Flags: -h, --help help for cluster get diagnostics $ storageos get diagnostics --help Fetch a cluster diagnostic bundle Usage: storageos get diagnostics [flags] Examples: $ storageos get diagnostics $ storageos get diagnostics --output-file ~/my-diagnostics Flags: -h, --help help for diagnostics --output-file string writes the generated diagnostic bundle to a specified file path get namespace $ storageos get namespace -h Retrieve basic details of cluster namespaces Usage: storageos get namespace [namespace names...] [flags] Aliases: namespace, namespaces Examples: $ storageos get namespaces $ storageos get namespace my-namespace-name Flags: -h, --help help for namespace -l, --selector stringArray filter returned results by a set of comma-separated label selectors get node $ storageos get node -h Retrieve basic details of nodes in the cluster Usage: storageos get node [node names...] [flags] Aliases: node, nodes Examples: $ storageos get node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors get user $ storageos get user -h Fetch user details Usage: storageos get user [user names...] [flags] Aliases: user, users Examples: $ storageos get user my-username $ storageos get user my-username-1 my-username-2 $ storageos get user --use-ids my-userid $ storageos get user --use-ids my-userid-1 my-userid-2 Flags: -h, --help help for user get volume $ storageos get volume -h Retrieve basic details of volumes Usage: storageos get volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos get volumes --all-namespaces $ storageos get volume --namespace my-namespace-name my-volume-name Flags: --all-namespaces retrieves volumes from all accessible namespaces. This option overrides the namespace configuration -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors ","excerpt":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available …","ref":"/v2.0/docs/reference/cli/get/","title":"Get"},{"body":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any of the nodes in the cluster. Initially you can log in as the default administrator, with the username and password from the storageos-api Secret. By default storageos, storageos.\n You can access the GUI by either port-forwarding with kubectl or using an Ingress rule. i.e kubectl -n kube-system port-forward svc/storageos 5705.\n Nodes Volumes You can create volumes, including replicated volumes, and view volume details:\nLicensing Cluster info Namespaces Volumes can be namespaced across different projects or teams, and you can switch namespace using the left hand panel:\n","excerpt":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any …","ref":"/v2.0/docs/reference/gui/","title":"Graphical user interface (GUI)"},{"body":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the status of nodes in the cluster.\n$ storageos get nodes NAME HEALTH AGE LABELS node1 online 44 minutes ago node2 online 44 minutes ago node3 online 44 minutes ago node4 online 44 minutes ago node5 online 44 minutes ago ","excerpt":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the …","ref":"/v2.0/docs/operations/health/","title":"Cluster health"},{"body":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Use \u0026#34;storageos [command] --help\u0026#34; for more information about a command. ","excerpt":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to …","ref":"/v2.0/docs/reference/cli/help/","title":"Help"},{"body":"StorageOS uses the storage available on the nodes where it is installed to present as available for volumes.\nIn order to mitigate against problems caused by filling the host root disk, we recommend mounting a separate device into the /var/lib/storageos directory. StorageOS is agnostic to the type of filesystem mounted in /var/lib/storageos.\nExtending Available Storage StorageOS uses subdirectories of /var/lib/storageos/data to hold user data. By default, the directory /var/lib/storageos/data/dev1 will be created when a node is bootstrapped, and used for pool data. It is possible to shard the data by creating more directories into this structure. StorageOS will save data in any directory that conforms to the pattern /var/lib/storageos/data/dev[0-9]+, such as /var/lib/storageos/data/dev2 or /var/lib/storageos/data/dev5. This functionality enables operators to mount different devices into devX directories and StorageOS will recognise them as available storage automatically.\nThere are two possible options to expand the available disk space for StorageOS to allocate:\n Mount filesystem in /var/lib/storageos/data/devX Use LVM to expand the logical volume available to StorageOS Option 1: Mount Additional Devices This option enables operators to expand the cluster\u0026rsquo;s available space at any time without having to stop applications or forcing operational downtime. The expansion of disk is transparent for applications and StorageOS Volumes. StorageOS will use the new available space to create new data files.\n Context\nWe assume that there is a disk available in our Linux system without formatting in addition to the root filesystem. StorageOS data dir dev1 (/var/lib/storageos/data/dev1) is using /dev/xvda1. We will use the device /dev/xvdf to expand StorageOS available space.\nList available block devices in the host.\nroot@node0:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Format device\nroot@node0:/var/lib/storageos/data# mkfs -t ext4 /dev/xvdf mke2fs 1.42.12 (29-Aug-2014) Creating filesystem with 26214400 4k blocks and 6553600 inodes Filesystem UUID: 380712fa-6f82-477a-81a5-d7466d4c6b7f Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 4096000, 7962624, 11239424, 20480000, 23887872 Allocating group tables: done Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done Mount filesystem\nroot@node0:~# mkdir -p /var/lib/storageos/data/dev2 root@node0:~# mount /dev/xvdf /var/lib/storageos/data/dev2 Verify available storage\nIn less than 30 seconds, StorageOS will see the new available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 244,491,013,324 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Note that the node node0 has increased the TOTAL capacity in 100Gi.\n Persist the mount at boot by adding the mount endpoint to /etc/fstab\n Option 2: Expand Existing Devices Backed by LVM This option enables operators to take advantage of LVM to manage disks.\n Context\nWe assume that /var/lib/storageos is mounted onto an LVM volume. We are using a volumegroup named storageos and logical volume called data. There is a second physical disk /dev/xvdg unused.\nList available block devices in the host.\nroot@node2:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk `-storageos-data 254:0 0 99G 0 lvm /var/lib/storageos xvdg 202:96 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 107,696,304,947 # --\u0026gt; LVM storageos/data volume Add physical disk to LVM\nroot@node2:~# vgextend storageos /dev/xvdg Volume group \u0026quot;storageos\u0026quot; successfully extended The volume group storageos must have 2 physical volumes (#PV)\nroot@node2:~# vgs VG #PV #LV #SN Attr VSize VFree storageos 2 1 0 wz--n- 199.99g 104.99g Extend logical volume data\nroot@node2:~# lvextend -L+100G /dev/storageos/data Size of logical volume storageos/data changed from 95.00 GiB (24320 extents) to 195.00 GiB (49920 extents). Logical volume data successfully resized Resize the FileSystem\n Your filesystem must support the option to be expanded, and to do so while in use. Otherwise, you need to unmount first.\n root@node2:~# resize2fs /dev/storageos/data resize2fs 1.42.12 (29-Aug-2014) Filesystem at /dev/storageos/data is mounted on /var/lib/storageos; on-line resizing required old_desc_blocks = 6, new_desc_blocks = 13 The filesystem on /dev/storageos/data is now 51118080 (4k) blocks long. Check new available space\nThe mounted file system to /var/lib/storageos has increased its size.\nroot@node2:~# df -h /dev/mapper/storageos-data Filesystem Size Used Avail Use% Mounted on /dev/mapper/storageos-data 192G 60M 183G 1% /var/lib/storageos StorageOS available storage has increased too.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 206,158,430,208 # --\u0026gt; 100G more available Persist the mount at boot by adding the mount point to /etc/fstab\n ","excerpt":"StorageOS uses the storage available on the nodes where it is installed to present as available for …","ref":"/v2.0/docs/operations/managing-host-storage/","title":"Managing Host Storage"},{"body":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init container that sets the system configuration for StorageOS. The container also manages configuration changes required when upgrading StorageOS versions.\nThe container belongs to the DaemonSet that the StorageOS Cluster Operator starts when a StorageOSCluster resource is created. The storageos-init container is executed as an initContainer as part of a Kubernetes Pod. Therefore, only successful execution of the storageos-init container processes will result in the main container starting.\nScript Framework The code responsible for fulfilling the requirements is based on a Script Framework.\nThe script framework executes a set of scripts, performing checks, verifications and other procedures needed for StorageOS to be able to start. The scripts stdout and stderr are written to the stdout and stderr of the init app. The container logs contain all the logs of the individual scripts that run. The exit statuses of the scripts are used to determine initialization failure or success. Any non-zero exit status is logged as an event in the Kubernetes Pod events.\nIf any of the scripts fail, the storageos-init container will propagate the failure to Kubernetes, showing the status of the Pod as Init:Err.\nTo view the output of all storageos-init containers the following command can be used:\nkubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init For more details, check the StorageOS init container project.\nScripts executed The storageos-init container executes the following scripts.\n enable-lio dbupgrade ","excerpt":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init …","ref":"/v2.0/docs/reference/init-container/","title":"Init container"},{"body":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a StorageOS persistent volume being mounted on /var/jenkins_home. Deploying Jenkins using StorageOS offers multiple benefits. Firstly Jenkins can spin up multiple build pods at once to allow concurrent builds of different projects. Secondly Jenkins configuration is on a PersistentVolume so even if the Jenkins pod is rescheduled the configuration will persist.\nUsing StorageOS volume replicas allows for failure of nodes holding the PersistentVolume without interrupting Jenkins. Lastly by enabling StorageOS fencing Jenkins time to recover, in case of node failures, is greatly reduced.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information\nDeploying Jenkins on Kubernetes You can find the latest files in the StorageOS example deployment repository.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./jenkins Confirm that Jenkins is up and running\n$ kubectl get pods -w -l app=jenkins NAME READY STATUS RESTARTS AGE jenkins-0 1/1 Running 0 1m Connect to the Jenkins UI through the Jenkins service.\nYou can do this by port forwarding the Jenkins Kubernetes service to your localhost and accessing the UI via your browser. Alternatively if you have network access to your Kubernetes nodes then you can create a NodePort service and access Jenkins like that. A NodePort service has been left in 10-service.yaml commented out.\nTo port-foward the Jenkins service use the following command.\n$ kubectl port-foward svc/jenkins 8080 To login to the Jenkins UI use the credentials specified in 07-config.yaml, unless these have been changed from the defaults the username/password is admin/password.\n Create a Jenkins job.\nOnce you are logged into the UI you can create a job that will be farmed out to a Kubernetes plugin build agent. Click New Item, enter a name for the project and select Freestyle project. Next add an Execute shell build step. As a proof of concept you can use the bash below to have the pod execute a sleep.\n#!/bin/bash sleep 1000 Save the project and select Schedule a build of your project. You can watch for the appearance of a build pod using kubectl get pods -l jenkins=agent -w. Once the pod is created you should see the Build Executor status in the Jenkins UI display the pod.\nTo see multiple projects being built at once create another project and try scheduling a build of both projects at the same time.\n ","excerpt":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a …","ref":"/v2.0/docs/usecases/jenkins/","title":"Jenkins"},{"body":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub and traditional queues.\nUsing StorageOS persistent volumes with Apache Kafka means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the kafka pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nKafka has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Kafka to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Apache Zookeeper is required by Kafka to function; we assume it to already exist and be accessible within the Kubernetes cluster as zookeeper, see how to run Zookeeper with StorageOS here StorageOS is assumed to have been installed; please check for the latest available version here Kafka pods require 1536 MB of memory for successful scheduling Helm To simplify the deployment of kafka, we\u0026rsquo;ve used this Kafka helm chart (incubator) (version 0.13.8, app version 5.0.1) and rendered it into the example deployment files you can find in our GitHub repo.\nClone the use cases repo You can find the latest files in the StorageOS use cases repository in /kafka/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition\n---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:kafkalabels:app:kafka...spec:serviceName:kafka-headlesspodManagementPolicy:OrderedReadyupdateStrategy:type:OnDeletereplicas:3# \u0026lt;--- number of kafa pods to runtemplate:...spec:serviceAccountName:kafkacontainers:...- name:kafka-brokerimage:\u0026#34;confluentinc/cp-kafka:5.0.1\u0026#34;imagePullPolicy:\u0026#34;IfNotPresent\u0026#34;...volumeMounts:- name:datadirmountPath:\u0026#34;/var/data\u0026#34;volumes:- name:jmx-configconfigMap:name:kafka-metricsterminationGracePeriodSeconds:60volumeClaimTemplates:- metadata:name:datadirspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]resources:requests:storage:10Gi# \u0026lt;--- storage requested for each podstorageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Create the kubernetes objects\nkubectl apply -f ./kafka/ Confirm kafka is up and running\n$ kubectl get pods -l app=kafka NAME READY STATUS RESTARTS AGE kafka-0 2/2 Running 0 10m kafka-1 2/2 Running 0 9m26s kafka-2 2/2 Running 0 7m59s Connect to kafka\nConnect to the kafka test client pod and send some test data to kafka through its service endpoint\n Connect to the pod\nkubectl exec -it kafka-test-client /bin/bash Create a topic\n/usr/bin/kafka-topics --zookeeper zookeeper:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1 Send some test data\n/usr/bin/kafka-run-class org.apache.kafka.tools.ProducerPerformance --topic test-rep-one --num-records 5000 --record-size 100 --throughput -1 --print-metrics --producer-props acks=1 bootstrap.servers=kafka:9092 buffer.memory=67108864 batch.size=8196 ","excerpt":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from …","ref":"/v2.0/docs/usecases/kafka/","title":"Kafka"},{"body":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual machines (VMs) in Kubernetes pods.\nDeploying Kubevirt using StorageOS offers multiple benefits. Kubevirt can spin up VMs as Kubernetes pods, using images on StorageOS persistent volumes. Doing this allows the VM data to persist through restarts and rescheduling. Using StorageOS volume replicas also allows for failure of nodes holding the PersistentVolume without interrupting the VM running off the PersistentVolume. Containerized Data Importer (CDI) can also be used to prepare StorageOS volumes with disk images in an automated fashion. Simply by declaring that a VirtualMachine will use a DataVolume and providing the disk image URL, a StorageOS volume can be dynamically provisioned and automatically prepared with the disk image.\nThis usecase will guide you through installing KubeVirt and CDI on your Kubernetes cluster, and create a VM. By the end of the guide you\u0026rsquo;ll be able to launch a shell inside the KubeVirt VM that\u0026rsquo;s running as a Kubernetes pod.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Please ensure you have met the Kubevirt prerequisites, please see the Kubevirt installation instructions for more information.\nAs part of this installation it is assumed that you are running a Kubernetes cluster on VMs. As such nested virtualization or hardware emulation need to be enabled.\nDeploying KubeVirt on Kubernetes For ease of installation we have enabled hardware emulation. If your VMs support nested virtualization then edit the Kubevirt ConfigMap ./kubevirt-install/10-cm.yaml, removing the line debug.useEmulation: \u0026quot;true\u0026quot;.\n In order to deploy Kubevirt you just need to clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/kubevirt $ kubectl create -f ./kubevirt-install Check that the Kubevirt pods are running.\n$ kubectl get pods -w -n kubevirt NAME READY STATUS RESTARTS AGE virt-api-57546d479b-p26d4 1/1 Running 0 1m virt-api-57546d479b-zs5dw 1/1 Running 0 1m virt-controller-56b5498854-7xsfz 1/1 Running 1 1m virt-controller-56b5498854-bz559 1/1 Running 1 1m virt-handler-6z4kq 1/1 Running 0 1m virt-handler-7szhl 1/1 Running 0 1m virt-handler-jmm6w 1/1 Running 0 1m virt-operator-79c9bdd859-8xq98 1/1 Running 0 1m virt-operator-79c9bdd859-kfjz6 1/1 Running 0 1m Once Kubevirt is running install CDI.\n$ kubectl create -f ./cdi Check that the CDI pods are running correctly.\n$ kubectl get pods -n cdi NAME READY STATUS RESTARTS AGE cdi-apiserver-8668f888df-s6pp4 1/1 Running 0 1m cdi-deployment-5cf794896b-whh4j 1/1 Running 0 1m cdi-operator-5887f96c-dz2hg 1/1 Running 0 1m cdi-uploadproxy-97fbbfcbf-6f9xs 1/1 Running 0 1m Now that CDI and Kubevirt are running, VMs can be created. In this example VMs running Cirros, a small and lightweight OS, will be created. The vm-cirros.yaml manifest creates a VirtualMachine that uses a DataVolume. This means that CDI will create a StorageOS backed PVC and download the image that the VirtualMachineInstance (VMI) will boot from onto the PVC.\n$ kubectl create -f ./vm-cirros.yaml Check that the VMI is running. Note that the VMI will only be created after CDI has downloaded the Cirros disk image onto a StorageOS persistent volume so depending on your connection speed this may take some time.\n$ kubectl get vmi NAME AGE PHASE IP NODENAME cirros 1m Running 10.244.2.12 ip-10-1-10-154.storageos.net $ kubectl get pods NAME READY STATUS RESTARTS AGE virt-launcher-cirros-drqhr 1/1 Running 0 1m Connect to the VM console.\nThis example uses the virtctl kubectl plugin in order to connect to the VMs console. The escape sequence ^] is ctrl + ]\n$ kubectl virt console cirros Successfully connected to cirros console. The escape sequence is ^] login as \u0026#39;cirros\u0026#39; user. default password: \u0026#39;gocubsgo\u0026#39;. use \u0026#39;sudo\u0026#39; for root. cirros login: cirros Password: $ Cloning Volumes CDI allows for images to be cloned using a DataVolume manifest. Verify that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n N.B. Ensure that the VMI is stopped before continuing!\n Verify that the VMI is stopped before continuing, and that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE cirros Bound pvc-f4833060-5a77-420c-927e-6bc518d9df3c 12Gi RWO fast 1m Once the PVC\u0026rsquo;s existence is confirmed then create a new DataVolume that uses the cirros PVC as its source.\n$ kubectl create -f ./cloned.yaml Watch as the CDI pods are created.\n$ kubectl get pods -w You\u0026rsquo;ll see that a cdi-upload-cloned-datavolume pod is created and then a cdi-clone-source pod is created. The cdi-source pod mounts the original cirros volume and sends the contents of the volume to the cdi-upload pod. The cdi-upload pod creates and mounts a new PVC and writes the contents of the original volume to it.\n ","excerpt":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual …","ref":"/v2.0/docs/usecases/kubevirt/","title":"Kubevirt"},{"body":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to various StorageOS artefacts. Applying specific feature labels triggers compression, replication and other storage features. No feature labels are present by default.\nStorageOS Node labels Nodes do not have any feature labels present by default. When StorageOS is run within Kubernetes, the Cluster Operator syncs any node labels set on Kubernetes nodes within StorageOS. Node labels may also be set with the CLI or UI.\n Feature Label Values Description Compute only storageos.com/computeonly true / false Specifies whether a node should be computeonly where it only acts as a client and does not host volume data locally, otherwise the node is hyperconverged (the default), where the node can operate in both client and server modes. You can set the computeonly label on the Kubernetes node and the label will be sync\u0026rsquo;d to the StorageOS node (labels take an eventual consistency reconciliation time of ~1min).\nkubectl label node $NODE storageos.com/computeonly=true StorageOS Volume labels Volumes do not have any feature labels present by default. WARNING: The caching and compression labels can only apply during provision time, they can\u0026rsquo;t be changed during execution.\n Feature Label Values Description Caching storageos.com/nocache true / false Switches off caching. Compression storageos.com/nocompress true / false Switches off compression of data at rest and in transit. Replication storageos.com/replicas integers [0, 5] Replicates entire volume across nodes. Typically 1 replica is sufficient (2 copies of the data); more than 2 replicas is not recommended. Once applied, this setting can only be changed in the StorageOS UI or CLI. To create a volume with a feature label:\n Option 1: PVC Label\nAdd the label in the PVC definition, for instance:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc-3labels:storageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;-----spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:1G Option 2: Set label in the StorageClass\nAny PVC using the StorageClass inherits the label. The PVC label takes precedence over the StorageClass parameters.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;--------provisioner:storageos# CSI driver (recommended) Option 3:\nOnce a PVC is created, you can update the Labels in StorageOS both in the UI or CLI. Those labels are going to be visible only for StorageOS and will not be synced to the Kubernetes resource.\n ","excerpt":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to …","ref":"/v2.0/docs/reference/labels/","title":"StorageOS Feature labels"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying a Free Developer licence.\nTo learn how to apply a licence to your cluster, check the operations licensing page.\nFree Developer licence The developer licence only requires the user to register and issue a licence through StorageOS.\nThe developer licence is free and grants a licence for a 5 node StorageOS cluster with 5TB of provisioned capacity. It is designed to enable basic cloud native workflows in Kubernetes that require the persistence of stateful application data. Dynamic provisioning, distributed access to data and high availability of volumes through synchronous replication and automatic failover are some of the features that are available under the developer licence.\nThe free developer licence includes community support through our public Slack channel..\nCommercial licences For information on our commercial offerings, including support, please contact sales@storageos.com.\nNote about capacity limits Some StorageOS licences have limits on capacity. StorageOS allows provisioning volumes until the limit of the licence is reached. Only the size of the volume requested by the Persistent Volume Claim counts for the licence limit, regardless whether of whether the volume has replication enabled.\nOnce the licence limit is reached, new volumes are not able to provision unless provisioned capacity is released, i.e deleting volumes. That behaviour is not tight to the capacity of the backend disks on your nodes.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2.0/docs/reference/licence/","title":"Licence"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying for a Free Developer licence.\nObtaining a Developer licence via the GUI You can obtain and apply a free Developer licence in the StorageOS web GUI by creating or logging in with a StorageOS account on the StorageOS portal via the licence page of the StorageOS web GUI:\n You can access the GUI on any node running the StorageOS Daemonset on the port 5705. Whether using an Ingress route or port-forwarding.\n Wait a few seconds for the licence generation process to complete, at which point your licence will be visible.\nApplying a previously obtained licence via the GUI Commercial licences are delivered through contact with the StorageOS team. To apply such keys, via the web GUI, visit the licence section of the GUI and click on the tab \u0026ldquo;Upgrade\u0026rdquo;, for the specific licence level you purchased. Then paste the licence key and click on \u0026ldquo;UPLOAD KEY TO CLUSTER\u0026rdquo;. Note that you can also view your cluster ID on the same page.\nApplying a licence via the CLI Before getting a licence, you need to know the ID of your StorageOS cluster.\nThis CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence. Once, given the key, you can apply the licence by using the following command.\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos apply licence --from-stdin Read the licence CLI command reference for further information.\nObtaining an Enterprise licence Please contact sales@storageos.com to discuss pricing for commercial licences.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2.0/docs/operations/licensing/","title":"Licensing"},{"body":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on linux.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MS SQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1...spec:serviceAccountName:mssql...volumeMounts:- name:datamountPath:/var/opt/mssql...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MS SQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mssql Confirm MS SQL is up and running.\n$ kubectl get pods -w -l app=mssql NAME READY STATUS RESTARTS AGE mssql-0 1/1 Running 0 1m Connect to the MS SQL client pod and connect to the MS SQL server through the service\n$ kubectl exec -it mssql-0 -- /opt/mssql-tools/bin/sqlcmd -S mssql-0.mssql -U SA -P \u0026#39;Password15\u0026#39; 1\u0026gt; USE master; 2\u0026gt; GO Changed database context to \u0026#39;master\u0026#39;. 1\u0026gt; SELECT name, database_id, create_date FROM sys.databases ; 2\u0026gt; GO name database_id create_date --------------------------- ----------- ----------------------- master 1 2003-04-08 09:13:36.390 tempdb 2 2018-11-02 16:30:37.907 model 3 2003-04-08 09:13:36.390 msdb 4 2018-10-19 01:18:57.300 (4 rows affected) Configuration If you need custom startup options, you can edit the ConfigMap file 15-mssql-configmap.yaml with your desired MS SQL configuration settings.\n","excerpt":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on …","ref":"/v2.0/docs/usecases/mssql/","title":"MS SQL"},{"body":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With mTLS both StorageOS and etcd authenticate each other ensuring that communication only happens between mutually authenticated end points, and that all communication is encrypted.\nStorageOS uses environment variables to specify the location of the necessary certificates and keys. This allows Kubernetes secrets to be mounted as volumes, directly into pods.\nSetting up mTLS with Etcd Below is an example StorageOSCluster resource that can be used to setup StorageOS with etcd using mTLS.\napiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:storageos-clusternamespace:\u0026#34;default\u0026#34;spec:images:nodeContainer:\u0026#34;storageos/node:v2.0.0\u0026#34;secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;default\u0026#34;namespace:\u0026#34;storageos\u0026#34;csi:enable:truedeploymentStrategy:\u0026#34;deployment\u0026#34;enableControllerPublishCreds:trueenableNodePublishCreds:trueenableProvisionCreds:true# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;etcd\u0026#34;# Namespace of the client certificates secretkvBackend:address:\u0026#34;https://storageos-etcd-cluster-client.storageos-etcd.svc:2379\u0026#34;# Etcd client service address.backend:\u0026#34;etcd\u0026#34;# Backend typetlsEtcdSecretRefName and tlsEtcdSecretRefNamespace are used to pass a reference to a secret that should contain the following key:values:\n etcd-client-ca.crt - containing the etcd Certificate Authority certificate etcd-client.crt - containing the etcd Client certificate etcd-client.key - cotaining the etcd Client key The StorageOS operator uses the etcd secret that contains the client certificates, to build a secret in the StorageOS installation namespace. This secret contains the certificate filenames and certificate file contents. The StorageOS daemonset that is created by the operator mounts the secret as a volume so that the certificate files are avaliable inside the pod. Environment variables containing the file paths are passed to the StorageOS process in order to use the files from the mounted path.\nA worked example of settings up StorageOS with external etcd using mTLS is avaliable here. For ease of use the example uses the CoreOS etcd operator and the CoreOS guide for Cluster TLS is followed.\n","excerpt":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With …","ref":"/v2.0/docs/operations/external-etcd/etcd-tls/","title":"Encrypting communication with Etcd"},{"body":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular web-based applications including WordPress.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MySQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiversion:apps/v1kind:statefulsetmetadata:name:mysqlspec:selector:matchlabels:app:mysqlenv:prodservicename:mysqlreplicas:1...spec:serviceaccountname:mysql...volumemounts:- name:datamountpath:/var/lib/mysqlsubpath:mysql- name:confmountpath:/etc/mysql/mysql.conf.d...volumeclaimtemplates:- metadata:name:datalabels:env:prodspec:accessmodes:[\u0026#34;readwriteonce\u0026#34;]storageclassname:\u0026#34;fast\u0026#34;# storageos storageclass resources:requests:storage:5giThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MySQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mysql Confirm MySQL is up and running.\n$ kubectl get pods -w -l app=mysql NAME READY STATUS RESTARTS AGE mysql-0 1/1 Running 0 1m Connect to the MySQL client pod and connect to the MySQL server through the service\n$ kubectl exec client -- mysql -h mysql-0.mysql -e \u0026#34;show databases;\u0026#34; Database information_schema mysql performance_schema Configuration If you need custom startup options, you can edit the ConfigMap file 15-mysqld-configmap.yaml with your desired MySQL configuration settings.\n","excerpt":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular …","ref":"/v2.0/docs/usecases/mysql/","title":"MySQL"},{"body":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a StorageOS cluster to be used by multiple teams across multiple projects.\nIt is not necessary to create StorageOS namespaces manually, as StorageOS maps Kubernetes namespaces on a one-to-one basis when PersistentVolumeClaims using the StorageOS StorageClass are created.\nAccess to Namespaces is controlled through user or group level policies.\n","excerpt":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a …","ref":"/v2.0/docs/concepts/namespaces/","title":"Namespaces"},{"body":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is created by default.\nNamespaces apply to volumes.\nManaging Namespaces In order to create a new namespace navigate to \u0026ldquo;Namespaces\u0026rdquo; in the GUI, and select \u0026ldquo;Create Namespace\u0026rdquo;.\nWhen a Kubernetes PVC is created in a namespace, StorageOS automatically maps the Volume in the same namespace. Namespaces are created by StorageOS to fulfil the RBAC rules enforced by Kubernetes roles.\nIn order to delete a namespace, all volumes must be deleted from the namespace before the namespace can be deleted.\n","excerpt":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is …","ref":"/v2.0/docs/operations/namespaces/","title":"Namespaces"},{"body":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load balancer or even as a Kubernetes ingress controller.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Nginx on Kubernetes You can find the latest files in the StorageOS use cases repostiory\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet definition\napiVersion:apps/v1kind:StatefulSetmetadata:name:nginxspec:serviceName:nginxspec:serviceAccountName:nginxcontainers:- name:nginximage:nginxports:- containerPort:80volumeMounts:- name:nginx-datamountPath:/usr/share/nginx/htmlsubPath:html- name:nginx-configmountPath:/etc/nginx/conf.dvolumes:- name:nginx-configconfigMap:name:nginxvolumeClaimTemplates:- metadata:name:nginx-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Nginx examples folder and create the objects\ncd storageos-usecases kubectl create -f ./nginx Confirm Nginx is up and running.\n$ kubectl get pods -w -l app=nginx NAME READY STATUS RESTARTS AGE nginx-0 1/1 Running 0 1m Connect to the nginx pod and write a file to /usr/share/nginx/html that Nginx will serve.\n$ kubectl exec nginx-0 -it -- bash root@nginx-0:/# echo Hello world! \u0026gt; /usr/share/nginx/html/greetings.txt Connect to the Busybox pod and connect to the Nginx server through the service and retrieve the directory index from Nginx.\n$ kubectl exec -it busybox -- /bin/sh / # wget -q -O- nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;greetings.txt\u0026#34;\u0026gt;greetings.txt\u0026lt;/a\u0026gt; 27-Feb-2019 12:04 13 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; Retrieve and display the contents of the greetings.txt file\n/ # wget -q -O- nginx/greetings.txt Hello world! Configuration If you need custom startup options, you can edit the ConfigMap file 15-nginx-configmap.yaml with your desired Nginx configuration settings.\n","excerpt":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load …","ref":"/v2.0/docs/usecases/nginx/","title":"Nginx"},{"body":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A node must be running a daemonset pod in order to consume and/or present storage.\nBy default StorageOS nodes run in hyperconverged mode. This means that the node hosts data from StorageOS volumes and can present volumes to applications.\nAlternatively, a node can run in computeonly mode, which means no storage is consumed on the node itself and the node only presents volumes hosted by other nodes. Volumes presented to applications running on compute only nodes are therefore all remote. Compute only nodes can be very useful for topologies where nodes are ephemeral and should not host data, but the ephemeral nodes host applications that require StorageOS volumes. The nodes that are not intended to hold data, but just to present StorageOS volumes, can be set as computeonly.\nA node can be marked as compute only at any point in time by adding the label storageos.com/computeonly=true, following the labels reference.\n","excerpt":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A …","ref":"/v2.0/docs/concepts/nodes/","title":"Nodes"},{"body":"The following document lists the open source software attributions in the StorageOS Control Plane, Data Plane and CLI.\nCNats The MIT License (MIT) Copyright (c) 2015 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CppUTest Copyright (c) 2007, Michael Feathers, James Grenning and Bas Vodde All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the \u0026lt;organization\u0026gt; nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE EARLIER MENTIONED AUTHORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL \u0026lt;copyright holder\u0026gt; BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Hayai Copyright (c) 2011 - Nick Bruun. This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. If you meet (any of) the author(s), you're encouraged to buy them a beer, a drink or whatever is suited to the situation, given that you like the software. This notice may not be removed or altered from any source distribution. LZ4 LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LibreSSL LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- ==================================================================== Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit. (http://www.openssl.org/)\u0026quot; 4. The names \u0026quot;OpenSSL Toolkit\u0026quot; and \u0026quot;OpenSSL Project\u0026quot; must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact openssl-core@openssl.org. 5. Products derived from this software may not be called \u0026quot;OpenSSL\u0026quot; nor may \u0026quot;OpenSSL\u0026quot; appear in their names without prior written permission of the OpenSSL Project. 6. Redistributions of any form whatsoever must retain the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)\u0026quot; THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==================================================================== This product includes cryptographic software written by Eric Young (eay@cryptsoft.com). This product includes software written by Tim Hudson (tjh@cryptsoft.com). Original SSLeay License ----------------------- Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) All rights reserved. This package is an SSL implementation written by Eric Young (eay@cryptsoft.com). The implementation was written so as to conform with Netscapes SSL. This library is free for commercial and non-commercial use as long as the following conditions are aheared to. The following conditions apply to all code found in this distribution, be it the RC4, RSA, lhash, DES, etc., code; not just the SSL code. The SSL documentation included with this distribution is covered by the same copyright terms except that the holder is Tim Hudson (tjh@cryptsoft.com). Copyright remains Eric Young's, and as such any Copyright notices in the code are not to be removed. If this package is used in a product, Eric Young should be given attribution as the author of the parts of the library used. This can be in the form of a textual message at program startup or in documentation (online or textual) provided with the package. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: \u0026quot;This product includes cryptographic software written by Eric Young (eay@cryptsoft.com)\u0026quot; The word 'cryptographic' can be left out if the rouines from the library being used are not cryptographic related :-). 4. If you include any Windows specific code (or a derivative thereof) from the apps directory (application code) you must include an acknowledgement: \u0026quot;This product includes software written by Tim Hudson (tjh@cryptsoft.com)\u0026quot; THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The licence and distribution terms for any publically available version or derivative of this code cannot be changed. i.e. this code cannot simply be copied and put under another distribution licence [including the GNU Public Licence.] protobuf This license applies to all parts of Protocol Buffers except the following: - Atomicops support for generic gcc, located in src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Code generated by the Protocol Buffer compiler is owned by the owner of the input file used when generating it. This code is not standalone and requires a support library to be linked with it. This support library is itself covered by the above license. rocksdb BSD License For rocksdb software Copyright (c) 2011-present, Facebook, Inc. All rights reserved. --------------------------------------------------------------------- Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/armon/go-metrics The MIT License (MIT) Copyright (c) 2013 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/armon/go-radix The MIT License (MIT) Copyright (c) 2014 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/bgentry/speakeasy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \u0026quot;[]\u0026quot; replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \u0026quot;printed page\u0026quot; as the copyright notice for easier identification within third-party archives. Copyright [2013] [the CloudFoundry Authors] Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/dgrijalva/jwt-go Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgrijalva/jwt-go/request Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgryski/go-metro MetroHash This package is a mechanical translation of the reference C++ code for MetroHash, available at https://github.com/jandrewrogers/MetroHash I claim no additional copyright over the original implementation. The MIT License (MIT) Copyright (c) 2015 J. Andrew Rogers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/docker/docker/pkg/mount Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv/store Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/golang/protobuf/proto Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/context Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/handlers Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/mux Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/websocket Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/consul/api Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/errwrap Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-cleanhttp Mozilla Public License, version 2.0 1. Definitions 1.1. \u0026quot;Contributor\u0026quot; means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. \u0026quot;Contributor Version\u0026quot; means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. \u0026quot;Contribution\u0026quot; means Covered Software of a particular Contributor. 1.4. \u0026quot;Covered Software\u0026quot; means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. \u0026quot;Incompatible With Secondary Licenses\u0026quot; means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. \u0026quot;Executable Form\u0026quot; means any form of the work other than Source Code Form. 1.7. \u0026quot;Larger Work\u0026quot; means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. \u0026quot;License\u0026quot; means this document. 1.9. \u0026quot;Licensable\u0026quot; means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. \u0026quot;Modifications\u0026quot; means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. \u0026quot;Patent Claims\u0026quot; of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. \u0026quot;Secondary License\u0026quot; means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. \u0026quot;Source Code Form\u0026quot; means the form of the work preferred for making modifications. 1.14. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) means an individual or a legal entity exercising rights under this License. For legal entities, \u0026quot;You\u0026quot; includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, \u0026quot;control\u0026quot; means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an \u0026quot;as is\u0026quot; basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - \u0026quot;Incompatible With Secondary Licenses\u0026quot; Notice This Source Code Form is \u0026quot;Incompatible With Secondary Licenses\u0026quot;, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-msgpack/codec Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/go-multierror Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/memberlist Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/coordinate Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/serf Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/kelseyhightower/envconfig Copyright (c) 2013 Kelsey Hightower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/mattn/go-isatty Copyright (c) Yasuhiro MATSUMOTO \u0026lt;mattn.jp@gmail.com\u0026gt; MIT License (Expat) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/miekg/dns Extensions of the original work are copyright (c) 2011 Miek Gieben As this is fork of the official Go code the same license applies: Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/mitchellh/cli Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/conf The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/logger The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server/pse The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/util The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats/encoders/builtin The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nuid The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/pborman/uuid Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/Sirupsen/logrus The MIT License (MIT) Copyright (c) 2014 Simon Eskildsen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/thejerf/suture Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/urfave/negroni The MIT License (MIT) Copyright (c) 2014 Jeremy Saenz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang.org/x/crypto/bcrypt Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/crypto/blowfish Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/net/context Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/unix Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/registry Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/svc/eventlog Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/time/rate Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. gopkg.in/yaml.v2 Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ","excerpt":"The following document lists the open source software attributions in the StorageOS Control Plane, …","ref":"/v2.0/docs/reference/open_source_attribution/","title":"StorageOS Open Source Software Attribution Notice"},{"body":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform actions such as creation, deletion and mounting of volumes through CSI. The CSI container running in the StorageOS Daemonset creates a Linux socket that allows the communication between OpenShift and StorageOS\nCSI (Container Storage Interface) Note CSI is the standard that enables storage drivers to release on their own schedule. This allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nStorageOS v2 implements communication with the OpenShift controlplane with CSI only. The native driver has been deprecated\nStorageOS PersistentVolumeClaims The user can provide standard PVC definitions and StorageOS will dynamically provision them. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nInstallation StorageOS v2 supports OpenShift 4.0, 4.1, 4.2 and 4.3.\nTo install StorageOS on OpenShift, please follow our installation instructions page.\n","excerpt":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform …","ref":"/v2.0/docs/platforms/openshift/","title":"OpenShift"},{"body":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a user or group access to a namespace, a policy needs to be created mapping the user or group to the namespace.\n Note: Users always have access to the default namespace\n For more information on how to use policies, see the Policies operations page.\n","excerpt":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a …","ref":"/v2.0/docs/concepts/policies/","title":"Policies"},{"body":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user level so access can be controlled granularly.\nUsers can belong to one or more groups to control their namespace permissions. Additionally user specific policies can be created to grant a user access to a namespace. Users can belong to any number of groups and have any number of user level policies configured.\n Note: Users are created with access to the default namespace. Policies cannot be applied to the default namespace.\n Managing Policies To start creating policies, at least one custom namespace and user are required. For more information on how to create namespaces see our Namespace guide, for users see our Users CLI reference.\nIn order to create a policy navigate to \u0026ldquo;Policies\u0026rdquo; in the GUI and select \u0026ldquo;Create Policy\u0026rdquo;. A policy controls access to a variety of StorageOS resources and is applied to a user, by placing the user in the policies group.\nIn order to delete a policy, all users must be removed from the policy group before deletion of the policy can be completed.\n","excerpt":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user …","ref":"/v2.0/docs/operations/policies/","title":"Policies"},{"body":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational database management system (ORDBMS).\nPostgres is deployed across a wide variety of platforms with a mix of workloads ranging from small, single-node use cases to large Internet-facing clusters with many concurrent users.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. If you need to setup StorageOS on Kubernetes then please see our guide to [installing StorageOS on Kubernetes]({% link \u0026ldquo;docs/install/kubernetes.md\u0026rdquo; \u0026gt;}}).\nDeploying PostgreSQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases PersistentVolumeClaim and Pod definition excerpts\nkind:PersistentVolumeClaimmetadata:name:pg-dataannotations:volume.beta.kubernetes.io/storage-class:fast...kind:Podmetadata:name:postgressspec:securityContext:fsGroup:26containers:- name:pgimage:crunchydata/crunchy-postgres:centos7-10.4-1.8.3volumeMounts:- mountPath:/pgdataname:data...volumes:- name:datapersistentVolumeClaim:claimName:pg-dataThis excerpt is from the PersistentVolumeClaim and Pod definition. The pod definition references the pg-data VolumeClaim so storage is dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the PostgreSQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./postgres Confirm PostgreSQL is up and running.\n$ kubectl get pod postgres-0 -w NAME READY STATUS RESTARTS AGE postgres-0 1/1 Running 0 1m Connect to the PostgreSQL client pod and connect to the PostgreSQL server through the service.\n$ kubectl exec -it postgres-0 -- psql -h postgres-0.postgres -U primaryuser postgres -c \u0026#34;\\l\u0026#34; Password for user primaryuser: password List of Databases Name | Owner | Encoding | Collate | Ctype | Access privileges +=========================================================================+ postgres | postgres | SQL_ASCII | C | C | template0 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres template1 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres userdb | postgres | SQL_ASCII | C | C | =Tc/postgres + | | | | | postgres=CTc/postgres+ | | | | | testuser=CTc/postgres (4 rows) The password for the primary user is password. You can see this is set in the ConfigMap file.\n Configuration If you need custom startup options, you can edit the ConfigMap file 15-postgresd-configmap.yaml with your desired PostgreSQL configuration settings.\n","excerpt":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational …","ref":"/v2.0/docs/usecases/postgres/","title":"PostgreSQL"},{"body":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting in Kubernetes.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Prometheus on Kubernetes This is the Prometheus use case for StorageOS. Following are the steps for creating a Prometheus instance and using StorageOS to handle its persistent storage.\n You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases Prometheus Custom Resource defintion\napiVersion:monitoring.coreos.com/v1kind:Prometheusmetadata:name:prometheus-storageoslabels:app:prometheus-operatorspec:...storage:volumeClaimTemplate:metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:storageos-replicatedresources:requests:storage:1GiThis excerpt is from the Prometheus Custom Resource definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Prometheus examples folder and create the objects\n$ cd storageos-usecases/prometheus $ ./install-prometheus.sh Confirm Prometheus is up and running.\n$ kubectl get pods -w -l app=prometheus NAME READY STATUS RESTARTS AGE prometheus-prometheus-storageos-0 3/3 READY 0 1m You can see the created PVC using.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-prometheus-prometheus-storageos-0 Bound pvc-b6c17c0a-e76b-4a0b-8fc6-46c0e1629210 1Gi RWO storageos-replicated 65m In the Prometheus deployment script, a service monitor is also created. The new Prometheus instance will use the storageos-etcd service monitor to start scraping metrics from the storageos-etcd pods. (Assuming the storageos cluster was setup using ETCD as pods) For more information about service monitors, have a look at the upstream documentation.\n$ kubectl get servicemonitor NAME AGE storageos-etcd 5d1h Port forward in the prometheus pod to access the prom webapp.\n$ kubectl port-forward prometheus-prometheus-storageos-0 9090 Then launch a web browser and type the url localhost:9090 to access the prometheus webapp. Confirm that prometheus is up and running there.\n Configuration In the storageos-usecases/prometheus/manifests/prometheus directory exist examples of Service Monitors. For more information at operating Prometheus, check out the prometheus documentation.\n","excerpt":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting …","ref":"/v2.0/docs/usecases/prometheus/","title":"Prometheus"},{"body":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS transparently supports Rancher deployments using CentOS, RHEL or Debian and can support other Linux distributions detailed in the systems supported page if the appropriate kernel modules are present.\n","excerpt":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS …","ref":"/v2.0/docs/platforms/rancher/","title":"Rancher"},{"body":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional durability to disk.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Redis on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:redisspec:selector:matchLabels:app:redisenv:prodserviceName:redisreplicas:1...spec:serviceAccountName:redis...volumeMounts:- name:datamountPath:/bitnami/redis/data...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the Redis examples folder and create the objects\ncd storageos-usecases kubectl create -f ./redis Confirm Redis is up and running.\n$ kubectl get pods -w -l app=redis NAME READY STATUS RESTARTS AGE redis-0 1/1 Running 0 1m Connect to the Redis client pod and connect to the Redis server through the service\n$ kubectl exec -it redis-0 -- redis-cli -a password Warning: Using a password with \u0026#39;-a\u0026#39; option on the command line interface may not be safe. 127.0.0.1:6379\u0026gt; CONFIG GET maxmemory 1) \u0026#34;maxmemory\u0026#34; 2) \u0026#34;0\u0026#34; Configuration If you need custom startup options, you can edit the ConfigMap file 15-redis-configmap.yaml with your desired Redis configuration settings.\n","excerpt":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional …","ref":"/v2.0/docs/usecases/redis/","title":"Redis"},{"body":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than \u0026ldquo;latest\u0026rdquo;, and to perform upgrades only after reading the release notes.\nThe latest tagged release is v2.0.0. For installation instructions please see our Install page.\nThe latest CLI release is v2.0.0, available from Github\nUpgrading To upgrade from version 1.x to 2.x, please contact support for assistance.\nv2.2.0 - Released 2020-08-18 This release focuses on performance. We analysed existing performance characteristics across a variety of real-world use cases and ended up with improvements across the board. Of particular note:\n Sequential reads have improved by up to 130% Sequential writes have improved by up to 737% Random reads have improved by up to 45% Random writes have improved by up to 135% I/O for large block sizes (128K) has improved by up to 353% We are extremely proud of our performance and we love to talk about it. Have a look at the Benchmarking section of the self-evaluation guide and consider sharing your results. Our PRE engineers are available to discuss in our slack channel.\nNew Data engine revamp focused on provable consistency and performance. Key characteristics:\n Metadata is stored in an optimised index, lowering I/O latency and improving performance for all workloads. Large block reads/writes are now be handled in a single operation. Applications like Kafka will go much faster. On-disk compression is now disabled by default as in most scenarios this offers better performance. To enable on-disk compression for a specific workload, see compression.\n Improved dataplane: The number of I/O threads are now determined by the number of processing cores available. This improves scalability and performance on larger servers. ha: Improve partition tolerance behaviour when a volume master that has lost its connection to etcd rejoins. ha: Allow replicas in unhealthy states to be remediated and re-used while maintaining partition tolerance. ha: When a master fails and the new master is not yet available, introduce a back-off to the redirection logic to avoid spamming the logs with connection failure errors. ha: Ignore health advertisements for local node. Local nodes are handled directly. node delete: Only refuse to delete a node if the node health can be authoritatively verified to be in use. api: Increase HTTP server write timeout. cli/ui: Allow partial diagnostic bundle downloads. ui: Namespace dropdown can now be scrolled. ui: Add \u0026ldquo;Job title\u0026rdquo; to UI licence form. logging: Log version at startup at INFO level. logging: Lower verbosity of SCSI warnings that do not apply to StorageOS. diagnostics: Include logs that have been rotated. diagnostics: Bundle collection across providers is now done in parallel. build: Update base image to RHEL 8.2. operator: Removed DB migration utility required for v1.3 -\u0026gt; v1.4 upgrades. operator: Automatically refreshes StorageOS API token without failing requests when the token expires. operator: Updated CSI attacher and provisioner to latest upstream. operator: Remove cluster.local suffix on Pod scheduler service address. This allows the scheduler to work in clusters with custom DNS configuration. operator: Defaults are now set for most CSI configuration options in the StorageOSCluster custom resource. Fixed csi: When unmount request is received for a volume that has already been unmounted, return success. csi: Verify volume is attached on the node before mounting it. xfs: Support older RHEL kernels which have an XFS library that does not allow reflinks/dedupe. dataplane: Reserve 1GiB of capacity on the target disk to allow manual cleanup operations, rather than filling target disk to capacity. operator: In some cases /var/lib/storageos could fail to unmount cleanly after a restart. This resulted in multiple entries in /proc/mounts. v2.1.0 - Released 2020-06-26 New csi: Volume expansion now supported in offline mode. To expand a volume, stop any workloads accessing the volume, then edit the PVC to increase the capacity. For more information, see our Volume Resize operations page and the CSI Volume Expansion page. api: Volume configuration including replica count can now be updated while the volume is in use. Other updateable fields include labels and description. failover: Before determining that a node is offline and performing recovery operations, the I/O path is also verified. This provides more robust failure detection and ensures that nodes that are still responding to I/O do not get replaced. This I/O path verification is in addition to the gossip-based failure detection. operator: Default tolerations are now set for the StorageOS node container. This helps ensure that the StorageOS node container does not get evicted when the node is running low on resources. Improved api: Added checks to prevent deletion of a node with active volumes, or if it is the master of at least one volume. This helps prevent orphaned volumes. cli: Add an --offline-delete flag to allow removal of volumes whose master and replica nodes are offline. This allows cleanup of orphaned volumes. ui: Add an offline volume delete option. ui: Volumes can now be detached from the UI. cli: Labels are no longer truncated. api: When a new node is added to the cluster, its capacity is available to use immediately. Fixed ui: Favicon was missing. ui: Duplicate volumes could be shown on the node details page. operator: During uninstall a ClusterRoleBinding was not removed. v2.0.0 - Released 2020-05-05 New operator: StorageOS containers now run in the kube-system namespace by default to allow the system-node-critical priority class to be set. This instructs Kubernetes to start StorageOS before application Pods, and to evict StorageOS only after application Pods have finished. This setting was previously recommended in documentation; it is now the default. operator: StorageOS CSI helper containers now run as privileged. This ensures that the CSI endpoint can be seen on systems with SELinux enabled. ui: replication progress for new or re-joining replicas is now displayed. ui: show warning for unlicensed clusters. cli: new commands: licence management get policy create namespace create policy describe user describe namespace describe policy delete user delete namespace delete policy licence: removed the default licence expiry date added for v2.0.0-rc.1. Improved dataplane: improved retry behaviour for network I/O. cli: \u0026ldquo;get volumes\u0026rdquo; for all namespaces should be done in parallel. cli: help text document config file ui: link node name and get to node details on the volume details page. ui: node details add available capacity spinner. ui: node list remove capacity values / address port. ui: node list show master/replica counts. ui: node list remove edit action. ui: format entity labels. ui: node details link volumes. ui: align buttons for licences. ui: k8s warning in \u0026ldquo;create volume\u0026rdquo; modal. ui: node list remove \u0026ldquo;API\u0026rdquo; from \u0026ldquo;API Address\u0026rdquo; ui: add some details about the Licence on the licence page. api: include valid for duration in login response. licence: restrict nodes which are unregistered after 24 hours. scheduler: return error for namespace/volume not found dataplane: start gRPC threads separately from rest of the supervisor. Fixed ui: centre licence types. ui: capacity in ui is per namespace. cli: fail gracefully if missing some output details (i.e. no node exists for id). v2.0.0-rc.1 - Released 2020-03-31 Initial release of version 2.x. See StorageOS v2.0 Release Blog for details.\n","excerpt":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than …","ref":"/v2.0/docs/reference/release_notes/","title":"Release notes"},{"body":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous replication ensures strong consistency for applications such as databases and Elasticsearch, incurring one network round trip on writes.\nThe basic model for StorageOS replication is of a master volume with distributed replicas. Each volume can be replicated between 0 and 5 times, which are provisioned to 0 to 5 nodes, up to the number of remaining nodes in the cluster.\nIn this diagram, the master volume D was created on node 1, and two replicas, D2 and D3 on nodes 3 and 5.\nWrites that come into D (step 1) are written in parallel to D2 and D3 (step 2). When both replicas and the master acknowledge that the data has been written (step 3), the write operation return successfully to the application (step 4).\nFor most applications, one replica is sufficient (storageos.com/replicas=1).\nAll replication traffic on the wire is compressed using the lz4 algorithm, then streamed over tcp/ip to target port tcp/5703.\nIf the master volume is lost, a replica is promoted to master (D2 or D3 above) and a new replica is created and synced on an available node (Node 2 or 4). This is transparent to the application and does not cause downtime.\nIf a replica volume is lost and there are enough remaining nodes, a new replica is created and synced on an available node. While a new replica is created and being synced, the volume\u0026rsquo;s health will be marked as degraded.\nIf the lost replica comes back online before the new replica has finished synchronizing, then StorageOS will calculate which of the two synchronizing replicas has the smallest difference compared to the master volume and keep that replica. The same holds true if a master volume is lost and a replica is promoted to be the new master. If possible, a new replica will be created and begin to sync. Should the former master come back online it will be demoted to a replica and the replica will the smallest difference to the current master will be kept.\nWhile the replica count is controllable on a per-volume basis, some environments may prefer to set default labels on the StorageClass.\nDelta Sync StorageOS implements a delta sync between a volume master and its replicas. This means that if a replica for a volume goes offline, that when the replica comes back online only the regions with changed blocks need to be synchronized. This optimization reduces the time it takes for replicas to catch up, improving volume resilience. Additionally, it reduces network and IO bandwidth which can reduce costs when running in public clouds.\n","excerpt":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous …","ref":"/v2.0/docs/concepts/replication/","title":"Replication"},{"body":"","excerpt":"","ref":"/v2.0/search/","title":"Search Results"},{"body":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for instance, StorageOS. A StorageClass defines parameters to pass to the provisioner, which in the case of StorageOS can be translated into behaviour applied to the Volumes. Many StorageClasses can be provisioned to apply different feature labels to the StorageOS Volumes.\nBy default the StorageOS Cluster Operator installs the fast StorageClass at bootstrap of StorageOS. You can define its name in the StorageOS Cluster Resource.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:fastprovisioner:csi.storageos.comparameters:csi.storage.k8s.io/fstype:ext4pool:default# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretStorageClasses can be created to define default labels for StorageOS volumes, but also to map to any semantic aggregation of volumes that suits your use case, whether there are different roles (dev, staging, prod), or a StorageClass maps to a team or customer using the cluster.\nExamples You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic StorageClass definition in v2-storageclass-replicated.yaml.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThat StorageClass can be used by a PVC:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels.\nYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\n","excerpt":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for …","ref":"/v2.0/docs/operations/storageclasses/","title":"Kubernetes StorageClasses"},{"body":"","excerpt":"","ref":"/v2.0/","title":"StorageOS"},{"body":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for two different purposes.\nTelemetry Telemetry is made up a DNS version check query and a once per day report. Error reporting is the sentry.io crash dump reporting. sentry.io StorageOS sends crash reports to sentry.io. This information helps our developers monitor and fix crashes. Information is sent to sentry.io when a process inside the StorageOS container crashes.\n The crash report contains the signal that triggered the shutdown (e.g. SIGSEGV), the exit code and whether or not the crash generated a core dump. All StorageOS clusters with a routable connection to the internet will send crash reports to sentry.io over tcp/443. StorageOS respects environment variables that ProxyFromEnvironment uses.\nAn exhaustive list of information included in the crash report is below:\n StorageOS version Crash description string Anonymized Cluster ID Anonymized Node ID DNS Query StorageOS will perform a \u0026ldquo;latest version check\u0026rdquo; using a DNS query in order to inform administrators that a new version is available. StorageOS will also send anonymized node ids, cluster id and StorageOS version information to StorageOS using a DNS query. The information that we send in the query is encoded as well as being anonymized. This query allows us to inform Cluster admins when StorageOS upgrades are available in the StorageOS GUI and in the logs.\nThe DNS query includes:\n Anonymized StorageOS Cluster ID Anonymized StorageOS node ID StorageOS version number Once Per Day Report The once per day report contains information about the StorageOS cluster and Kubernetes versions to help StorageOS focus our development efforts on the most popular platforms. The once per day data is encrypted and sent to a StorageOS telemetry server so it is never processed outside of StorageOS assets.\nAn exhaustive list of information included in the once per day report is below:\n api_call_metrics cluster_disable_crash_reporting cluster_disable_version_check cluster_log_format cluster_log_level cluster_tls_provided k8s_distribution k8s_in_k8s k8s_scheduler_extender_enabled k8s_version node_available_bytes node_capacity node_crash_files_on_disk node_created_at_time node_etcd_config node_etcd_namespacing_enabled node_etcd_tls_enabled node_free_bytes node_health node_http_tls_enabled node_id node_labels node_status node_storageos_version node_system_clock_time node_total_bytes node_version volume_fs_type volume_id volume_labels volume_master_attach volume_master_delete_deployment volume_master_detach volume_master_failover_deployment volume_master_promote volume_master_provision volume_master_recover_replica volume_master_trigger_rejoin volume_metrics volume_placement_strategy volume_provision_source_user_agent volume_replica_delete_deployment volume_replica_failover_deployment volume_replica_promote volume_replica_provision volume_replicas volume_replica_trigger_rejoin volume_size_bytes Disable Telemetry It is possible to disable telemetry using the GUI, CLI, API, environment variables or the StorageOS Cluster Spec.\nStorageOS Cluster Spec Disable telemetry explicitly through the configurable spec parameters of the StorageOSCluster custom resource.\nEnvironment Variables You can use the following environmental variables to disable or enable telemetry.\nDISABLE_VERSION_CHECK # Disable the DNS version check DISABLE_TELEMETRY # Disable the once per day reporting DISABLE_ERROR_REPORTING # Disable sentry.io crash reports ","excerpt":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for …","ref":"/v2.0/docs/reference/telemetry/","title":"StorageOS Telemetry"},{"body":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The bundle packages the information needed for the engineering team to understand the context of your cluster and begin troubleshooting any issues you may be experiencing.\nGenerate Bundle CLI The StorageOS CLI can generate the bundle.\n$ storageos get diagnostics The file generated is in the form of diagnostics-${TIME_STAMP}.gz\n GUI Or you can use the StoregeOS GUI.\n Go to section \u0026ldquo;Cluster\u0026rdquo; Press the button \u0026ldquo;DOWNLOAD DIAGNOSTICS\u0026rdquo;. Data collected in the bundle Most of the data collected in the bundle is regarding the state of the StorageOS cluster, however some other information regarding the infrastructure is also gathered. The information is used to have a clear view of the cluster where StorageOS is running.\nThe bundle incorporates for each node:\n StorageOS Daemonset Pod logs lshw dmesg (kernel logs) StorageOS metadata for the ControlPlane and DataPlane StorageOS metadata collected cluster metadata namespaces metadata nodes metadata volumes metadata capacity stats environment variables health Privacy StorageOS can only obtain the bundle if it is downloaded by the user and given to our engineering team, or uploaded for analysis. The data received by StorageOS is private and never leaves nor will leave StorageOS Inc.\nThe data contained in the cluster diagnostic bundle has the sole purpose of helping customers troubleshoot their issues.\n","excerpt":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The …","ref":"/v2.0/docs/reference/bundle/","title":"Support bundle"},{"body":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base directory for storing configuration and blob files. Recommended host filesystem types are ext4 or xfs. If you require a specific filesystem please contact StorageOS.\nPersistent Volume Filesystems StorageOS provides a block device on which a file system can be created. The creation of the filesystem is either handled by StorageOS or by Kubernetes which affects what filesystems can be created.\nCSI Driver When using StorageOS with the CSI driver, StorageOS is responsible for running mkfs against the block device that pods mount. StorageOS is able to create ext2, ext3, ext4 and xfs file systems.\n","excerpt":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base …","ref":"/v2.0/docs/reference/filesystems/","title":"Supported File systems"},{"body":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain node taints. Taints can be thought of as the opposite of Node affinity in that taints repel pods. Node taints are automatically applied by Kubernetes in response to node resources coming under contention. As StorageOS provides storage to pods it should not be evicted during periods of resource contention, as any pods using StorageOS volumes on the same node would need to be restarted.\nAs StorageOS runs as a daemonset some Kubernetes tolerations are added by Kubernetes while others are automatically added by the StorageOS operator.\nFor more information about tolerations please see the Kubernetes documentation.\ntolerations:# The unreachable and not-ready tolerations are added by Kubernetes to daemonsets automatically- key:\u0026#34;node.kubernetes.io/unreachable\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;- key:\u0026#34;node.kubernetes.io/not-ready\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;# The following tolerations are added to the StorageOS daemonset by the StorageOS operator- key:node.kubernetes.io/disk-pressureoperator:Exists- key:node.kubernetes.io/memory-pressureoperator:Exists- key:node.kubernetes.io/network-unavailableoperator:Exists- key:node.kubernetes.io/out-of-diskoperator:Exists- key:node.kubernetes.io/pid-pressureoperator:Exists- key:node.kubernetes.io/unschedulableoperator:ExistsAdding Custom Tolerations To add custom tolerations to the StorageOS daemonset configure them in the StorageOSCluster resource.\n","excerpt":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain …","ref":"/v2.0/docs/reference/tolerations/","title":"Tolerations"},{"body":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to the StorageOS installation, integration with orchestrators or common misconfigurations.\nTools To be able to troubleshoot issues the StorageOS cli is required.\nPod in pending because of mount error Issue: The output of kubectl describe pod $POD_ID contains no such file or directory and references the StorageOS volume device file.\nroot@node1:~# kubectl -n kube-system describe $POD_ID (...) Events: (...) Normal Scheduled 11s default-scheduler Successfully assigned default/d1 to node3 Warning FailedMount 4s (x4 over 9s) kubelet, node3 MountVolume.SetUp failed for volume \u0026#34;pvc-f2a49198-c00c-11e8-ba01-0800278dc04d\u0026#34; : stat /var/lib/storageos/volumes/d9df3549-26c0-4cfc-62b4-724b443069a1: no such file or directory Reason: There are two main reasons this issue may arise:\n The StorageOS DEVICE_DIR location is wrongly configured when using Kubelet as a container Mount Propagation is not enabled (Option 1) Misconfiguration of the DeviceDir/SharedDir\nSome Kubernetes distributions such as Rancher, DockerEE or some installations of OpenShift deploy the Kubelet as a container, because of this, the device files that StorageOS creates to mount into the containers need to be visible to the kubelet. StorageOS can be configured to share the device directory.\nModern installations use CSI, which handles the complexity internally.\nAssert: root@node1:~# kubectl -n default describe stos | grep \u0026#34;Shared Dir\u0026#34; Shared Dir: # \u0026amp;lt;-- Shouldn\u0026#39;t be blank Solution: The Cluster Operator Custom Definition should specify the SharedDir option as follows.\nspec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container ... See example on how to configure the StorageOS Custom Resource.\n (Option 2) Mount propagation is not enabled.\n Applies only if Option 1 is configured properly.\n Assert: If not using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/storageos/volumes is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/storageos/volumes/ root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/storageos/volumes bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nIf using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/kubelet/plugins/kubernetes.io~storageos/devices is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/kubelet/plugins/kubernetes.io~storageos/devices root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/kubelet/plugins/kubernetes.io~storageos/devices bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nSolution: Older versions of Kubernetes need to enable mount propagation as it is not enabled by default. Most Kubernetes distributions allow MountPropagation to be enabled using FeatureGates. Rancher specifically, needs to enable it in the \u0026ldquo;View in API\u0026rdquo; section of your cluster. You need to edit the section \u0026ldquo;rancherKubernetesEngineConfig\u0026rdquo; to enable the Kubelet feature gate.\nPVC pending state - Failed to dial StorageOS A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: root@node1:~/# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE vol-1 Pending fast 7s kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 7s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: Get http://storageos-cluster/version: failed to dial all known cluster members, (10.233.59.206:5705) Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount volume can\u0026rsquo;t be transmitted to StorageOS, hence the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nIn this case, the Event message indicates that StorageOS API is not responding, implying that StorageOS is not running. For Kubernetes to define StorageOS pods ready, the health check must pass.\nAssert: Check the status of StorageOS pods.\nroot@node1:~/# kubectl -n kube-system get pod --selector app=storageos # for CSI add --selector kind=daemonset NAME READY STATUS RESTARTS AGE storageos-qrqkj 0/1 Running 0 1m storageos-s4bfv 0/1 Running 0 1m storageos-vcpfx 0/1 Running 0 1m storageos-w98f5 0/1 Running 0 1m If the pods are not READY, the service will not forward traffic to the API they serve hence PVC will remain in pending state until StorageOS pods are available.\n Kubernetes keeps trying to execute the action until it succeeds. If a PVC is created before StorageOS finish starting, the PVC will be created eventually.\n Solution: StorageOS health check takes 60 seconds of grace before reporting as READY. If StorageOS is starting properly after that period, the volume will be created when StorageOS finishes its bootstrap. If StorageOS is not running or is not starting properly, the solution would be to troubleshoot the installation. PVC pending state - Secret Missing A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 13s (x2 over 28s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: failed to get secret from [\u0026#34;storageos\u0026#34;/\u0026#34;storageos-api\u0026#34;] Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount a volume can\u0026rsquo;t be transmitted to StorageOS, and the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nThe StorageClass provisioned for StorageOS references a Secret from where it retrieves the API endpoint and the authentication parameters. If that secret is incorrect or missing, the connections won\u0026rsquo;t be established. It is common to see that the Secret has been deployed in a different namespace where the StorageClass expects it or that is has been deployed with a different name.\nAssert: Check the StorageClass parameters to know where the Secret is expected to be found.\nkubectl get storageclass fast -o yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: 2018-09-25T08:44:57Z labels: app: storageos name: fast resourceVersion: \u0026#34;108853\u0026#34; selfLink: /apis/storage.k8s.io/v1/storageclasses/fast uid: 48490a9b-c09f-11e8-ba01-0800278dc04d parameters: adminSecretName: storageos-api adminSecretNamespace: storageos description: Kubernetes volume fsType: ext4 pool: default provisioner: kubernetes.io/storageos reclaimPolicy: Delete Note that the parameters specify adminSecretName and adminSecretNamespace.\n Check if the secret exists according to those parameters\nkubectl -n kube-system get secret storageos-api No resources found. Error from server (NotFound): secrets \u0026#34;storageos-api\u0026#34; not found If no resources are found, it is clear that the Secret doesn\u0026rsquo;t exist or it is not deployed in the right location.\n Solution: Deploy StorageOS following the installation procedures. If you are using the manifests provided for Kubernetes to deploy StorageOS rather than using automated provisioners, make sure that the StorageClass parameters and the Secret reference match.\nPeer discovery - Pod allocation Issue: StorageOS nodes can\u0026rsquo;t join the cluster and show the following log entries.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover the nodes in the cluster. When StorageOS starts, one or more active nodes must be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nIf there are no active StorageOS nodes, the bootstrap process will elect the first node in the JOIN variable as master, and the rest will try to discover from it. In case of that node not starting, the whole cluster will remain unable to bootstrap.\nInstallations of StorageOS use a DaemonSet, and by default do not schedule StorageOS pods to master nodes, due to the presence of the node-role.kubernetes.io/master:NoSchedule taint that is typically present. In such cases the JOIN variable must not contain master nodes or the StorageOS cluster will remain unable to start.\nAssert: Check that the first node of the JOIN variable started properly.\nroot@node1:~/# kubectl -n kube-system describe ds/storageos | grep JOIN JOIN: 172.28.128.3,172.28.128.4,172.28.128.5 root@node1:~/# kubectl -n kube-system get pod -o wide | grep 172.28.128.3 storageos-8zqxl 1/1 Running 0 2m 172.28.128.3 node1 Solution: Make sure that the JOIN variable doesn\u0026rsquo;t specify the master nodes. In case you are using the discovery service, it is necessary to ensure that the DaemonSet won\u0026rsquo;t allocate Pods on the masters. This can be achieved with taints, node selectors or labels.\nFor installations with the StorageOS operator you can specify which nodes to deploy StorageOS on using nodeSelectors. See examples in the Cluster Operator Examples page.\nFor more advanced installations using compute-only and storage nodes, check the storageos.com/deployment=computeonly label that can be added to the nodes through Kubernetes node labels, or StorageOS in the Labels page.\nLIO Init:Error Issue: StorageOS pods not starting with Init:Error\nkubectl -n kube-system get pod NAME READY STATUS RESTARTS AGE storageos-2kwqx 0/3 Init:Err 0 6s storageos-cffcr 0/3 Init:Err 0 6s storageos-d4f69 0/3 Init:Err 0 6s storageos-nhq7m 0/3 Init:Err 0 6s Reason: This indicates that since the Linux open source SCSI drivers are not enabled, StorageOS cannot start. The StorageOS DaemonSet enables the required kernel modules on the host system. If you are seeing these errors it is because that container couldn\u0026rsquo;t load the modules.\nAssert Check the logs of the init container.\nkubectl -n kube-system logs $ANY_STORAGEOS_POD -c storageos-init In case of failure, it will show the following output, indicating which kernel modules couldn\u0026rsquo;t be loaded or that they are not properly configured:\nChecking configfs configfs mounted on sys/kernel/config Module target_core_mod is not running executing modprobe -b target_core_mod Module tcm_loop is not running executing modprobe -b tcm_loop modprobe: FATAL: Module tcm_loop not found. Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and delete StorageOS pods, allowing the DaemonSet to create the pods again.\nLIO not enabled Issue: StorageOS node can\u0026rsquo;t start and shows the following log entries.\ntime=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;liocheck returned error\u0026#34; category=liocheck error=\u0026#34;exit status 1\u0026#34; module=dataplane stderr=\u0026#34;Sysfs root \u0026#39;/sys/kernel/config/target\u0026#39; is missing, is kernel configfs present and target_core_mod loaded? category=fslio level=warn\\nRuntime error checking stage \u0026#39;target_core_mod\u0026#39;: SysFs root missing category=fslio level=warn\\nliocheck: FAIL (lio_capable_system() returns failure) category=fslio level=fatal\\n\u0026#34; stdout= time=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;failed to start dataplane services\u0026#34; error=\u0026#34;system dependency check failed: exit status 1\u0026#34; module=command Reason: This indicates that one or more kernel modules required for StorageOS are not loaded.\nAssert The following kernel modules must be enabled in the host.\nlsmod | egrep \u0026#34;^tcm_loop|^target_core_mod|^target_core_file|^configfs\u0026#34; Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and restart the container.\n(OpenShift) StorageOS pods missing \u0026ndash; DaemonSet error StorageOS DaemonSet doesn\u0026rsquo;t have any pod replicas. The DaemonSet couldn\u0026rsquo;t allocate any Pod due to security issues.\nIssue: [root@master02 standard]# oc get pod No resources found. [root@master02 standard]# oc describe daemonset storageos (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedCreate 0s (x12 over 10s) daemonset-controller Error creating: pods \u0026#34;storageos-\u0026#34; is forbidden: unable to validate against any security context constraint: [provider restricted: .spec.securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used provider restricted: .spec.securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.volumes[0]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[1]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[2]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[3]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.initContainers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.initContainers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.initContainers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.initContainers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.containers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.containers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.containers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used] Reason: The OpenShift cluster has security context constraint policies enabled that forbid any pod, without an explicitly set policy for the service account, to be allocated.\nAssert: Check if the StorageOS ServiceAccount can create pods with enough permissions\noc get scc privileged -o yaml # Or custom scc with enough privileges (...) users: - system:admin - system:serviceaccount:openshift-infra:build-controller - system:serviceaccount:management-infra:management-admin - system:serviceaccount:management-infra:inspector-admin - system:serviceaccount:storageos:storageos \u0026amp;lt;-- - system:serviceaccount:tiller:tiller If the StorageOS sa system:serviceaccount:storageos:storageos is in the privileged scc it will be able to create pods.\nSolution: Add the ServiceAccount system:serviceaccount:storageos:storageos to a scc with enough privileges.\noc adm policy add-scc-to-user privileged system:serviceaccount:storageos:storageos Getting Help If our troubleshooting guides do not help resolve your issue, please see our support section for details on how to get in touch with us.\n","excerpt":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to …","ref":"/v2.0/docs/operations/troubleshooting/","title":"Troubleshooting"},{"body":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces using policies.\n Note: Users are created with access to the default namespace. This access is only revoked when a policy is created for the user or their group.\n Managing users Users can be created and deleted by navigating to the \u0026ldquo;Users\u0026rdquo; section of the GUI.\nAlternatively users can also be managed using the CLI.\nTo create a user with the CLI, run:\n$ storageos user create jim The above command will create a user named jim. The command will also prompt you to enter a password for the newly created user.\n","excerpt":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces …","ref":"/v2.0/docs/operations/users/","title":"User Management"},{"body":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version [flags] Flags: -h, --help help for version Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version …","ref":"/v2.0/docs/reference/cli/version/","title":"Version"},{"body":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating the volume config via the CLI or UI. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure the volume is not in use. Please note that StorageOS only supports increasing volume size. For more information about how the resize works please see our Resize concepts page.\nResizing a Volume In order to resize a PVC the storage request field must be updated.\napiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: fast accessModes: - ReadWriteOnce resources: requests: storage: 5Gi In order to edit a PVC you can use kubectl edit or kubectl apply to make changes.\nStorageOS supports offline resize of volumes through editing a PVC storage request. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This requires that any pods using a volume must be scaled down for the resize to take place.\n N.B. Resizing a volume without updating the PVC directly will NOT result in the PVC being updated. The methods below are included for completeness, in Kubernetes environments editing the PVC is the preferred method for resizing a volume.\n To resize a volume using the StorageOS CLI use the volume update command\n$ storageos update volume size pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa 10GiB Name: pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa ID: 925e667f-91d3-465a-9391-8fdb56d0c9ff Size: 10 GiB Description: AttachedOn: Replicas: 1x ready Labels: - csi.storage.k8s.io/pv/name pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa - csi.storage.k8s.io/pvc/name pvc-1 - csi.storage.k8s.io/pvc/namespace default - pool default - storageos.com/replicas 1 Volume pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa (925e667f-91d3-465a-9391-8fdb56d0c9ff) updated. Size changed. To resize a volume using the StorageOS UI, navigate to the volumes section and click the edit pencil in order to update the volume config.\n","excerpt":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating …","ref":"/v2.0/docs/operations/resize/","title":"Volume Resize"},{"body":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard POSIX semantics. StorageOS presents volumes as mounts into containers via the Linux LIO subsystem.\nConceptually, StorageOS volumes have a frontend presentation, which is the what the application sees, and a backend presentation, which is the actual on-disk format. Depending on the configuration, frontend and backend components may be on the same or different hosts.\nVolumes are formatted using the linux standard ext4 filesystem by default. Kubernetes users may change the default filesystem type to ext2, ext3, ext4, or xfs by setting the fsType parameter in their StorageClass (See Supported Filesystems for more information). Different filesystems may be supported in the future.\nStorageOS volumes are represented on disk in two parts. Actual volume data is written to blob files in /var/lib/storageos/data/dev[\\d+]. Inside these directories, each StorageOS block device gets two blob files of the form vol.xxxxxx.y.blob, where x is the inode number for the device, and y is an index between 0 and 1. We provide two blob files in order to ensure that certain operations which require locking do not impede in-flight writes to the volume.\nIn systems which have multiple /var/lib/storageos/data/dev[\\d+] directories, two blob files are created per block device. This allows us to load-balance writes across multiple devices. In cases where dev directories are added after a period of run time, later directories are favoured for writes until the data is distributed evenly across the blob files.\nMetadata is kept in directories named /var/lib/storageos/data/db[\\d+]. We maintain an index of all blocks written to the blob file inside the metadata store, including checksums. These checksums allow us to detect bitrot, and return errors on reads, rather than serve bad data. In future versions we may implement recovery from replicas for volumes with one or more replicas defined.\nStorageOS metadata requires approximately 2.7GB of storage per 1TB of allocated blocks in the associated volume. This size is consistent irrespective of data compression defined on the volume.\nTo ensure deterministic performance, individual StorageOS volumes must fit on a single node.\nVolume Resize StorageOS v2.1 supports offline resize of volumes. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure that the volume is not in use.\nThis means that if a Kubernetes pod is currently consuming a volume that a resize request has been issued for, the resize will not be actioned until the pod is terminated and the volume is detached from the node. The StorageOS controlplane will then attach the volume to the node that holds the master deployment and resize the underlying block device and then run resize2fs to expand the filesystem.\nFor a walk through of how to resize a volume please see the Volume Resize operations page.\n","excerpt":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard …","ref":"/v2.0/docs/concepts/volumes/","title":"Volumes"},{"body":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.\nUsing StorageOS persistent volumes with Apache Zookeeper means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Zookeeper pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nAs Zookeeper has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Zookeeper to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Zookeeper on Kubernetes Pre-requisites StorageOS is assumed to have been installed; please check for the latest available version here. Helm To simplify the deployment of Zookeeper, we\u0026rsquo;ve used this Zookeeper helm chart (incubator) (version 1.2.2, app version 3.4.10) and rendered it into the example deployment files you can find in our GitHub repo.\nDeployment Clone the use cases repo You can find the latest files in the StorageOS use cases repository in /zookeeper/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion ---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:zookeeper...spec:replicas:3# \u0026lt;--- number of zookeeper pods...containers:- name:zookeeperimage:\u0026#34;gcr.io/google_samples/k8szk:v3\u0026#34;imagePullPolicy:IfNotPresent...volumeMounts:- name:datamountPath:/var/lib/zookeepervolumeClaimTemplates:- metadata:name:dataspec:accessModes:- \u0026#34;ReadWriteOnce\u0026#34;storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useresources:requests:storage:\u0026#34;5Gi\u0026#34;# \u0026lt;--- storage requested per podThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nCreate the kubernetes objects cd storageos-usecases kubectl apply -f ./zookeeper/ Confirm Zookeeper is up and running $ kubectl get pods NAME READY STATUS RESTARTS AGE zookeeper-0 1/1 Running 0 2m30s zookeeper-1 1/1 Running 0 112s zookeeper-2 1/1 Running 0 56s zookeeper-test-client 1/1 Running 0 2m30s Connect to Zookeeper Connect to the zookeeper client pod and list existing topics using the service endpoint\nkubectl exec -it zookeeper-test-client /bin/bash and issue a command to the zookeeper service\nzkCli.sh -server zookeeper ls /zookeeper ","excerpt":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration …","ref":"/v2.0/docs/usecases/zookeeper/","title":"Zookeeper"}]
\ No newline at end of file
diff --git a/v2.0/search/index.html b/v2.0/search/index.html
index d9c49af71..5f9c0eabe 100644
--- a/v2.0/search/index.html
+++ b/v2.0/search/index.html
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.0/offline-search-index.c660a2f25ef300726aaa7cc3069cdd72c6d31a298862843fdc787ba4da20596098529428839bbe146ff3cf76b2747adb7ea9e2825d15ec121ef2b1e2606a0a09.json"
+ data-offline-search-index-json-src="/v2.0/offline-search-index.64487cf0025f09c286b27646dbe89286279cc5177c072286d97aaeb8daaaeef3dc3539cfba23bf2f77af021bcfa0be6ce91288df895c88109e34ee811cfdd310.json"
data-offline-search-base-href="/"
>
diff --git a/v2.0/yaml/bundle-configuration.yaml b/v2.0/yaml/bundle-configuration.yaml
index f1c1004a0..5b21e3ed3 100644
--- a/v2.0/yaml/bundle-configuration.yaml
+++ b/v2.0/yaml/bundle-configuration.yaml
@@ -40,11 +40,18 @@ spec:
# InternalAPIPort = 5710 # Grpc API
# GossipPort = 5711 # Gossip+Healthcheck
- echo \"Source node for the test:\";
+ date;
+ echo \"Source node:\";
hostname -f -I; echo;
+ echo \"TCP:\";
parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
- ::: 5703 5704 5705 5710 5711
+ ::: 5703 5704 5705 5710 5711;
+ echo;
+
+ echo \"UDP:\";
+ parallel -j2 nc -vunz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
+ ::: 5711;
"
timeout: 90s
- exec:
diff --git a/v2.1/404.html b/v2.1/404.html
index 5c800aad5..3d46e8c7d 100644
--- a/v2.1/404.html
+++ b/v2.1/404.html
@@ -181,7 +181,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/best-practices/index.html b/v2.1/docs/best-practices/index.html
index ac998b30b..be605e0a6 100644
--- a/v2.1/docs/best-practices/index.html
+++ b/v2.1/docs/best-practices/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/architecture/index.html b/v2.1/docs/concepts/architecture/index.html
index e7ce61006..fef7605b9 100644
--- a/v2.1/docs/concepts/architecture/index.html
+++ b/v2.1/docs/concepts/architecture/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/clusters/index.html b/v2.1/docs/concepts/clusters/index.html
index 295c8ed48..9c58c3341 100644
--- a/v2.1/docs/concepts/clusters/index.html
+++ b/v2.1/docs/concepts/clusters/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/compression/index.html b/v2.1/docs/concepts/compression/index.html
index 37d3cd374..f4643b944 100644
--- a/v2.1/docs/concepts/compression/index.html
+++ b/v2.1/docs/concepts/compression/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/etcd/index.html b/v2.1/docs/concepts/etcd/index.html
index 3d1ae9d85..b14fabc88 100644
--- a/v2.1/docs/concepts/etcd/index.html
+++ b/v2.1/docs/concepts/etcd/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/index.html b/v2.1/docs/concepts/index.html
index 440fcb928..7a9982186 100644
--- a/v2.1/docs/concepts/index.html
+++ b/v2.1/docs/concepts/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/namespaces/index.html b/v2.1/docs/concepts/namespaces/index.html
index a3804279a..f3e64752a 100644
--- a/v2.1/docs/concepts/namespaces/index.html
+++ b/v2.1/docs/concepts/namespaces/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/nodes/index.html b/v2.1/docs/concepts/nodes/index.html
index d2befa7a8..832318b4d 100644
--- a/v2.1/docs/concepts/nodes/index.html
+++ b/v2.1/docs/concepts/nodes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/policies/index.html b/v2.1/docs/concepts/policies/index.html
index 02a975f09..ec2e8a24d 100644
--- a/v2.1/docs/concepts/policies/index.html
+++ b/v2.1/docs/concepts/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/replication/index.html b/v2.1/docs/concepts/replication/index.html
index acf89498c..88bc2072a 100644
--- a/v2.1/docs/concepts/replication/index.html
+++ b/v2.1/docs/concepts/replication/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/concepts/volumes/index.html b/v2.1/docs/concepts/volumes/index.html
index 4f28c3fc8..634fe4a6a 100644
--- a/v2.1/docs/concepts/volumes/index.html
+++ b/v2.1/docs/concepts/volumes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/index.html b/v2.1/docs/index.html
index 95a812526..b0f012a70 100644
--- a/v2.1/docs/index.html
+++ b/v2.1/docs/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/index.xml b/v2.1/docs/index.xml
index cd7598755..739489154 100644
--- a/v2.1/docs/index.xml
+++ b/v2.1/docs/index.xml
@@ -1878,6 +1878,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -1910,7 +1915,7 @@ the init container, perform the following steps:</p>
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -2023,6 +2028,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2.1/docs/install/index.html b/v2.1/docs/install/index.html
index dcf683993..1030ded0c 100644
--- a/v2.1/docs/install/index.html
+++ b/v2.1/docs/install/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/install/kubernetes/index.html b/v2.1/docs/install/kubernetes/index.html
index 44bfc2fde..8d94f4544 100644
--- a/v2.1/docs/install/kubernetes/index.html
+++ b/v2.1/docs/install/kubernetes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/install/openshift/index.html b/v2.1/docs/install/openshift/index.html
index 98a9aedaf..3f934af56 100644
--- a/v2.1/docs/install/openshift/index.html
+++ b/v2.1/docs/install/openshift/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/install/rancher/index.html b/v2.1/docs/install/rancher/index.html
index 6118de2ee..80e897ccd 100644
--- a/v2.1/docs/install/rancher/index.html
+++ b/v2.1/docs/install/rancher/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/introduction/index.html b/v2.1/docs/introduction/index.html
index 01329a774..898c38bd3 100644
--- a/v2.1/docs/introduction/index.html
+++ b/v2.1/docs/introduction/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/introduction/overview/index.html b/v2.1/docs/introduction/overview/index.html
index 22280d1a0..7670441c9 100644
--- a/v2.1/docs/introduction/overview/index.html
+++ b/v2.1/docs/introduction/overview/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/introduction/platforms/index.html b/v2.1/docs/introduction/platforms/index.html
index 7ec8d8186..4b1f0d4ed 100644
--- a/v2.1/docs/introduction/platforms/index.html
+++ b/v2.1/docs/introduction/platforms/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/introduction/quickstart/index.html b/v2.1/docs/introduction/quickstart/index.html
index 8acaf124f..022204011 100644
--- a/v2.1/docs/introduction/quickstart/index.html
+++ b/v2.1/docs/introduction/quickstart/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/etcd/index.html b/v2.1/docs/operations/etcd/index.html
index 320292b57..aa8337dd2 100644
--- a/v2.1/docs/operations/etcd/index.html
+++ b/v2.1/docs/operations/etcd/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/etcd/storageos-secret-info/index.html b/v2.1/docs/operations/etcd/storageos-secret-info/index.html
index 571a3d88f..404a4e93f 100644
--- a/v2.1/docs/operations/etcd/storageos-secret-info/index.html
+++ b/v2.1/docs/operations/etcd/storageos-secret-info/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/firstpvc/index.html b/v2.1/docs/operations/firstpvc/index.html
index b02522f67..81460b8d0 100644
--- a/v2.1/docs/operations/firstpvc/index.html
+++ b/v2.1/docs/operations/firstpvc/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/health/index.html b/v2.1/docs/operations/health/index.html
index add32a85c..d49bb3e49 100644
--- a/v2.1/docs/operations/health/index.html
+++ b/v2.1/docs/operations/health/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/index.html b/v2.1/docs/operations/index.html
index e44dc5bb8..f48078af6 100644
--- a/v2.1/docs/operations/index.html
+++ b/v2.1/docs/operations/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/licensing/index.html b/v2.1/docs/operations/licensing/index.html
index 2de385477..1e5172f2e 100644
--- a/v2.1/docs/operations/licensing/index.html
+++ b/v2.1/docs/operations/licensing/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/managing-host-storage/index.html b/v2.1/docs/operations/managing-host-storage/index.html
index 9f41b967d..b703e5250 100644
--- a/v2.1/docs/operations/managing-host-storage/index.html
+++ b/v2.1/docs/operations/managing-host-storage/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/namespaces/index.html b/v2.1/docs/operations/namespaces/index.html
index bc60f4073..709c0bd79 100644
--- a/v2.1/docs/operations/namespaces/index.html
+++ b/v2.1/docs/operations/namespaces/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/policies/index.html b/v2.1/docs/operations/policies/index.html
index 56f1b974e..d62597527 100644
--- a/v2.1/docs/operations/policies/index.html
+++ b/v2.1/docs/operations/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/resize/index.html b/v2.1/docs/operations/resize/index.html
index 5988bc5bf..5ccaa22e8 100644
--- a/v2.1/docs/operations/resize/index.html
+++ b/v2.1/docs/operations/resize/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/storageclasses/index.html b/v2.1/docs/operations/storageclasses/index.html
index 1e6c1a218..4c3e4d2db 100644
--- a/v2.1/docs/operations/storageclasses/index.html
+++ b/v2.1/docs/operations/storageclasses/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/troubleshooting/index.html b/v2.1/docs/operations/troubleshooting/index.html
index 5411241f6..07cdaff30 100644
--- a/v2.1/docs/operations/troubleshooting/index.html
+++ b/v2.1/docs/operations/troubleshooting/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/uninstall/index.html b/v2.1/docs/operations/uninstall/index.html
index cee8de5c5..dd40fc59b 100644
--- a/v2.1/docs/operations/uninstall/index.html
+++ b/v2.1/docs/operations/uninstall/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/operations/users/index.html b/v2.1/docs/operations/users/index.html
index 4691e0929..6dfcd0241 100644
--- a/v2.1/docs/operations/users/index.html
+++ b/v2.1/docs/operations/users/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/platforms/aws-eks/index.html b/v2.1/docs/platforms/aws-eks/index.html
index b885d810a..b1b1b5e7f 100644
--- a/v2.1/docs/platforms/aws-eks/index.html
+++ b/v2.1/docs/platforms/aws-eks/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/platforms/azure-aks/index.html b/v2.1/docs/platforms/azure-aks/index.html
index ffcef455a..1baa58369 100644
--- a/v2.1/docs/platforms/azure-aks/index.html
+++ b/v2.1/docs/platforms/azure-aks/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/platforms/dockeree/index.html b/v2.1/docs/platforms/dockeree/index.html
index 5e6e6381c..069728a8f 100644
--- a/v2.1/docs/platforms/dockeree/index.html
+++ b/v2.1/docs/platforms/dockeree/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/platforms/index.html b/v2.1/docs/platforms/index.html
index ffa7f1b36..eb6dd97ef 100644
--- a/v2.1/docs/platforms/index.html
+++ b/v2.1/docs/platforms/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/platforms/openshift/index.html b/v2.1/docs/platforms/openshift/index.html
index c05287a03..18f97c18b 100644
--- a/v2.1/docs/platforms/openshift/index.html
+++ b/v2.1/docs/platforms/openshift/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/platforms/rancher/index.html b/v2.1/docs/platforms/rancher/index.html
index 443f1ccdb..65b516e2b 100644
--- a/v2.1/docs/platforms/rancher/index.html
+++ b/v2.1/docs/platforms/rancher/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/prerequisites/etcd/index.html b/v2.1/docs/prerequisites/etcd/index.html
index 4b8bea920..d87cadc2b 100644
--- a/v2.1/docs/prerequisites/etcd/index.html
+++ b/v2.1/docs/prerequisites/etcd/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/prerequisites/firewalls/index.html b/v2.1/docs/prerequisites/firewalls/index.html
index e0bb52e20..bdcb40804 100644
--- a/v2.1/docs/prerequisites/firewalls/index.html
+++ b/v2.1/docs/prerequisites/firewalls/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/prerequisites/index.html b/v2.1/docs/prerequisites/index.html
index bd01c6c62..43c60936d 100644
--- a/v2.1/docs/prerequisites/index.html
+++ b/v2.1/docs/prerequisites/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/prerequisites/mountpropagation/index.html b/v2.1/docs/prerequisites/mountpropagation/index.html
index 9b8841caf..5dc391feb 100644
--- a/v2.1/docs/prerequisites/mountpropagation/index.html
+++ b/v2.1/docs/prerequisites/mountpropagation/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/prerequisites/systemconfiguration/index.html b/v2.1/docs/prerequisites/systemconfiguration/index.html
index 743129e61..7d80777e8 100644
--- a/v2.1/docs/prerequisites/systemconfiguration/index.html
+++ b/v2.1/docs/prerequisites/systemconfiguration/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/api/index.html b/v2.1/docs/reference/api/index.html
index 0260fa06d..da418b3b9 100644
--- a/v2.1/docs/reference/api/index.html
+++ b/v2.1/docs/reference/api/index.html
@@ -189,7 +189,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/bundle/index.html b/v2.1/docs/reference/bundle/index.html
index 6b6595407..0107062b5 100644
--- a/v2.1/docs/reference/bundle/index.html
+++ b/v2.1/docs/reference/bundle/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/apply/index.html b/v2.1/docs/reference/cli/apply/index.html
index ba0a5786e..ed7feaed0 100644
--- a/v2.1/docs/reference/cli/apply/index.html
+++ b/v2.1/docs/reference/cli/apply/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/attach/index.html b/v2.1/docs/reference/cli/attach/index.html
index 8d5a88d00..9c392bc2e 100644
--- a/v2.1/docs/reference/cli/attach/index.html
+++ b/v2.1/docs/reference/cli/attach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/create/index.html b/v2.1/docs/reference/cli/create/index.html
index 7518dab14..51056e71e 100644
--- a/v2.1/docs/reference/cli/create/index.html
+++ b/v2.1/docs/reference/cli/create/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/delete/index.html b/v2.1/docs/reference/cli/delete/index.html
index cd144d9fb..1e30edf9a 100644
--- a/v2.1/docs/reference/cli/delete/index.html
+++ b/v2.1/docs/reference/cli/delete/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/describe/index.html b/v2.1/docs/reference/cli/describe/index.html
index ca0060f19..897f298bb 100644
--- a/v2.1/docs/reference/cli/describe/index.html
+++ b/v2.1/docs/reference/cli/describe/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/detach/index.html b/v2.1/docs/reference/cli/detach/index.html
index 8048f5e1f..66b56ae6d 100644
--- a/v2.1/docs/reference/cli/detach/index.html
+++ b/v2.1/docs/reference/cli/detach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/get/index.html b/v2.1/docs/reference/cli/get/index.html
index 317e4b744..24e586733 100644
--- a/v2.1/docs/reference/cli/get/index.html
+++ b/v2.1/docs/reference/cli/get/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/help/index.html b/v2.1/docs/reference/cli/help/index.html
index 66b5b4ff8..58a41682f 100644
--- a/v2.1/docs/reference/cli/help/index.html
+++ b/v2.1/docs/reference/cli/help/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/index.html b/v2.1/docs/reference/cli/index.html
index 3e7f045b1..22c8fb956 100644
--- a/v2.1/docs/reference/cli/index.html
+++ b/v2.1/docs/reference/cli/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cli/version/index.html b/v2.1/docs/reference/cli/version/index.html
index 99c7dfd18..9e5e24d57 100644
--- a/v2.1/docs/reference/cli/version/index.html
+++ b/v2.1/docs/reference/cli/version/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cluster-operator/configuration/index.html b/v2.1/docs/reference/cluster-operator/configuration/index.html
index 38e94ca66..d79640a4b 100644
--- a/v2.1/docs/reference/cluster-operator/configuration/index.html
+++ b/v2.1/docs/reference/cluster-operator/configuration/index.html
@@ -30,7 +30,7 @@
-
+
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -1127,6 +1127,11 @@ StorageOSCluster Resource Confi
false |
+images.apiManagerContainer |
+StorageOS API Manager container image |
+storageos/api-manager:v1.0.0 |
+
+
images.csiClusterDriverRegistrarContainer |
CSI Cluster Driver Registrar Container image |
quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 |
@@ -1159,7 +1164,7 @@ StorageOSCluster Resource Confi
images.initContainer |
StorageOS init container image |
-storageos/init:1.0.0 |
+storageos/init:2.1.0 |
images.kubeSchedulerContainer |
@@ -1272,6 +1277,11 @@ StorageOSCluster Resource Confi
/var/lib/kubelet/plugins/kubernetes.io~storageos |
+storageClassName |
+The name of the default StorageClass created for StorageOS volumes |
+fast |
+
+
tlsEtcdSecretRefName |
Secret containing etcd client certificates |
|
diff --git a/v2.1/docs/reference/cluster-operator/examples/index.html b/v2.1/docs/reference/cluster-operator/examples/index.html
index 50b84695d..60cb72eb8 100644
--- a/v2.1/docs/reference/cluster-operator/examples/index.html
+++ b/v2.1/docs/reference/cluster-operator/examples/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cluster-operator/index.html b/v2.1/docs/reference/cluster-operator/index.html
index 8abd91f6a..b2ffd60c7 100644
--- a/v2.1/docs/reference/cluster-operator/index.html
+++ b/v2.1/docs/reference/cluster-operator/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cluster-operator/index.xml b/v2.1/docs/reference/cluster-operator/index.xml
index f502ba21c..b68336c81 100644
--- a/v2.1/docs/reference/cluster-operator/index.xml
+++ b/v2.1/docs/reference/cluster-operator/index.xml
@@ -108,6 +108,11 @@
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -140,7 +145,7 @@
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -253,6 +258,11 @@
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2.1/docs/reference/cluster-operator/install/index.html b/v2.1/docs/reference/cluster-operator/install/index.html
index 4cdefead0..7eb15ada7 100644
--- a/v2.1/docs/reference/cluster-operator/install/index.html
+++ b/v2.1/docs/reference/cluster-operator/install/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/cluster-operator/upgrade/index.html b/v2.1/docs/reference/cluster-operator/upgrade/index.html
index 7658fac0c..6cfb17c69 100644
--- a/v2.1/docs/reference/cluster-operator/upgrade/index.html
+++ b/v2.1/docs/reference/cluster-operator/upgrade/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/contributing/index.html b/v2.1/docs/reference/contributing/index.html
index 035530f07..32fa08dfb 100644
--- a/v2.1/docs/reference/contributing/index.html
+++ b/v2.1/docs/reference/contributing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/filesystems/index.html b/v2.1/docs/reference/filesystems/index.html
index 9ab23f93b..33052ccb8 100644
--- a/v2.1/docs/reference/filesystems/index.html
+++ b/v2.1/docs/reference/filesystems/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/gui/index.html b/v2.1/docs/reference/gui/index.html
index 5e7d1175c..5ea94ebad 100644
--- a/v2.1/docs/reference/gui/index.html
+++ b/v2.1/docs/reference/gui/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/index.html b/v2.1/docs/reference/index.html
index 8e2e975e3..277394ffb 100644
--- a/v2.1/docs/reference/index.html
+++ b/v2.1/docs/reference/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/init-container/index.html b/v2.1/docs/reference/init-container/index.html
index f6e749e7b..fd333c81a 100644
--- a/v2.1/docs/reference/init-container/index.html
+++ b/v2.1/docs/reference/init-container/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/labels/index.html b/v2.1/docs/reference/labels/index.html
index 1afce2434..0be461a35 100644
--- a/v2.1/docs/reference/labels/index.html
+++ b/v2.1/docs/reference/labels/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/licence/index.html b/v2.1/docs/reference/licence/index.html
index a663f16cf..791979716 100644
--- a/v2.1/docs/reference/licence/index.html
+++ b/v2.1/docs/reference/licence/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/open_source_attribution/index.html b/v2.1/docs/reference/open_source_attribution/index.html
index c76d55b25..e4a0d6b78 100644
--- a/v2.1/docs/reference/open_source_attribution/index.html
+++ b/v2.1/docs/reference/open_source_attribution/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/release_notes/index.html b/v2.1/docs/reference/release_notes/index.html
index 299dce129..13c2d2dd2 100644
--- a/v2.1/docs/reference/release_notes/index.html
+++ b/v2.1/docs/reference/release_notes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/resource_requests_and_limits/index.html b/v2.1/docs/reference/resource_requests_and_limits/index.html
index 6bd61ff07..7dd429406 100644
--- a/v2.1/docs/reference/resource_requests_and_limits/index.html
+++ b/v2.1/docs/reference/resource_requests_and_limits/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/telemetry/index.html b/v2.1/docs/reference/telemetry/index.html
index 2b107077f..c8a764fed 100644
--- a/v2.1/docs/reference/telemetry/index.html
+++ b/v2.1/docs/reference/telemetry/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/reference/tolerations/index.html b/v2.1/docs/reference/tolerations/index.html
index 9b986287c..66f4a48ae 100644
--- a/v2.1/docs/reference/tolerations/index.html
+++ b/v2.1/docs/reference/tolerations/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/self-eval/index.html b/v2.1/docs/self-eval/index.html
index 83c573e45..43b3a7110 100644
--- a/v2.1/docs/self-eval/index.html
+++ b/v2.1/docs/self-eval/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/support/index.html b/v2.1/docs/support/index.html
index 34cb73b9c..cbb550abf 100644
--- a/v2.1/docs/support/index.html
+++ b/v2.1/docs/support/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/backups/index.html b/v2.1/docs/usecases/backups/index.html
index 1a8fd6226..db7cf7c2a 100644
--- a/v2.1/docs/usecases/backups/index.html
+++ b/v2.1/docs/usecases/backups/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/cassandra/index.html b/v2.1/docs/usecases/cassandra/index.html
index 3932ef5b8..28ada4b3a 100644
--- a/v2.1/docs/usecases/cassandra/index.html
+++ b/v2.1/docs/usecases/cassandra/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/elasticsearch/index.html b/v2.1/docs/usecases/elasticsearch/index.html
index 56942bdce..cad7c1358 100644
--- a/v2.1/docs/usecases/elasticsearch/index.html
+++ b/v2.1/docs/usecases/elasticsearch/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/index.html b/v2.1/docs/usecases/index.html
index 4df15e85d..c355c8dda 100644
--- a/v2.1/docs/usecases/index.html
+++ b/v2.1/docs/usecases/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/jenkins/index.html b/v2.1/docs/usecases/jenkins/index.html
index b82719e87..4f55b7450 100644
--- a/v2.1/docs/usecases/jenkins/index.html
+++ b/v2.1/docs/usecases/jenkins/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/kafka/index.html b/v2.1/docs/usecases/kafka/index.html
index 4b5329e0d..2f509d294 100644
--- a/v2.1/docs/usecases/kafka/index.html
+++ b/v2.1/docs/usecases/kafka/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/kubevirt/index.html b/v2.1/docs/usecases/kubevirt/index.html
index 18322b37c..d18a7e1c9 100644
--- a/v2.1/docs/usecases/kubevirt/index.html
+++ b/v2.1/docs/usecases/kubevirt/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/mssql/index.html b/v2.1/docs/usecases/mssql/index.html
index 5d739fbee..4f4147929 100644
--- a/v2.1/docs/usecases/mssql/index.html
+++ b/v2.1/docs/usecases/mssql/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/mysql/index.html b/v2.1/docs/usecases/mysql/index.html
index 4ea5f1fa2..0ad79b7bb 100644
--- a/v2.1/docs/usecases/mysql/index.html
+++ b/v2.1/docs/usecases/mysql/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/nginx/index.html b/v2.1/docs/usecases/nginx/index.html
index 0c564d3f8..5773619f9 100644
--- a/v2.1/docs/usecases/nginx/index.html
+++ b/v2.1/docs/usecases/nginx/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/postgres/index.html b/v2.1/docs/usecases/postgres/index.html
index 4f167e4ee..65bccd666 100644
--- a/v2.1/docs/usecases/postgres/index.html
+++ b/v2.1/docs/usecases/postgres/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/prometheus/index.html b/v2.1/docs/usecases/prometheus/index.html
index 8164c9152..9843c6f52 100644
--- a/v2.1/docs/usecases/prometheus/index.html
+++ b/v2.1/docs/usecases/prometheus/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/redis/index.html b/v2.1/docs/usecases/redis/index.html
index 66aa40268..fc4b1a791 100644
--- a/v2.1/docs/usecases/redis/index.html
+++ b/v2.1/docs/usecases/redis/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/docs/usecases/zookeeper/index.html b/v2.1/docs/usecases/zookeeper/index.html
index 3d7415a1b..46b5c65b8 100644
--- a/v2.1/docs/usecases/zookeeper/index.html
+++ b/v2.1/docs/usecases/zookeeper/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/index.html b/v2.1/index.html
index e9a259fb9..db3df374d 100644
--- a/v2.1/index.html
+++ b/v2.1/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/index.json b/v2.1/index.json
index d1608975d..667a531cb 100644
--- a/v2.1/index.json
+++ b/v2.1/index.json
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json b/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json
new file mode 100644
index 000000000..bcd0df194
--- /dev/null
+++ b/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json
@@ -0,0 +1 @@
+[{"body":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher OpenShift ","excerpt":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher …","ref":"/v2.1/docs/reference/cluster-operator/install/","title":"Install"},{"body":"Current Version The current StorageOS version is v2.1.0\nSections Our documentation is arranged into sections, accessible from the navigation bar on the left.\nIntroduction - Quickstart and support information.\nConcepts - Architectural and deep technical information.\nPrerequisites - We require certain prerequisites to be met for the product to function correctly. Please read these carefully and ensure they are implemented.\nPlatforms - Due to differences in the various orchestrators that StorageOS can run under, we list install guides and other platform specific operations here.\nOperations - Platform agnostic operations.\nUse Cases - A set of examples to get up and running with StorageOS quickly.\nReference - Information on our GUI, CLI, and other important information.\n","excerpt":"Current Version The current StorageOS version is v2.1.0\nSections Our documentation is arranged into …","ref":"/v2.1/docs/introduction/","title":"Introduction"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed service such as EKS, AKS, GKE, DO or DockerEE platform can use the following Kubernetes guide to install StorageOS.\n StorageOS v2 uses CSI only. The native driver has been deprecated.\n Make sure to add a StorageOS licence after installing.\n 1.18 1.17 1.16 1.15 1.14 1.13 Install StorageOS on Kubernetes 1.18 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.17 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.16 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.15 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.14 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.13 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed …","ref":"/v2.1/docs/install/kubernetes/","title":"Kubernetes"},{"body":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some significant enhancements over our v1 product. We\u0026rsquo;ve built V2 based on our observations of trends in the industry, as well as our own experience.\nMany of our customers want to run big clusters - in the tens or hundreds of nodes. In these sorts of big environments, the challenges multiply. Not only do we need to scale well, but we also need to be more failure tolerant. Bigger environments typically suffer higher failure rates (more nodes = greater chance of something failing), but are also subject to all sorts of transient conditions such as network partitions.\nThe second trend we\u0026rsquo;ve seen become increasingly common is the desire to run multiple clusters, and consume storage between them in some way - sometimes to implement novel topologies such as a centralised storage cluster with satellites consuming the storage, and sometimes to replicate data between those clusters for HA or DR purposes.\nWe\u0026rsquo;ve built V2 with these architectures and design patterns in mind. Not only does it scale well, but it contains the foundations we need to implement a rich set of multi-cluster functionality.\nUpgraded Control Plane At the heart of the V2 release is an upgraded control plane. We\u0026rsquo;ve changed a lot here. Firstly, our usage of etcd is vastly improved. We\u0026rsquo;ve learnt a lot about the subtleties of distributed consensus in the last year, particularly in noisy or unpredictable environments. Not only is StorageOS V2 much lighter on your etcd cluster, but it\u0026rsquo;s a lot more tolerant of transient failure conditions that are often found in cloud environments, or clusters under heavy load.\nWe spent some time describing and testing our internal state machine using the TLA+ formal verification language. This allows us to have a much higher degree of confidence that our algorithms will behave correctly, particularly under hard-to-test edge cases and failure conditions.\nAdditionally, we\u0026rsquo;ve changed the way volumes behave with respect to centralised scheduling. Each volume group (consisting of a master and 0 or more replicas) now behaves as a strongly consistent unit allowing it to take action independent of the activities of the rest of the cluster. Other state can be distributed around the cluster using eventually consistent mechanisms. This approach inherently scales better and allows StorageOS V2 to effectively manage many more nodes and volumes than before.\nWe\u0026rsquo;ve implemented TLS on all endpoints. Not only does this give you encrypted traffic between nodes in your storage cluster, it also protects all endpoints with strong, public key based authentication. Today\u0026rsquo;s IT environments can\u0026rsquo;t rely on firewalls to keep bad actors out - they must implement security at all layers within the stack - defense in depth. While we recognise that this brings a welcome relief to many security conscious administrators, we also know that managing certificate authorities (CAs) can be an unwelcome source of complexity. For this reason, StorageOS V2 implements an internal CA by default, to manage this complexity for you. If you\u0026rsquo;d prefer to integrate your own CA, we support that too - it\u0026rsquo;s up to you.\nFinally - our logging has undergone a complete transformation in this edition. We know that systems engineers and operators don\u0026rsquo;t just value headline features, but that observability and diagnostics are equally important. All logs are now decorated with rich context to help you understand what is happening within your cluster, and we\u0026rsquo;ll output in json by default, for easy ingestion into log aggregators such as Elasticsearch.\nUpgraded Data Plane Not to be outdone, our data plane contains some significant improvements.\nFirstly, we\u0026rsquo;ve completely re-written our sync algorithm (see Delta Sync, used when seeding or catching up replicas that have been offline or partitioned. Our new algorithm uses a Hash List to sync only changed sections of a volume (similar in some ways to what rsync does). StorageOS maintains these hashes during normal operation, meaning that when resyncing a failed replica, for example after a node reboot, we can very quickly and efficiently catch this replica up, rather than needing to promote and build a new one from scratch. This improves resiliency within your cluster, and prevents using excessive network bandwidth during failover conditions - at a time when it might be needed the most.\nSecondly, a new threading model, with dynamic pool sizing, means that StorageOS is faster, a lot faster. In our tests we observed improvements across the board, with improvements in throughput of up to 135% for some scenarios.\n","excerpt":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some …","ref":"/v2.1/docs/introduction/overview/","title":"What is StorageOS V2"},{"body":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source implementation of the SCSI target.\nWe require the following modules to be loaded:\n target_core_mod tcm_loop target_core_file configfs target_core_user uio N.B. Other applications utilising TCMU cannot be run concurrently with StorageOS. Doing so may result in corruption of data. On startup, StorageOS will detect if other applications are using TCMU and fall back to FUSE. However if StorageOS is started first there is no mechanism for StorageOS to fallback to FUSE if another application begins to use TCMU. TCMU can be disabled using the DISABLE_TCMU StorageOSCluster spec parameter.\n Depending on the distribution, the modules are shipped as part of the base kernel package or as part of a kernel extras package which needs to be installed.\nDistribution Specifics The following distributions are supported by default:\n RHEL 7.5 CentOS 7 Debian 9 Ubuntu Azure RancherOS - Note CSI is not supported on RancherOS Ubuntu 16.04/18.04 requires the installation of additional packages.\n N.B. Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not provide the necessary linux-image-extra package - see below for more information\n Ubuntu Package Installation Ubuntu 16.04/18.04 Generic and Ubuntu 16.04 GCE require extra packages:\nUbuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Ubuntu With AWS Or GCE Kernels Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not yet provide the linux-image-extra package. As such you should either use Debian, CentOS or RHEL, or install the non-cloud-provider optimised Ubuntu kernel.\nInstalling the non-cloud-provider optimised Ubuntu kernel is something that should only be done with full understanding of potential ramifications.\nsudo apt -y update sudo apt install -y linux-virtual linux-image-extra-virtual sudo apt purge -y linux*aws # Reboot the machine sudo shutdown -r now Automatic Configuration Once required kernel modules are installed on the system, for convenience we provide a container which will ensure the appropriate modules are loaded and ready for use at runtime. On Docker installations, you will need to run the init container prior to starting StorageOS. Our installation guides for Kubernetes and OpenShift include this step.\n# Load the required kernel modules. The Kubernetes and OpenShift installations include this step. docker run --name enable_lio \\ --privileged \\ --rm \\ --cap-add=SYS_ADMIN \\ -v /lib/modules:/lib/modules \\ -v /sys:/sys:rshared \\ storageos/init:0.2 Manual Configuration For those wishing to manage their own kernel configuration, rather than using the init container, perform the following steps:\n Ensure kernel modules are all loaded per list above Ensure configfs is loaded and mounted at /sys/kernel/config ","excerpt":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source …","ref":"/v2.1/docs/prerequisites/systemconfiguration/","title":"System Configuration"},{"body":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters of the StorageOSCluster custom resource and their default values.\n Parameter Description Default csi.deploymentStrategy CSI helper deployment strategy (statefulset or deployment) statefulset csi.enable Enable CSI setup false csi.enableControllerPublishCreds Enable CSI controller publish credentials false csi.enableNodePublishCreds Enable CSI node publish credentials false csi.enableProvisionCreds Enable CSI provision credentials false debug Enable debug mode for all the cluster nodes false disableFencing Disable Pod fencing false disableScheduler Disable StorageOS scheduler false disableTCMU Disable TCMU to allow co-existence with other TCMU users. Disabling TCMU degrades performance false disableTelemetry Disable telemetry reports false forceTCMU Forces TCMU to be enabled or causes StorageOS to abort startup false images.apiManagerContainer StorageOS API Manager container image storageos/api-manager:v1.0.0 images.csiClusterDriverRegistrarContainer CSI Cluster Driver Registrar Container image quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 images.csiExternalAttacherContainer CSI External Attacher Container image quay.io/k8scsi/csi-attacher:v1.0.1 images.csiExternalProvisionerContainer CSI External Provisioner Container image storageos/csi-provisioner:v1.0.1 ìmages.csiLivenessProbeContainer CSI Liveness Probe Container Image quay.io/k8scsi/livenessprobe:v1.0.1 images.csiNodeDriverRegistrarContainer CSI Node Driver Registrar Container image quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 images.hyperkubeContainer Deprecated field - HyperKube Container image Default dependent on Scheduler version images.initContainer StorageOS init container image storageos/init:2.1.0 images.kubeSchedulerContainer Kube scheduler container image Default dependent on Scheduler version images.nfsContainer StorageOS nfs container image storageos/nfs:1.0.0 images.nodeContainer StorageOS node container image storageos/node:v2.1.0 ingress.annotations Annotations of the ingress used by the cluster ingress.enable Enable ingress for the cluster false ingress.hostname Hostname to be used in cluster ingress storageos.local ingress.tls Enable TLS for the ingress false k8sDistro The name of the Kubernetes distribution is use, e.g. rancher or eks kvBackend.address Comma-separated list of addresses of external key-value store. (1.2.3.4:2379,2.3.4.5:2379) kvBackend.backend (v2 deprecated) Name of the key-value store to use. Set to etcd for external key-value store. embedded namespace Namespace where storageos cluster resources are created kube-system nodeSelectorTerms Set node selector for storageos pod placement pause Pause the operator for cluster maintenance false resources Set resource requirements for the containers secretRefName Reference name of storageos secret secretRefNamespace Namespace of storageos secret service.annotations Annotations of the Service used by the cluster service.externalPort External port of the Service used by the cluster 5705 service.internalPort Internal port of the Service used by the cluster 5705 service.name Name of the Service used by the cluster storageos service.type Type of the Service used by the cluster ClusterIP sharedDir Path to be shared with kubelet container when deployed as a pod /var/lib/kubelet/plugins/kubernetes.io~storageos storageClassName The name of the default StorageClass created for StorageOS volumes fast tlsEtcdSecretRefName Secret containing etcd client certificates tlsEtcdSecretRefNamespace Namespace of the tlsEtcdSecretRefName tolerations Set pod tolerations for storageos pod placement ","excerpt":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters …","ref":"/v2.1/docs/reference/cluster-operator/configuration/","title":"Configuration"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment of an etcd cluster.\n If you have installed OpenShift in AWS ensure that the requisite ports are opened for the worker nodes\u0026rsquo; security group.\n Make sure to add a StorageOS licence after installing.\n StorageOS v2 supports OpenShift v4. For more information check the OpenShift platform page.\nManual install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\nFirst StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment …","ref":"/v2.1/docs/install/openshift/","title":"OpenShift"},{"body":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all nodes in the cluster:\n Port Number TCP/UDP Use 5701 TCP gRPC 5703 TCP DirectFS 5704 TCP Dataplane Supervisor 5705 TCP REST API 5711 TCP \u0026amp; UDP Gossip service StorageOS also uses ephemeral ports to dial-out to these ports on other StorageOS nodes. For this reason, outgoing traffic should be enabled.\nFirewalls and VPS providers Some VPS providers (such as Digital Ocean) ship default firewall rulesets which must be updated to allow StorageOS to run. Some example rules are shown below - modify to taste.\nUFW For distributions using UFW, such as RHEL and derivatives:\nufw default allow outgoing ufw allow 5701:5711/tcp ufw allow 5711/udp Firewalld For distributions that enable firewalld to control iptables such as some installations of OpenShift.\nfirewall-cmd --permanent --new-service=storageos firewall-cmd --permanent --service=storageos --add-port=5700-5800/tcp firewall-cmd --add-service=storageos --zone=public --permanent firewall-cmd --reload Iptables For those using plain iptables:\n# Inbound traffic iptables -I INPUT -i lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I INPUT -m state --state ESTABLISHED,RELATED -m comment --comment \u0026#39;Permit established traffic\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 5701:5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p udp --dport 5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT # Outbound traffic iptables -I OUTPUT -o lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I OUTPUT -d 0.0.0.0/0 -m comment --comment \u0026#39;Permit outbound traffic\u0026#39; -j ACCEPT Please ensure that the iptables rules you have added above come before any default DROP or REJECT rules.\n","excerpt":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all …","ref":"/v2.1/docs/prerequisites/firewalls/","title":"Firewalls"},{"body":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, OpenShift or Rancher.\n Dive into our Use Case documentation for Workloads deployed in Kubernetes.\n ","excerpt":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, …","ref":"/v2.1/docs/introduction/quickstart/","title":"Quickstart"},{"body":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and Password in base64 encoding.\nkubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;default\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z END This example contains a default password, for production installations, use a unique, strong password.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Create a cluster-config.yaml according to your needs from the examples below.\nkubectl create -f cluster-config.yaml Note that StorageOS will be deployed in spec.namespace (storageos by default), irrespective of what NameSpace the CR is defined in.\n Examples You can checkout all the parameters configurable in the configuration page.\n All examples must reference the storageos-api Secret.\nspec:secretRefName:\u0026#34;storageos-api\u0026#34;# Reference to the Secret created in the previous stepsecretRefNamespace:\u0026#34;storageos-operator\u0026#34;# Namespace of the SecretCheck out Cluster Definition examples for full CR files.\nInstalling with an external etcd spec:kvBackend:address:\u0026#39;10.43.93.95:2379\u0026#39;# IP of the SVC that exposes ETCD# address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can specify individual IPs of the etcd serversIf using Etcd with mTLS, you need to specify the secret that hold the certificates with the following parameters:\nspec:# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;storageos-etcd\u0026#34;# Namespace of the client certificates secretFollow the etcd operations page to setup the secret with the Etcd client certificate, client key and CA.\nInstalling to a subset of nodes In this case we select nodes that are workers. To make sure that StorageOS doesn\u0026rsquo;t start in Master nodes.\nYou can see the labels in the nodes by kubectl get node --show-labels.\nspec:nodeSelectorTerms:- matchExpressions:- key:\u0026#34;node-role.kubernetes.io/worker\u0026#34;operator:Invalues:- \u0026#34;true\u0026#34;# OpenShift uses \u0026#34;node-role.kubernetes.io/compute=true\u0026#34;# Rancher uses \u0026#34;node-role.kubernetes.io/worker=true\u0026#34;# Kops uses \u0026#34;node-role.kubernetes.io/node=\u0026#34; Different provisioners and Kubernetes distributions use node labels differently to specify master vs workers. Node Taints are not enough to make sure StorageOS doesn\u0026rsquo;t start in a node. The JOIN variable is defined by the operator by selecting all the nodes that match the nodeSelectorTerms.\n Enabling CSI spec:csi:enable:truedeploymentStrategy:deploymentenableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:trueenableControllerExpandCreds:trueThe credentials must be defined in the storageos-api Secret\napiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# Add base64 encoded creds below for CSI credentials.csiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9zSpecifying a shared directory for use with kubelet as a container spec:sharedDir:\u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39;Defining pod resource requests and reservations spec:resources:requests:memory:\u0026#34;512Mi\u0026#34;# cpu: \u0026#34;1\u0026#34;# limits:# memory: \u0026#34;4Gi\u0026#34;Limiting StorageOS can cause malfunction for IO to StorageOS volumes, therefore we do not currently recommend applying upper limits to resources for StorageOS pods.\nSpecifying custom Tolerations spec:tolerations:- key:\u0026#34;key1\u0026#34;operator:\u0026#34;Equal\u0026#34;value:\u0026#34;value1\u0026#34;effect:\u0026#34;EffectToTolerate\u0026#34;- key:\u0026#34;key2\u0026#34;operator:\u0026#34;Exists\u0026#34;Custom tolerations specified in the StorageOSCluster definition are added to all StorageOS components; the StorageOS daemonset, CSI helper and scheduler.\nIn the above example a toleration key1=value1:EffectToTolerate would be tolerated and key2 would be tolerated regardless of the value and effect. For more information about tolerations please see the Kubernetes documentation.\n","excerpt":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and …","ref":"/v2.1/docs/reference/cluster-operator/examples/","title":"Cluster Operator examples"},{"body":" Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by default.\n StorageOS requires mount propagation enabled to present devices as volumes for containers (see linux kernel documentation here).\nCertain versions of docker ship with a systemd manifest with MountFlags set to \u0026lsquo;slave\u0026rsquo;, thus preventing StorageOS from working. This can be removed or set to \u0026lsquo;shared\u0026rsquo; with a systemd drop in:\nmkdir -p /etc/systemd/system/docker.service.d/ cat \u0026lt;\u0026lt;EOF \u0026gt; /etc/systemd/system/docker.service.d/mount_propagation_flags.conf [Service] MountFlags=shared EOF # systemctl daemon-reload # systemctl restart docker.service To confirm behaviour, the following command should run without error.\ndocker run -it --rm -v /mnt:/mnt:shared busybox sh -c /bin/date Orchestrators such as Kubernetes or OpenShift have their own ways of exposing this setting. Kubernetes 1.10 and OpenShift 3.10 have mount propagation enabled by default. Previous versions require that feature gates are enabled on the Kubernetes master\u0026rsquo;s controller-manager and apiserver services and in the kubelet service on each node.\nInstallations of orchestrators using Docker require that mount propagation is enabled for both.\nRefer to our installation pages for the orchestrators to see details on how to check and enable mount propagation where appropriate.\n","excerpt":"Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by …","ref":"/v2.1/docs/prerequisites/mountpropagation/","title":"Mount Propagation"},{"body":"StorageOS is a certified Rancher application. We offer two installation methods:\n Rancher Catalogue - this is the easiest and requires just a few clicks Manual - allowing more control and visibility Before proceeding, ensure that you have followed our prerequisites. On Rancher, pay particular attention to the OS version and image used - some platforms require extra mainline kernel modules to be enabled.\n Catalog Manual Catalog Install StorageOS is a Certified application in the Rancher Catalog. You can install StorageOS using the Rancher application install.\nBefore completing the steps below, you will need an etcd cluster. For evaluations it is sufficient to use our simple test recipe. For production installations, please follow our production recipe. Make a note of the etcd endpoint URL in either case.\n Select the System project of your cluster\n Select the Apps tab and click Launch\n Search for StorageOS and click on the App\nThis will install the StorageOS operator, which manages the StorageOS DaemonSet.\n Check and ammend installation options\nA generic configuration for StorageOS is preset using the default values in the form. Be sure to check the etcd address and ensure it matches the value you noted at the beginning of this guide.\nThe catalog form exposes several useful parameters - documented below.\nFor further customization, you can opt to set the option to \u0026lsquo;Install StorageOS Cluster\u0026rsquo; to false and install a custom CR. See below for this.\n Launch the StorageOS cluster\n Verify the cluster bootstrap has successfully completed\n License the newly installed cluster\nNewly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nYou will need access to the StorageOS API on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 Now follow the instructions on our licensing operations page to obtain and apply a license.\nInstallation of StorageOS is now complete.\n Simple Customization - Modify Catalog Form The following options are exposed by the catalog form to allow some simple customization of the StorageOS installation.\n Cluster Operator namespace : The Kubernetes namespace where the StorageOS Cluster Operator and other resources will be created. Container Images : By default images are pulled from DockerHub, you can specify the image URLs when using private registries. Install StorageOS cluster : Controls the automatic deployment of StorageOS after installing the Cluster Operator. If set to false, the Operator will be created, but a Custom Resource will not be applied to the cluster. Launch the operator and proceed to the section Advanced Customization below. Namespace : The Kubernetes namespace where StorageOS will be installed. By default, StorageOS installs into the kube-system namespace, which will add a priority class to ensure high priority resource allocation. Installing StorageOS with the priority class prevents StorageOS from being evicted during periods of resource contention. It is inadvisable to modify this under normal circumstances. Username/Password : Default Username and Password for the admin account to be created at StorageOS bootstrap. A random password will be generated by leaving the field empty or clicking the Generate button. External etcd address(es) : Connection and configuration details for an external Etcd cluster.See our documentation here. Node Selectors and Tolerations : Control placement of StorageOS DaemonSet Pods. StorageOS will only be installed on the selected nodes. Tolerations : Define any tolerations you wish the DaemonSet to observe. Advanced Customization - Apply Custom CR If Install StorageOS Cluster was set to false, StorageOS will not be bootstrapped automatically. After the StorageOS Operator is installed, you can now create a Custom Resource that describes the StorageOS cluster.\n Select the System Workloads and Import YAML Create the Secret and CustomResource This is an example.\n--- apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z --- apiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret namespace: \u0026#34;kube-system\u0026#34; k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips csi: enable: true deploymentStrategy: deployment enableControllerPublishCreds: true enableNodePublishCreds: true enableProvisionCreds: true enableControllerExpandCreds: true sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container resources: requests: memory: \u0026#34;512Mi\u0026#34; nodeSelectorTerms: - matchExpressions: - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation operator: In values: - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Manual Installation The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.1.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"StorageOS is a certified Rancher application. We offer two installation methods:\n Rancher Catalogue …","ref":"/v2.1/docs/install/rancher/","title":"Rancher"},{"body":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following yaml manifest.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n Upgrade StorageOS Operator using Helm If you have installed the StorageOS Operator using the Helm Chart, then you can upgrade the operator using the following commands.\n$ helm list NAME REVISION STATUS CHART APP VERSION NAMESPACE storageos-v1 4 DEPLOYED storageos-operator-0.2.11 1.3.0 storageos-operator $ helm repo update $ helm upgrade $NAME storageos/storageos-operator When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n ","excerpt":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following …","ref":"/v2.1/docs/reference/cluster-operator/upgrade/","title":"Cluster Operator Upgrade"},{"body":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 active volumes per node 4.x kernels have a limitation of 4096 active volumes per node We are distribution agnostic as long as our prerequisites are met Orchestrators Kubernetes 1.13+ OpenShift 4.0+ ","excerpt":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 …","ref":"/v2.1/docs/introduction/platforms/","title":"Supported Platforms and Orchestrators"},{"body":"","excerpt":"","ref":"/v2.1/docs/concepts/","title":"Concepts"},{"body":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a 64-bit architecture. Kubernetes 1.9+ with CSI. Container Runtime Engine: CRI-O, Containerd or Docker 1.10+ with mount propagation enabled. The necessary ports should be open. See the ports and firewall settings. Etcd cluster for StorageOS A mechanism for device presentation. Recommended: At least three nodes for replication and high availability. Kubernetes 1.13+. Install the storageos CLI. If using Helm2, make sure the tiller ServiceAccount has enough privileges to create resources such as Namespaces, ClusterRoles, etc. For instance, following this installation procedure. System clocks synchronized using NTP or similar methods. While our distributed consensus algorithm does not require synchronised clocks, it does help to more easily correlate logs across multiple nodes. ","excerpt":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a …","ref":"/v2.1/docs/prerequisites/","title":"Prerequisites"},{"body":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different distributions such as OpenShift, Rancher, EKS, AKS, GKE, etc. The user can provide standard PVC definitions and StorageOS will dynamically provision matching volumes. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nKubernetes and StorageOS communicate with each other to perform actions such as creation, deletion or mounting of volumes. The CSI (Container Storage Interface) driver is the standard method of communication. Using CSI, Kubernetes and StorageOS communicate over a Unix domain socket.\nIt is recommended to use the CSI StorageOS installation from Kubernetes 1.13 onwards. Although the former communication procedure, the native driver, is still in use and StorageOS maintains support for it, its use is discouraged as it is a deprecated method, to be removed in newer versions of Kubernetes.\nCSI (Container Storage Interface) Note CSI is the standard method of communication that enables storage drivers for Kubernetes to release on their own schedule. The CSI standard allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI was released GA from Kubernetes 1.13. StorageOS v2 only supports the use of CSI as a storage driver. In addition, the StorageOS Cluster Operator handles the configuration of the CSI driver and its complexity by detecting the version of the Kubernetes installed.\nCheck out the status of the CSI release cycle in relation with Kubernetes in the CSI project page.\nCSI communication is fully supported by StorageOS if the cluster is deployed with any supported Linux Distribution.\nKubernetes Upgrades on Managed Services Managed services that support in place upgrades are fully supported. However, upgrading Kubernetes using green/blue deployments is not supported. This is because nodes are replaced rather than being upgraded. Any data stored on the nodes is lost when new nodes replace the previous ones.\nDocker Some managed Kubernetes platforms such as Azure AKS, enable the \u0026lsquo;Live-Restore\u0026rsquo; Docker feature, enabling containers to continue running while Docker is stopped or upgraded. This feature can cause nodes to hang while shutting-down or rebooting, as rather than going through an orderly shutdown, StorageOS (and other processes) are killed before the disks are synced and unmounted. Devices in this inaccessible state will log a warning similar to:\nTransport endpoint not connected To prevent this behaviour, we advise disabling this feature by setting\n{ ... \u0026quot;live-restore\u0026quot;: false } in /etc/docker/daemon.json.\nHere\u0026rsquo;s an example Ansible snippet that might be used to achieve this\n... - name: configure /etc/docker/daemon.json lineinfile: path: /etc/docker/daemon.json regexp: '^.*\u0026quot;live-restore\u0026quot;: true,$' line: ' \u0026quot;live-restore\u0026quot;: false,' backrefs: yes notify: restart docker ... Note: Use at own risk; you may need to adapt the example to work in your environment\n ","excerpt":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different …","ref":"/v2.1/docs/install/","title":"Install"},{"body":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to a specific platform.\nStorageOS can be used in any Kubernetes distribution when the prerequisites are met. That includes Kubernetes manged services.\n","excerpt":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to …","ref":"/v2.1/docs/platforms/","title":"Platforms"},{"body":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential cluster metadata and manage distributed configuration state. An embedded etcd instance is included in the StorageOS container, but for production environments and testing of production workloads, we recommend deploying an external etcd cluster. For more details about, and an example of, how to run etcd, see the External etcd Operations page.\nIt is highly recommended to use external etcd for cloud environments and place the etcd cluster on stable nodes. Placing the etcd on nodes that are recycled often might affect the normal operations of StorageOS.\nSetup of storage on the hosts We recommend creating a separate filesystem for StorageOS to mitigate the risk of filling the root filesystem on nodes. This has to be done for each node in the cluster.\nFollow the managing host storage best practices page for more details.\nResource reservations StorageOS resource consumption depends on the workloads and the StorageOS features in use.\nThe recommended minimum memory reservation for the StorageOS Pods is 512MB for non-production environments. However it is recommended to prepare nodes so StorageOS can operate with at least with 1-2GB of memory. StorageOS frees memory when possible.\nFor production environments, we recommend 4GB of Memory and 1 CPU as a minimum and to test StorageOS using realistic workloads and tune resources accordingly.\nStorageOS Pods resource allocation will impact directly on the availability of volumes in case of eviction or resource limit triggered restart. It is recommended to not limit StorageOS Pods.\nStorageOS implements a storage engine, therefore limiting CPU consumption might affect the I/O throughput of your volumes.\nStorageOS API username/password The API grants full access to StorageOS functionality, therefore we recommend that the default administrative password of \u0026lsquo;storageos\u0026rsquo; is reset to something unique and strong.\nYou can change the default parameters by encoding the apiUsername and apiPassword values (in base64) into the storageos-api secret.\nTo generate a unique password, a technique such as the following, which generates a pseudo-random 24 character string, may be used:\n# Generate strong password PASSWORD=$(cat -e /dev/urandom | tr -dc \u0026#39;a-zA-Z0-9-!@#$%^\u0026amp;*()_+~\u0026#39; | fold -w 24 | head -n 1) # Convert password to base64 representation for embedding in a K8S secret BASE64PASSWORD=$(echo -n $PASSWORD | base64) Note that the Kubernetes secret containing a strong password must be created before bootstrapping the cluster. Multiple installation procedures use this Secret to create a StorageOS account when the cluster first starts.\nStorageOS Pod placement StorageOS must run on all nodes that will contribute storage capacity to the cluster or that will host Pods which use StorageOS volumes. For production environments, it is recommended to avoid placing StorageOS Pods on Master nodes.\nStorageOS is deployed with a DaemonSet controller, and therefore tolerates the standard unschedulable (:NoSchedule) action. If that is the only taint placed on master or cordoned nodes StorageOS pods might start on them (see the Kubernetes docs for more details). To avoid scheduling StorageOS pods on master nodes, you can add an arbitrary taint to them for which the StorageOS DaemonSet won\u0026rsquo;t have a toleration.\nDedicated instance groups Cloud environments give users the ability to quickly scale the number of nodes in a cluster in response to their needs. Because of the ephemeral nature of the cloud, StorageOS recommends setting conservative downscaling policies.\nFor production clusters, it recommended to use dedicated instance groups for Stateful applications that allow the user to set different scaling policies and define StorageOS pools based on node selectors to collocate volumes.\nLosing a few nodes at the same time could cause the loss of data even when volume replicas are being used.\nPort blocking StorageOS exposes ports to operate. It is recommended that the ports are not accessible from outside the scope of your cluster.\nStorageOS in Docker EE StorageOS does not support running on Swarm nodes nor on mixed (Kubernetes and Swarm) nodes. StorageOS volumes have to be provisioned and used from Kubernetes nodes.\n","excerpt":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential …","ref":"/v2.1/docs/best-practices/","title":"StorageOS Best Practices"},{"body":"","excerpt":"","ref":"/v2.1/docs/operations/","title":"Operations"},{"body":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or other Orchestrators that are derived from Kubernetes such as OpenShift or Rancher. This is useful for running stateful applications, such as databases or CI/CD applications, under the control of Kubernetes as Kubernetes can make scheduling decisions without the application data being lost.\nWhat we have outlined in the cookbooks below are some quick deployments of stateful applications into a Kubernetes cluster. These examples are not production ready but have been provided to give you some insight into how to use StorageOS with stateful applications.\nStatefulSets The examples we have provided use StatefulSets as a way to deploy applications into Kubernetes. The reason for this is that the StatefulSet controller is designed to manage stateful applications and it \u0026ldquo;provides guarantees about the ordering and uniqueness\u0026rdquo; of sets of pods.\nPractically this means that when a StatefulSet scales, pods are created in order from 0-(N-1), if a StatefulSet scales down then pods are deleted in reverse order from (N-1)-0.\nSecondly, it means that the behaviour of the StatefulSet controller differs from that of Deployment and ReplicaSet controllers. For Deployment and ReplicaSet controllers \u0026ldquo;\u0026hellip; at many points in the lifetime of a replica set there will be 2 copies of a pod\u0026rsquo;s processes running concurrently\u0026rdquo;. Having two different pods mount a volume at the same time can cause corruption of data. Currently Kubernetes accessModes only apply restrictions to nodes mounting volumes rather than pods, so it is important that StatefulSets are used with StorageOS volumes so the necessary pod uniqueness guarantees are maintained.\nStatefulSet Manifests The StorageOS specific part of the Kubernetes manifests for these examples lies in the VolumeClaimTemplate that\u0026rsquo;s part of the statefulset definition.\nVolumeClaimTemplate\napiVersion:apps/v1kind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1template:metadata:labels:app:mssqlenv:prodspec:serviceAccountName:mssqlcontainers:- name:fooimage:barvolumeMounts:- name:bazmountPath:/var/opt/barenvFrom:- configMapRef:name:mssqlvolumeClaimTemplates:- metadata:name:bazlabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiIn the StatefulSet definition above the container has a volume mount defined called baz. The definition for this volume is found in the VolumeClaimTemplate where the fast storageClass will be used to dynamically provision storage if the persistent volume does not already exist.\n","excerpt":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or …","ref":"/v2.1/docs/usecases/","title":"Use Cases: Install applications on Kubernetes with StorageOS"},{"body":"","excerpt":"","ref":"/v2.1/docs/reference/","title":"Reference"},{"body":"There are several ways to reach us if you require support. The fastest way to get in touch is to join our public Slack channel. \nYou can file a support ticket via email to support@storageos.com, or use our Help Desk portal\nTo help us provide effective support, we request that you provide as much information as possible when contacting us. The list below is a suggested starting point. Additionally, please include anything specific, such as log entries, that may help us debug your issue.\nInformation about the cluster can be automatically sent to StorageOS engineers as mentioned in the section StorageOS Diagnostic Bundle.\nPlatform Cloud provider/Bare metal OS distribution and version Kernel version docker version and installation procedure (distro packages or docker install) StorageOS Version of StorageOS storageos get nodes storageos get volumes storageos describe volume VOL_ID # in case of issues with a specific volume Orchestrator related (Kubernetes, OpenShift, etc) Version and installation method Managed or self managed? kubectl -n kube-system get pod kubectl -n kube-system logs -lapp=storageos -c storageos kubectl -n kube-system get storageclass Specific for your namespaces: kubectl describe pvc PVC_NAME Specific for your namespaces: kubectl describe pod POD_NAME StorageOS Cluster Diagnostic Bundle StorageOS has a cluster diagnostic function that aggregates cluster information.\nThe cluster diagnostic bundle is created only when a user chooses to do so. For convenience the cluster diagnostic bundle can either be uploaded to StorageOS or downloaded to the machine running the browser or CLI. The cluster diagnostic bundle is uploaded from a StorageOS node to a StorageOS GCP encrypted bucket using a TLS encrypted connection. The upload takes place only after user confirmation.\nAs a general outline the following information is sent to us:\n StorageOS logs StorageOS cluster state - node config, volumes and namespaces output of storageos describe node output of lshw output of dmesg See Support Bundle for an exhaustive list of what is included in the cluster diagnostic bundle.\nStorageOS engineers might ask for a cluster diagnostic bundle to be generated during support cases.\nThe information given in the cluster cluster diagnostic bundle is only used for support purposes and it will be removed once the data is no longer needed for such purposes. In case the information is sensitive and can\u0026rsquo;t be given to StorageOS, please make sure that support engineers have as much information about your environment as possible.\nYou can generate a cluster diagnostic bundle through the StorageOS GUI by navigating to the Cluster menu and downloading the StorageOS support Bundle. The cluster cluster diagnostic bundle can also be generated using the CLI command below.\n$ storageos get diagnostics -o storageos-diagnostic-bundle ","excerpt":"There are several ways to reach us if you require support. The fastest way to get in touch is to …","ref":"/v2.1/docs/support/","title":"Support"},{"body":"Our self-evaluation guide is a step by step recipe for installing and testing StorageOS. This guide is divided into three sections:\n Installation - install StorageOS with a single command Feature Testing - short walkthrough of some of our features Benchmarking - a recipe to benchmark StorageOS on your infrastructure For more comprehensive documentation including installation advice for complex setups, operational guides, and use-cases, please see our main documentation site.\nSupport for Self Evaluations Should you have questions or require support, there are several ways to get in touch with us. The fastest way to get in touch is to join our public Slack channel. You can also get in touch via email to info@storageos.com.\nFurthermore you can fill out the form below and we will get in touch.\n hbspt.forms.create({ portalId: \"3402546\", formId: \"a07fecd3-ce5b-4835-b136-51a94a35632b\", sfdcCampaignId: \"70158000000BAZzAAO\" }); Installation In this document we detail a simple installation suitable for evaluation purposes. The etcd we install uses a 3 node cluster with ephemeral storage, and as such is not suitable for production workloads. However, for evaluation purposes it should be sufficient. For production deployments, please see our main documentation pages.\nA standard StorageOS installation uses the StorageOS operator, which performs most platform-specific configuration for you. The StorageOS operator has been certified by Red Hat and is open source.\nThe basic installation steps are:\n Check prerequisites Install an etcd cluster Install the StorageOS Operator Install StorageOS by applying a CR for the Operator to action Prerequisites While we do not require custom kernel modules or additional userspace tooling, StorageOS does have a few basic prerequisites that are met by default by most modern distributions:\n At least 1 CPU core, 2GB RAM free TCP ports 5701-5710 and TCP \u0026amp; UDP 5711 open between all nodes in the cluster A 64bit supported operating system - StorageOS can run without additional packages in Debian 9, RancherOS, RHEL7.5,8 and CentOS7,8 and need the package linux-image-extra for Ubuntu. Mainline kernel modules target_core_mod, tcp_loop, target_core_file, target_core_user, configfs, and ui. These are present by default on most modern linux distributions, and can be installed with standard package managers. See our system configuration page for instructions. Quick Installation For those feeling brave and/or lucky, we have encapsulated the installation instructions in this guide into a short bash script. As with similar instructions elsewhere on the internet, caution is advised here. The script is liberally commented and we recommend that you inspect it in your text editor of choice before running.\n Run the following command where kubectl is installed and with the context set for your Kubernetes cluster\n curl -sL https://storageos.run | bash Note that this script deploys a cluster with etcd on transient storage, suitable for evaluation purposes only. Do not use this cluster for production workloads.\nYour new cluster can be used straight away. You can now jump to the Provision a Volume section below to provision your first volume and license the cluster.\nInstalling StorageOS Step By Step This section describes the installation steps in some more detail.\nInstall etcd Here we install a 3 node etcd cluster using the CoreOS operator to install on ephemeral storage within your Kubernetes cluster. This is suitable for evaluation purposes. For production deployment instructions, please see our production etcd page.\n Download repo\n$ git clone https://github.com/coreos/etcd-operator.git Configure NS, Role and RoleBinding\n$ export ROLE_NAME=etcd-operator $ export ROLE_BINDING_NAME=etcd-operator $ export NAMESPACE=etcd Create Namespace\n$ kubectl create namespace $NAMESPACE Deploy Operator\n$ ./etcd-operator/example/rbac/create_role.sh $ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END Create the EtcdCluster resource\n$ kubectl -n etcd create -f - \u0026lt;\u0026lt;YAML apiVersion: \u0026#34;etcd.database.coreos.com/v1beta2\u0026#34; kind: \u0026#34;EtcdCluster\u0026#34; metadata: name: \u0026#34;storageos-etcd\u0026#34; spec: size: 3 # Set to 1 if insufficient nodes version: \u0026#34;3.4.7\u0026#34; pod: etcdEnv: - name: ETCD_QUOTA_BACKEND_BYTES value: \u0026#34;2147483648\u0026#34; # 2 GB - name: ETCD_AUTO_COMPACTION_RETENTION value: \u0026#34;1000\u0026#34; # Keep 1000 revisions - name: ETCD_AUTO_COMPACTION_MODE value: \u0026#34;revision\u0026#34; # Set the revision mode resources: requests: cpu: 200m memory: 300Mi securityContext: runAsNonRoot: true runAsUser: 9000 fsGroup: 9000 tolerations: - operator: \u0026#34;Exists\u0026#34; # add tolerations to match all taints on the master nodes where etcd will be deployed affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster operator: In values: - storageos-etcd topologyKey: kubernetes.io/hostname YAML This creates a 3 node etcd cluster and associated service in Kubernetes. We can use the service endpoint when creating the StorageOS cluster.\nInstall StorageOS Operator In order to install the StorageOS operator download the requisite yaml manifests or apply them with kubectl.\n$ kubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml You can verify the operator is running using the following command\n$ kubectl get pods -n storageos-operator Install StorageOS Once the StorageOS operator has been installed a StorageOS cluster can be generated by creating a StorageOSCluster resource.\nA StorageOSCluster resource describes the state of the StorageOS cluster that is desired and the StorageOS operator will create the desired StorageOS cluster. For examples of StorageOSCluster resources please see our examples page here. For a full list of the configurable spec parameters of the StorageOSCluster resource please see here.\n Create a secret defining the API username and password\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9zEND Create a StorageOSCluster resource\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:example-storageosclusternamespace:\u0026#34;storageos-operator\u0026#34;spec:secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;storageos-operator\u0026#34;k8sDistro:\u0026#34;upstream\u0026#34;# Set the Kubernetes distribution for your cluster (upstream, eks, aks, gke, rancher, dockeree)# storageClassName: fast # The storage class creates by the StorageOS operator is configurablecsi:enable:truedeploymentStrategy:\u0026#34;deployment\u0026#34;enableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:truekvBackend:address:\u0026#34;storageos-etcd-client.etcd.svc.cluster.local:2379\u0026#34;END Confirm that the cluster has been created and that StorageOS pods are running\n$ kubectl -n kube-system get pods StorageOS pods enter a ready state after a minimum of 45s has passed.\n Deploy the StorageOS CLI as a container\n$ kubectl -n kube-system run \\ --image storageos/cli:v2.1.0 \\ --restart=Never \\ --env STORAGEOS_ENDPOINTS=storageos:5705 \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sh -c \u0026#34;while true; do sleep 999999; done\u0026#34; Provision a StorageOS Volume Now that we have a working StorageOS cluster, we can provision a volume to test everything is working as expected.\n Create a PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Verify that the volume was created with the CLI\n pvc-1 should be listed in the CLI output\n $ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that consumes the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Check that the pod starts successfully. If the pod starts successfully then the StorageOS cluster is working correctly\n$ kubectl get pod d1 -w The pod mounts a StorageOS volume under /mnt so any files written there will persist beyond the lifetime of the pod. This can be demonstrated using the following commands.\n Execute a shell inside the pod and write some data to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo Hello World! \u0026gt; /mnt/hello root@d1:/# cat /mnt/hello Hello World! should be printed out.\n Delete the pod\n$ kubectl delete pod d1 Recreate the pod\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Open a shell inside the pod and check the contents of /mnt/hello\n$ kubectl exec -it d1 -- cat /mnt/hello Hello World! should be printed out.\n License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nYou will need access to the StorageOS API on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 Now follow the instructions on our operations licensing page to obtain and apply a license.\nInstallation of StorageOS is now complete.\nStorageOS Features Now that you have a fully functional StorageOS cluster we will explain some of our features that may be of use to you as you complete application and synthetic benchmarks.\nStorageOS features are all enabled/disabled by applying labels to volumes. These labels can be passed to StorageOS via persistent volume claims (PVCs) or can be applied to volumes using the StorageOS CLI or GUI.\nThe following is not an exhaustive feature list but outlines features which are commonly of use during a self-evaluation.\nVolume Replication StorageOS enables synchronous replication of volumes using the storageos.com/replicas label.\nThe volume that is active is referred to as the master volume. The master volume and its replicas are always placed on separate nodes. In fact if a replica cannot be placed on a node without a replica of the same volume, the volume will fail to be created. For example, in a three node StorageOS cluster a volume with 3 replicas cannot be created as the third replica cannot be placed on a node that doesn\u0026rsquo;t already contain a replica of the same volume.\nSee our replication documentation for more information on volume replication.\n To test volume replication create the following PersistentVolumeClaim\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-replicated labels: storageos.com/replicas: \u0026#34;1\u0026#34; spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Note that volume replication is enabled by setting the storageos.com/replicas label on the volume.\n Confirm that a replicated volume has been created by using the StorageOS CLI or UI\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that uses the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: replicated-pod spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-replicated END Write data to the volume\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# echo Hello World! \u0026gt; /mnt/hello root@replicated-pod:/# cat /mnt/hello Hello World! should be printed out.\n Find the location of the master volume and shutdown the node\nShutting down a node causes all volumes, with online replicas, on the node to be evicted. For replicated volumes this immediately promotes a replica to become the new master.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-replicated Bound pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 5Gi RWO fast 1m $ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker (online) kind-worker2 1/1 26 seconds ago Check the location of the master volume and notice that it is on a new node. If the pod that mounted the volume was located on the same node that was shutdown then the pod will need to be recreated.\n$ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker2 (online) kind-worker2 1/1 46 seconds ago Check that the data is still accessible to the pod\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# cat /mnt/hello Hello World! should be printed out.\n Benchmarking Benchmarking storage is a complex topic. Considering the many books and papers that have been written about benchmarking, we could write many paragraphs here and only begin to scratch the surface.\nTaking this complexity into account we present recipes for both synthetic benchmarks using FIO, and a sample application benchmark to test PostgreSQL using pgbench. The approaches are complementary - synthetic benchmarks allow us to strictly control the parameters of the IO we put through the system in order to stress various aspects of it. Application benchmarks allow us to get a sense of the performance of the system when running an actual representative workload - which of course is the ultimate arbiter of performance.\nDespite the inherent complexity of benchmarking storage there are a few general considerations to keep in mind.\nConsiderations Local vs. Remote Volumes When a workload is placed on the same node as a volume, there is no network round trip for IO, and performance is consequently improved. When considering the performance of StorageOS it is important to evaluate both local and remote volumes; since for certain workloads we want the high performance of a local attachment, but we also desire the flexibility of knowing that our less performance-sensitive workloads can run from any node in the cluster.\nThe Effect of Synchronous Replication Synchronous replication does not impact the read performance of a volume, but it can have a significant impact on the write performance of the volume, since all writes must be propagated to replicas before being acked to the application. The impact varies in proportion to the inter-node latency. For an inter-node latency of 1ms, we have a max ceiling of 1000 write IOPS, and in reality a little less than that to allow for processing time on the nodes. This is less concerning then it may first appear, since many applications will issue multiple writes in parallel (known as increasing the queue depth).\nSynthetic Benchmarks Prerequisites Kubernetes cluster with a minimum of 3 nodes and a minimum of 30 Gb available capacity StorageOS CLI running as a pod in the cluster Synthetic benchmarks using tools such as FIO are a useful way to begin measuring StorageOS performance. While not fully representative of application performance, they allow us to reason about the performance of storage devices without the added complexity of simulating real world workloads, and provide results easily comparable across platforms.\nFIO has a number of parameters that can be adjusted to simulate a variety of workloads and configurations. Parameters that are particularly important are:\n Block Size - the size of the IO units performed. Queue Depth - the amount of concurrent IOs in flight IO Pattern - access can be random across the disk, or sequentially. IO subsystems typically perform better with sequential IO, because of the effect of read ahead caches, and other factors For this self-evaluation we will run a set of tests based on the excellent DBench, which distills the numerous FIO options into a series of well-crafted scenarios:\n Random Read/Write IOPS and BW - these tests measure the IOPS ceiling (with a 4k block size) and bandwidth ceiling (with a 128K block size) of the volume using a random IO pattern and high queue depth Average Latency - these tests measure the IO latency of the volume under favourable conditions using a random access pattern, low queue depth and low block size Sequential Read/Write - these tests measure the sequential read/write throughput of the volume with a high queue depth and high block size Mixed Random Read/Write IOPS - these tests measure the performance of the volume under a 60/40 read/write workload using a random access pattern and low blocksize For convenience we present a single script to run the scenarios using local and remote volumes both with and without a replica. Deploy the FIO tests for the four scenarios using the following command:\ncurl -sL https://docs.storageos.com/v2.1/sh/deploy-synthetic-benchmarks.sh | bash The script will take approximately 20 minutes to complete, and will print the results to STDOUT.\nThe exact results observed will depend on the particular platform and environment, but the following trends should be observable:\n local volumes perform faster than remote volumes read performance is independent of the number of replicas write performance is dependent on the number of replicas Application Benchmarks While synthetic benchmarks are useful for examining the behaviour of StorageOS with very specific workloads, in order to get a realistic picture of StorageOS performance actual applications should be tested.\nMany applications come with test suites which provide standard workloads. For best results, test using your application of choice with a representative configuration and real world data.\nAs an example of benchmarking an application the following steps lay out how to benchmark a Postgres database backed by a StorageOS volume.\n Start by cloning the StorageOS use cases repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/pgbench.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the Postgres examples folder\n$ cd storageos-usecases/pgbench Decide which node you want the pgbench pod and volume to be located on. The node needs to be labelled app=postgres\n$ kubectl label node \u0026lt;NODE\u0026gt; app=postgres Then set the storageos.com/hint.master label in 20-postgres-statefulset.yaml file to match the StorageOS nodeID for the node you have chosen before creating all the files. The StorageOS nodeID can be obtained using the cli and doing a describe node\n$ kubectl create -f . Confirm that Postgres is up and running\n$ kubectl get pods -w -l app=postgres Use the StorageOS CLI or the GUI to check the master volume location and the mount location. They should match\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Exec into the pgbench container and run pgbench\n$ kubectl exec -it pgbench -- bash -c \u0026#39;/opt/cpm/bin/start.sh\u0026#39; Conclusion After completing these steps you will have benchmark scores for StorageOS. Please keep in mind that benchmarks are only part of the story and that there is no replacement for testing actual production or production like workloads.\nStorageOS invites you to provide feedback on your self-evaluation to the slack channel or by directly emailing us at info@storageos.com]\n","excerpt":"Our self-evaluation guide is a step by step recipe for installing and testing StorageOS. This guide …","ref":"/v2.1/docs/self-eval/","title":"Self Evaluation Guide"},{"body":"","excerpt":"","ref":"/v2.1/index.json","title":""},{"body":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: '#ohpen_swagger_ui', presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ] }) window.ui = ui } \t","excerpt":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: …","ref":"/v2.1/docs/reference/api/","title":"API Reference"},{"body":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply licence --from-file \u0026lt;path-to-licence-file\u0026gt; $ echo \u0026#34;\u0026lt;licence file contents\u0026gt;\u0026#34; | storageos apply licence --from-stdin Flags: --from-file string reads a StorageOS product licence from a specified file path --from-stdin reads a StorageOS product licence from the standard input -h, --help help for licence Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply …","ref":"/v2.1/docs/reference/cli/apply/","title":"Apply licence"},{"body":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes.\nFundamentally, StorageOS uses the storage attached to the nodes in the StorageOS cluster to create and present virtual volumes into containers. Space on the host is consumed from the mount point /var/lib/storageos/data, so it is therefore recommended that disk devices are used exclusively for StorageOS, as described in Managing Host Storage \nStorageOS is agnostic to the underlying storage and runs equally well on bare metal, in virtual machines or on cloud providers.\nRead about the cloud native storage principles behind StorageOS.\nStorageOS on Kubernetes StorageOS is architected as a series of containers that fulfil separate, discrete functions.\n StorageOS Controlplane\nResponsible for monitoring and maintaining the state of volumes and nodes in the cluster\n StorageOS Dataplane\nResponsible for all I/O path related tasks; reading, writing, compression and caching\n StorageOS Scheduler\nResponsible for scheduling applications on the same node as applications volume\n CSI helper\nResponsible for registering StorageOS with Kubernetes as a CSI driver\n StorageOS Operator\nResponsible for the creation and maintenance of the StorageOS cluster\n StorageOS is deployed by the StorageOS operator. In Kubernetes, the StorageOS Controlplane and Dataplane are deployed in a single pod managed by a daemonset. This daemonset runs on every node in the cluster that will consume or present storage. The scheduler, CSI helper and Operator run as separate pods and are controlled as deployments.\nStorageOS is designed to feel familiar to Kubernetes and Docker users. Storage is managed through standard StorageClasses and PersistentVolumeClaims, and features are controlled by Kubernetes-style labels and selectors, prefixed with storageos.com/. By default, volumes are cached to improve read performance and compressed to reduce network traffic.\nAny pod may mount a StorageOS virtual volume from any node that is also running StorageOS, regardless of whether the pod and volume are collocated on the same node. Therefore, applications may be started or restarted on any node and access volumes transparently.\n","excerpt":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes. …","ref":"/v2.1/docs/concepts/architecture/","title":"Architecture"},{"body":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos attach --namespace my-namespace-name my-volume my-node Flags: -h, --help help for attach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos …","ref":"/v2.1/docs/reference/cli/attach/","title":"Attach"},{"body":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our Kubernetes installation instructions page.\nAn EKS deployment of Kubernetes uses AWS Linux by default with an optimized kernel. As the requisite kernel modules are not available for StorageOS to use TCMU, FUSE will be used as a fallback. Using FUSE instead of TCMU has performance implications.\nFor more details about the OS Distributions check the System Configuration page.\n","excerpt":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our …","ref":"/v2.1/docs/platforms/aws-eks/","title":"AWS EKS"},{"body":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our Kubernetes installation instructions page.\nAn AKS deployment of Kubernetes uses Ubuntu by default with an optimized kernel. All versions of Ubuntu with a kernel version later than4.15.0-1029-azure meet the StorageOS prerequisites.\n","excerpt":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our …","ref":"/v2.1/docs/platforms/azure-aks/","title":"Azure AKS"},{"body":"In this example use case we provide three different strategies for accessing files that have been written to a StorageOS persistent volume.\nIn the following examples the \u0026ldquo;application\u0026rdquo; container is the container main, which has a rsync, Nginx or sftp sidecar container. The StorageOS volume that the application is writing to will be mounted into the sidecar container so files written by the application are available for export. Files can be exported using Nginx as a web file server, transferred using rsync or accessed via SFTP.\nThe files create a stateful set that can be used AFTER a StorageOS cluster has been created. See our guide on how to install StorageOS on Kubernetes for more information.\nClone Repository In order to deploy the examples, clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/backup Before deploying the backup-example stateful set we recommend looking through the examples to understand how the different containers are configured\n Exfiltrating files through HTTP Deploy the Nginx example\n$ kubectl create -f nginx/ service/backup-example created configmap/nginx-config created statefulset.apps/backup-example created pod/busybox created Check that a backup-example pod is running\n$ kubectl get pods -w -l app=backup-example-nginx NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-nginx-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-nginx ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 80/TCP 46s Use wget to access the files served by Nginx. Nginx is sharing files from the same volume that the main application container is writing to. The connection to the Nginx container is made via the backup-example service.\n$ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;lost%2Bfound/\u0026#34;\u0026gt;lost+found/\u0026lt;/a\u0026gt; 12-Feb-2019 12:32 - \u0026lt;a href=\u0026#34;date.txt\u0026#34;\u0026gt;date.txt\u0026lt;/a\u0026gt; 12-Feb-2019 12:49 29 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx/date.txt Tue Feb 12 12:49:15 UTC 2019 Depending on what files have been written to the StorageOS volume the output of the index file will be different. In the example the date.txt file we created in Step 2 is present on the volume.\nExfiltrating files through Rsync Deploy the rsync example\n$ kubectl create -f rsync/ service/backup-example created configmap/rsync-config created secret/rsync-credentials created statefulset.apps/backup-example created pod/rsync created Check that a backup-example pod is running\n$ kubectl get pods -w -l app=backup-example-rsync NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-rsync-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-rsync NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-rsync ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 873/TCP 46s Use rsync to access the files shared by the rsync daemon. rsync is sharing files from the same volume that the main container is writing to. A username and password that are set in the rsync-credentials secret. The secret supplied in the example has the username and password set to username and password.\n$ kubectl exec -it rsync sh / # rsync --list-only rsync://username@backup-example-rsync/share Password: drwxr-xr-x 4,096 2019/02/12 12:49:15 . -rw-r--r-- 29 2019/02/12 12:49:15 date.txt drwx------ 16,384 2019/02/12 12:32:40 lost+found / # rsync -chavzP rsync://username@backup-example-rsync/share/date.txt . Password: receiving incremental file list date.txt 29 100% 28.32kB/s 0:00:00 (xfr#1, to-chk=0/1) sent 43 bytes received 135 bytes 50.86 bytes/sec total size is 29 speedup is 0.16 / # cat date.txt Tue Feb 12 12:49:15 UTC 2019 In the example above the list of available files was shown and a file called date.txt was synchronized to the rsync container.\nExfiltrating files through SFTP Deploy the sftp example\n$ kubectl create -f sftp/ Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-sftp-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-sftp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-sftp ClusterIP 100.70.50.56 \u0026lt;none\u0026gt; 22/TCP 2h Use SFTP to access the files shared by the SFTP container. If you have made no changes to the sftp-config secret the password is password.\n$ kubectl exec -it sftp -- bash root@sftp:/# sftp alex@backup-example-sftp alex@backup-example-sftp\u0026#39;s password: Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found sftp\u0026gt; get date.txt Fetching /date.txt to date.txt /date.txt 100% 29 15.9KB/s 00:00 sftp\u0026gt; bye root@sftp:/# cat date.txt Tue Feb 12 17:51:32 UTC 2019 In order to do this a SFTP user needs to be configured. The details for the user are stored in the sftp-config secret (see sftp/17-secret.yaml). The secret consists of base64 encoded username:password:uid:guid and the user is chroot\u0026rsquo;ed inside their home directory so the mount point for the StorageOS volume in the SFTP container in sftp/20-backup-pod.yaml needs to be configured.\nUsing custom SSH Keys The ConfigMap ssh-key-pub (see sftp/15-configmap.yaml) needs to be populated with a public key. The corresponding private key needs to be base64 encoded and put into the ssh-key-private secret (see sftp/17-secret.yaml). The user to connect as is determined by the user that is configured in the sftp-config configMap. To restrict logins to the SSH key edit the sftp-config secret so it contains no password (user::uid:guid).\n Connect to the sftp pod and connect through the service to the SFTP container running inside the backup-example pod.\n$ kubectl exec -it sftp -- bash root@sftp:/# sftp -i /home/alex/.ssh/id_rsa alex@backup-example-sftp Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found ","excerpt":"In this example use case we provide three different strategies for accessing files that have been …","ref":"/v2.1/docs/usecases/backups/","title":"Backing up files from StorageOS volumes"},{"body":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing StorageOS persistent volumes with Cassandra means that if a Cassandra pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up the pod data is immediately available. Should Kubernetes schedule the Cassandra pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not a StorageOS master is located on the same node.\nAs Cassandra has features to allow it to handle replication careful consideration of whether to allow StorageOS or Cassandra to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Cassandra on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiVersion:apps/v1kind:StatefulSetmetadata:name:cassandraspec:selector:matchLabels:app:cassandraserviceName:cassandrareplicas:3...spec:...volumeMounts:- name:cassandra-datamountPath:/var/lib/cassandra...volumeClaimTemplates:- metadata:name:cassandra-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Move into the Cassandra examples folder and create the objects\ncd storageos-usecases kubectl create -f ./cassandra Confirm Cassandra is up and running.\n$ kubectl get pods -w -l app=cassandra NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 8m32s cassandra-1 1/1 Running 0 7m51s cassandra-2 1/1 Running 0 6m36s Connect to the Cassandra client pod and connect to the Cassandra server through the service\n$ kubectl exec -it cassandra-0 -- cqlsh cassandra-0.cassandra Connected to K8Demo at cassandra-0.cassandra:9042. [cqlsh 5.0.1 | Cassandra 3.11.3 | CQL spec 3.4.4 | Native protocol v4] Use HELP for help. cqlsh\u0026gt; SELECT cluster_name, listen_address FROM system.local; cluster_name | listen_address --------------+---------------- K8Demo | 100.96.7.124 (1 rows) ","excerpt":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing …","ref":"/v2.1/docs/usecases/cassandra/","title":"Cassandra"},{"body":"The storageos command line interface (CLI) is used to manage cluster-wide configuration.\nInstallation # linux/amd64 curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.1.0/storageos_linux_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ # MacOS curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.1.0/storageos_darwin_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ You will need to provide the correct credentials to connect to the API. The default installation uses the storageos-api Secret to generate the first admin user. By default, it creates a single user with username storageos and password storageos:\nexport STORAGEOS_USERNAME=storageos export STORAGEOS_PASSWORD=storageos export STORAGEOS_ENDPOINTS=10.1.5.249:5705 Usage $ storageos Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Formatting CLI Output StorageOS CLI output can be formatted using the --output option. The strings that are passed to --output are \u0026lsquo;json\u0026rsquo;, \u0026lsquo;yaml\u0026rsquo; or \u0026lsquo;text\u0026rsquo;.\nCheatsheet Command Subcommand Description apply licence Apply licence to cluster. attach Attach Volume to a node. create user, volume Create resources such as users or volumes. delete volume Delete resources. describe node, volume Show detailed view of resources. detach volume Detach volume from a node. get cluster, diagnostics, namespace, node, user, volume List resources. help Help version Show CLI version. Source is available on Github.\n","excerpt":"The storageos command line interface (CLI) is used to manage cluster-wide configuration. …","ref":"/v2.1/docs/reference/cli/","title":"Command line interface"},{"body":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator acts as a Kubernetes controller that watches the StorageOSCluster CR (Custom Resource). Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\nYou can find the source code in the cluster-operator repository.\nInstall the operator following orchestrator specific procedure.\nTo deploy a StorageOS cluster you will need to fulfil the following steps.\n Install Etcd Install operator Create storageos-api Secret Create a CR (Custom Resource) to deploy StorageOS ","excerpt":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS …","ref":"/v2.1/docs/reference/cluster-operator/","title":"Cluster Operator"},{"body":"StorageOS clusters represent groups of nodes which run a common distributed control plane.\nTypically, a StorageOS cluster maps one-to-one to a Kubernetes (or similar orchestrator) cluster, and we expect our daemonset to run on all worker nodes within the cluster that will consume or present storage.\nClusters use etcd to maintain state and manage distributed consensus between nodes.\n","excerpt":"StorageOS clusters represent groups of nodes which run a common distributed control plane. …","ref":"/v2.1/docs/concepts/clusters/","title":"Clusters"},{"body":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is generally increased when compression is enabled due to fewer read/write operations taking place on the backend store (the volumes\u0026rsquo; blob files). Compression can be disabled by setting the label storageos.com/nocompress=true on a volume.\nStorageOS utilises the lz4 compression algorithm when writing to the backend store and when compressing replication traffic before it is sent across the network. Compression is granular per 4k block and data will remain compressed/uncompressed once written to a volume. Therefore, compression can be dynamically enabled and disabled by setting the storageos.com/nocompress label on a volume.\nStorageOS detects whether a block can be compressed or not by creating a heuristic that predicts the size of a compressed block. If the heuristic indicates that the compressed block is likely to be larger than the original block then the uncompressed block is stored. Block size increases post compression if the compression dictionary is added to a block that cannot be compressed. By verifying whether blocks can be compressed, disk efficiency is increased and CPU resources are not wasted on attempts to compress uncompressible blocks. StorageOS\u0026rsquo; patented on disk format is used to tell whether individual blocks are compressed without overhead. As such volume compression can be dynamically enabled/disabled even while a volume is in use.\nWhen compression and encryption are both enabled for a volume, blocks are compressed then encrypted.\n","excerpt":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is …","ref":"/v2.1/docs/concepts/compression/","title":"Compression"},{"body":"We are always looking to improve our documentation. If you like to help people and can write, read on for the process for submitting your contributions. If your guide is published, you\u0026rsquo;ll receive $250 per article by PayPal.\nContent guidelines A guide contains step by step instructions for how to accomplish a specific task using StorageOS. To be accepted, guides must be:\n Written in English. Relevant, accurate and complete. Technically correct and thoroughly tested. Follow the writing style guide. Guides should avoid:\n Duplicating an existing guide or other sources, such as blogs or forum posts. Including irrelevant material. Submission and review You should submit your guide as a pull request to the GitHub repo.\nYour submission will be left open for community review for two weeks. After this, your submission will be reviewed internally for about another week.\nIf accepted, your pull request will be approved and you will have 36 hours to send your submission title and PayPal account information. Non-response will be taken as a go-ahead to publish.\nLegal COPYRIGHT OWNERSHIP. The StorageOS Guides \u0026amp; Tutorials repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license.\nCREDIT. Nothing contained in this Agreement shall be deeded to require StorageOS to use the Work, or any part thereof, in connection with StorageOS Guides \u0026amp; Tutorials or otherwise. Credit for the Work shall read, “Contributed by writer’s name.”\nPAYMENT. Upon publication of a submission to the StorageOS Guides \u0026amp; Tutorials Repository, the writer will be paid the sum of USD $250.00 as an electronic payment.\n","excerpt":"We are always looking to improve our documentation. If you like to help people and can write, read …","ref":"/v2.1/docs/reference/contributing/","title":"Contributing to the docs"},{"body":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: user Create a new user account volume Provision a new volume Flags: -h, --help help for create Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos create [command] --help\u0026#34; for more information about a command. create volume $ storageos create volume --help Provision a new volume Usage: storageos create volume [flags] Examples: $ storageos create volume --description \u0026#34;This volume contains the data for my app\u0026#34; --fs-type \u0026#34;ext4\u0026#34; --labels env=prod,rack=db-1 --size 10GiB --namespace my-namespace-name my-app $ storageos create volume --replicas 1 --namespace my-namespace-name my-replicated-app Flags: --async perform the operation asynchronously, using the configured timeout duration --cache caches volume data (default true) --compress compress data stored by the volume at rest and during transit (default true) -d, --description string a human-friendly description to give the volume -f, --fs-type string the filesystem to format the new volume with once provisioned (default \u0026#34;ext4\u0026#34;) -h, --help help for volume -l, --labels strings an optional set of labels to assign to the new volume, provided as a comma-separated list of key=value pairs -r, --replicas uint the number of replicated copies of the volume to maintain -s, --size string the capacity to provision the volume with (default \u0026#34;5GiB\u0026#34;) --throttle deprioritises the volume\u0026#39;s traffic by reducing the rate of disk I/O Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) create user $ storageos create user --help Create a new user account Usage: storageos create user [flags] Examples: $ storageos create user --with-username=alice --with-admin=true Flags: -h, --help help for user --with-admin control whether the user is given administrative priviledges --with-groups stringArray the list of policy groups to assign to the user --with-password string the password to assign to the user. If not specified, this will be prompted for. --with-username string the username to assign Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: …","ref":"/v2.1/docs/reference/cli/create/","title":"Create"},{"body":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available Commands: volume Delete a volume Flags: -h, --help help for delete Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos delete [command] --help\u0026#34; for more information about a command. delete volume $ storageos delete volume --help Delete a volume Usage: storageos delete volume [volume name] [flags] Examples: $ storageos delete volume my-test-volume my-unneeded-volume $ storagoes delete volume --namespace my-namespace my-old-volume Flags: --async perform the operation asynchronously, using the configured timeout duration --cas string make changes to a resource conditional upon matching a provided version -h, --help help for volume Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available …","ref":"/v2.1/docs/reference/cli/delete/","title":"Delete"},{"body":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] Available Commands: node Retrieve detailed information for nodes in the cluster volume Show detailed information for volumes Flags: -h, --help help for describe Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos describe [command] --help\u0026#34; for more information about a command. describe node $ storageos describe node --help Retrieve detailed information for nodes in the cluster Usage: storageos describe node [node names...] [flags] Aliases: node, nodes Examples: $ storageos describe node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) describe volume $ storageos describe volume --help Show detailed information for volumes Usage: storageos describe volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos describe volumes $ storageos describe volume --namespace my-namespace-name my-volume-name Flags: -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] …","ref":"/v2.1/docs/reference/cli/describe/","title":"Describe"},{"body":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] Examples: $ storageos detach volume --namespace my-namespace-name my-volume Flags: --cas string make changes to a resource conditional upon matching a provided version -h, --help help for detach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] …","ref":"/v2.1/docs/reference/cli/detach/","title":"Detach Volume"},{"body":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page.\nDocker EE and the Universal Control Plane can run on different Linux distributions. StorageOS supports RHEL, CentOS, Debian, and selected Ubuntu images. For more details, check out the supported OSs in the prerequisites page.\nStorageOS only supports Kubernetes nodes managed by Docker Enterprise Edition, not those nodes running Swarm. Mixed nodes (those running Kubernetes and Swarm workloads) are not supported. As a consequence, StorageOS volumes can only be provisioned on Kubernetes nodes, and only these nodes should be used for stateful workloads.\n","excerpt":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page. …","ref":"/v2.1/docs/platforms/dockeree/","title":"Docker Enterprise Edition"},{"body":"","excerpt":"","ref":"/v2.1/docs/","title":"Documentation"},{"body":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, most popularly used to aggregate logs, but also to serve as a search backend to a number of different applications.\nUsing StorageOS persistent volumes with ElasticSearch (ES) means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Elasticsearch pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nElasticsearch has features to allow it to handle data replication, and as such careful consideration of whether to allow StorageOS or Elasticsearch to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Elasticsearch on Kubernetes Prerequisites Some OS tuning is required, which is done automatically when using our example from the use cases repository.\nElasticsearch requires vm.max_map_count to be increased to a minimum of 262144, which is a system wide setting. One way to achieve this is to run sysctl -w vm.max_map_count=262144 and update /etc/sysctl.conf to ensure it persists over a reboot. See ElasicSearch reference here.\nAdministrators should be aware that this impacts the behaviour of nodes and that there may be collisions with other application settings. Administrators are advised to centrally collate sysctl settings using the tooling of their choice.\nDeployment of the application StatefulSet defintion ---apiVersion:apps/v1kind:StatefulSetmetadata:name:esdata[...]spec:serviceAccountName:elasticsearchcontainers:- name:dataimage:elasticsearch:6.7.0imagePullPolicy:IfNotPresent[...]volumeMounts:- name:datamountPath:/usr/share/elasticsearch/data/data[...]volumeClaimTemplates:- metadata:name:\u0026#34;data\u0026#34;spec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- default StorageOS storage class nameresources:requests:storage:10Gi# \u0026lt;--- change this to the appropriate valueThis excerpt is from the StatefulSet definition (/elasticsearch/10-es-data.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nInstallation Clone the use cases repo You can find the latest files in the StorageOS use cases repostiory in /elasticsearch/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases Create the kubernetes objects\nThis will install an ES cluster with 3 master, 3 data and 3 coordinator nodes. Combined they will require ~ 14 GiB of available memory in your cluster, however, more may be used as the application is being used\nkubectl apply -f ./elasticsearch/ Once completed, an internal service object will have been created making the cluster available as http://elasticsearch:9200/ which is the default Kibana (when installed via Helm) will be using.\n Confirm Elasticsearch is up and running\nkubectl get pods -l component=elasticsearch NAME READY STATUS RESTARTS AGE elasticsearch-exporter-d86ffd94-zw45l 1/1 Running 0 5m44s es-coordinator-b7b984dd4-7wlz5 1/1 Running 0 5m44s es-coordinator-b7b984dd4-89w26 1/1 Running 0 5m44s es-coordinator-b7b984dd4-b4t6j 1/1 Running 0 5m44s es-master-78dfd5b49f-9gf5c 1/1 Running 0 5m44s es-master-78dfd5b49f-smsbw 1/1 Running 0 5m44s es-master-78dfd5b49f-z4qpj 1/1 Running 0 5m44s esdata-0 1/1 Running 0 5m44s esdata-1 1/1 Running 0 4m34s esdata-2 1/1 Running 0 3m22s Connect to ElasticSearch\nTo connect to ES directly, you can use the following port-forward command\nkubectl port-forward svc/elasticsearch 9200 and then access it via http://localhost:9200\n Kibana (optional) One of the most popular uses of ES is to use it for log aggregation and indexing, Kibana helps us visualize the data in these indices and can be easily used when installed via its Helm chart\n Install the helm chart.\nhelm install stable/kibana Once installed, use a port-foward to Kibana instead of directly to ES\nkubectl port-forward --namespace default $(kubectl get pods --namespace default -l \u0026#34;app=kibana\u0026#34; -o jsonpath=\u0026#34;{.items[0].metadata.name}\u0026#34;) 5601 and then access it via http://localhost:5601\n Monitoring (optional) As part of the example deployment, ES metrics are exposed and can be scraped by Prometheus on port 9108 (see 77-es-exporter.yaml). This is enabled by default, and should work with the default Prometheus install via Helm. If you\u0026rsquo;re using the Prometheus service monitors, you can monitor this installation by creating a monitor for the es-exporter service. For an example of how this is done to monitor StorageOS, please see prometheus-setup.\n","excerpt":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, …","ref":"/v2.1/docs/usecases/elasticsearch/","title":"Elasticsearch"},{"body":"Etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to durably persist the StorageOS cluster state. As the backing store for Kubernetes, StorageOS uses etcd for many of the same reasons.\nStorageOS uses etcd as the single source of truth for all StorageOS objects. Whenever a request is made to create, update or delete an object the result is written to etcd before the request is completed. Using etcd as a configuration store allows nodes to retrieve the current cluster state after being offlined, allowing offlined nodes to rejoin the cluster.\n N.B. StorageOS v2.0 does not provide an embedded etcd server as previous versions did. You will need to setup an etcd server for StorageOS to use prior to installation of StorageOS. Please see our etcd prerequisites page for more information on how to install and configure etcd.\n ","excerpt":"Etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to …","ref":"/v2.1/docs/concepts/etcd/","title":"Etcd"},{"body":"Check the Etcd prerequisites page for a step by step installation of Etcd.\nBest practices StorageOS uses etcd as a service, whether it is deployed following the step by step instructions or as a custom installation. It is expected that the user maintains the availability and integrity of the etcd cluster.\nIt is highly recommended to keep the cluster backed up and ensure high availability of its data.\nLow latency It is important to keep the latency between StorageOS nodes and the etcd replicas low. Deploying an etcd cluster in a different data center or region can make StorageOS detect etcd nodes as unavailable due to latency. 10ms latency between StorageOS and etcd would be the maximum threshold for proper functioning of the system.\nMonitoring It is highly recommended to add monitoring to the etcd cluster. Etcd serves Prometheus metrics on the client port http://etcd-url:2379/metrics.\nYou can use StorageOS developed Grafana Dashboards for etcd. When using etcd for production, you can use the etcd-cluster-as-service, while the etcd-cluster-as-pod can be used when using etcd from the operator.\nDefragmentation Etcd uses revisions to store multiple versions of keys. Compaction removes all key revision prior to a certain revision from etcd. Typically the etcd configuration enables the automatic compaction of keys to prevent performance degradation and limit the storage required. Compaction of revisions can create fragmentation that means space on disk is available for use by etcd but is unavailable for use by the file system. In order to reclaim this space, etcd can be defragmented.\nReclaiming space is important because when the etcd database file grows over the \u0026ldquo;DB_BACKEND_BYTES\u0026rdquo; parameter, the cluster triggers an alarm and sets itself read only and only allows reads and deletes. To avoid hitting the db backend bytes limit, compaction and defragmentation are required. How often defragmentation is required depends on the churn of key revisions in etcd.\nThe Grafana Dashboards mentioned above indicate when nodes require defragmentation. Be aware that defragmentation is a blocking operation that is performed per node, hence the etcd node will be locked for the duration of the defragmentation. Defragmentation usually takes a few milliseconds to complete.\nYou can also set cronjobs that execute the following defragmentation script. It will run a defrag when the DB is at 80% full. A defragmentation operation has to be executed per etcd node and it is a blocking operation. It is recommended to not execute the degragmentation in all etcd members at the same time. If using a cronjob, set them up for different times.\ncurl -sSLo defrag-etcd.sh https://raw.githubusercontent.com/storageos/deploy/master/k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd/roles/install_etcd/templates/defrag-etcd.sh.j2 chmod +x defrag-etcd.sh Known CoreOS Etcd Operator issues This topology is only recommended for deployments where isolated nodes cannot be used.\nEtcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd in pods it is therefore important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster causes the loss of all the etcd keystore and make StorageOS unable to perform metadata changes.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. A restore of the etcd keyspace from a backup might cause issues due to the disparity between the cluster state and its metadata in a different point in time. If you need to restore from a backup after a failure of etcd, contact the StorageOS support team.\n","excerpt":"Check the Etcd prerequisites page for a step by step installation of Etcd.\nBest practices StorageOS …","ref":"/v2.1/docs/operations/etcd/","title":"Etcd"},{"body":"StorageOS requires an etcd cluster in order to function. For more information on why etcd is required please see our etcd concepts page.\nWe do not support using the Kubernetes etcd for StorageOS installations.\nWe provide two methods for installing etcd. For those looking for a quick route to evaluating StorageOS, our first method installs etcd into your Kubernetes cluster using the CoreOS operator. Due to limitations in the CoreOS operator, this installation is not backed by any persistent storage, and therefore is unsuitable for production installations.\nFor production installations, there is currently no satisfactory way of installing a production-grade etcd cluster inside Kubernetes (although the landscape is changing rapidly, watch this space), and our production guidance remains to install etcd on separate machines outside of your Kubernetes cluster. This method is the best way to ensure a stable StorageOS cluster.\n Ephemeral pods within Kubernetes (Testing) External Virtual Machines (Production) Click the tabs below to select the installation method of your choice.\n Testing Production Testing - Installing Etcd Into Your Kubernetes Cluster This fast and convenient method is useful for quickly creating an etcd cluster in order to evaluate StorageOS. Do not use it for production installations.\nThis method uses the CoreOS etcd-operator to install a 3 node etcd cluster within your Kubernetes cluster, in the storageos-etcd namespace. We then install a Kubernetes service in that same namespace.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. A restore of the etcd keyspace from a backup might cause issues due to the disparity between the cluster state and its metadata in a different point in time. If you need to restore from a backup after a failure of etcd, contact the StorageOS support team.\nQuick Install For a one command install, the following script uses kubectl to create an etcd cluster in the storageos-etcd namespace. It requires kubectl in the system path, and the context set to the appropriate cluster.\ncurl -s https://docs.storageos.com/v2.1/sh/deploy-etcd.sh | bash Installation Step by Step For those who would prefer to execute the steps by themselves, they are as follows:\n Configure Namespace\nexport NAMESPACE=storageos-etcd Create Namespace\nkubectl create namespace $NAMESPACE If running in Openshift, an SCC is needed to start Pods\noc adm policy add-scc-to-user anyuid system:serviceaccount:$NAMESPACE:default Create ClusterRole and ClusterRoleBinding\n$ kubectl -n $NAMESPACE create -f-\u0026lt;\u0026lt;END apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: etcd-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: etcd-operator subjects: - kind: ServiceAccount name: default namespace: $NAMESPACE END $ kubectl -n $NAMESPACE create -f-\u0026lt;\u0026lt;END apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: etcd-operator rules: - apiGroups: - etcd.database.coreos.com resources: - etcdclusters - etcdbackups - etcdrestores verbs: - \u0026#34;*\u0026#34; - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - \u0026#34;*\u0026#34; - apiGroups: - \u0026#34;\u0026#34; resources: - pods - services - endpoints - persistentvolumeclaims - events verbs: - \u0026#34;*\u0026#34; - apiGroups: - apps resources: - deployments verbs: - \u0026#34;*\u0026#34; # The following permissions can be removed if not using S3 backup and TLS - apiGroups: - \u0026#34;\u0026#34; resources: - secrets verbs: - get END Deploy Etcd Operator\n$ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END Wait for the Etcd Operator Pod to start\nkubectl -n $NAMESPACE get pod -lapp=etcd-operator Create the EtcdCluster resource\n$kubectl-n$NAMESPACEcreate-f- \u0026lt;\u0026lt;ENDapiVersion:\u0026#34;etcd.database.coreos.com/v1beta2\u0026#34;kind:\u0026#34;EtcdCluster\u0026#34;metadata:name:\u0026#34;storageos-etcd\u0026#34;spec:size:3version:\u0026#34;3.4.7\u0026#34;pod:etcdEnv:- name:ETCD_QUOTA_BACKEND_BYTESvalue:\u0026#34;2147483648\u0026#34;# 2 GB- name:ETCD_AUTO_COMPACTION_RETENTIONvalue:\u0026#34;1000\u0026#34;# Keep 1000 revisions (default)- name:ETCD_AUTO_COMPACTION_MODEvalue:\u0026#34;revision\u0026#34;# Set the revision moderesources:requests:cpu:200mmemory:300MisecurityContext:runAsNonRoot:truerunAsUser:9000fsGroup:9000tolerations:- operator:\u0026#34;Exists\u0026#34;affinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight:100podAffinityTerm:labelSelector:matchExpressions:- key:etcd_clusteroperator:Invalues:- storageos-etcdtopologyKey:kubernetes.io/hostnameEND Installation Verification $ kubectl -n storageos-etcd get pod,svc NAME READY STATUS RESTARTS AGE pod/etcd-operator-55978c4587-8kx7b 1/1 Running 0 2h pod/storageos-etcd-qm9tmrpnlm 1/1 Running 0 2h pod/storageos-etcd-rzhjdz74hp 1/1 Running 0 2h pod/storageos-etcd-wvvv2d9g98 1/1 Running 0 2h NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/storageos-etcd ClusterIP None \u0026lt;none\u0026gt; 2379/TCP,2380/TCP 22h service/storageos-etcd-client ClusterIP 172.30.132.255 \u0026lt;none\u0026gt; 2379/TCP 22h The URL from the Service storageos-etcd-client.storageos-etcd.svc:2379 will be used later in the StorageOS Cluster CustomResource the kvBackend.address.\n Known etcd-operator issues Etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd using the CoreOS Operator, it is important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster causes the loss of all the etcd keystore and make StorageOS unable to perform metadata changes.\n Production - Etcd on External Virtual Machines For production installations, StorageOS strongly recommends running etcd outside of Kubernetes on a minimum of 3 dedicated virtual machines. This topology offers strong guarantees of resilience and uptime. We recommend this architecture in all environments, including those where Kubernetes is being deployed as a managed service.\nStorageOS doesn\u0026rsquo;t require a high performance etcd cluster as the throughput of metadata to the cluster is low.\nDepending on the level of redundancy you feel comfortable with you can install etcd on the Kubernetes Master nodes. Take extreme care to avoid collisions of the StorageOS etcd installation with the Kubernetes etcd when using the Kubernetes Master nodes. Precautions such as changing the default configuration for the client and peer ports, and ensuring the etcd data directory is modified. The ansible playbook below will default the etcd installation directory to /var/lib/storageos-etcd.\nYou can choose between two installation options.\n Manual Installation Ansible Installation Installation - Manual This section documents the steps required for manual installation of etcd using standard package management commands and systemd manifests.\n Repeat the following steps on all the nodes that will run etcd as a systemd service.\n Configure Etcd version and ports\nexport ETCD_VERSION=\u0026#34;3.4.9\u0026#34; export CLIENT_PORT=\u0026#34;2379\u0026#34; export PEERS_PORT=\u0026#34;2380\u0026#34; If targeting Kubernetes Master nodes, you must change CLIENT_PORT, PEERS_PORT\n Download Etcd from CoreOS official site\ncurl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz mkdir -p /tmp/etcd-v${ETCD_VERSION}-linux-amd64 tar -xzvf /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -C /tmp/etcd-v${ETCD_VERSION}-linux-amd64 --strip-components=1 rm /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz Install Etcd binaries\ncd /tmp/etcd-v${ETCD_VERSION}-linux-amd64 mv etcd /usr/local/sbin/etcd3 mv etcdctl /usr/local/sbin/etcdctl chmod 0755 /usr/local/sbin/etcd3 /usr/local/sbin/etcdctl Set up persistent Etcd data directory\nmkdir /var/lib/storageos-etcd Create the systemd environment file\nOn all nodes that will run etcd create a systemd environemnt file /etc/etcd.conf which has the IPs of all the nodes. The NODE_IP will need to change to correspond to the node IP where the environment file resides. NODE1_IP, NODE2_IP and NODE3_IP will remain the same across all three files.\n$ cat \u0026lt;\u0026lt;END \u0026gt; /etc/etcd.conf # NODE_IP is the IP of the node where this file resides. NODE_IP=10.64.10.228 # Node 1 IP NODE1_IP=10.64.10.228 # Node 2 IP NODE2_IP=10.64.14.233 # Node 3 IP NODE3_IP=10.64.12.111 CLIENT_PORT=${CLIENT_PORT} PEERS_PORT=${PEERS_PORT} END # Verify that variables are expaneded in the file $ cat /etc/etcd.conf Create the systemd unit file for etcd3 service\nCreate a systemd unit file /etc/systemd/system/etcd3.service with the following information:\n[Unit] Description=etcd3 Documentation=https://github.com/coreos/etcd Conflicts=etcd2.service [Service] Type=notify Restart=always RestartSec=5s LimitNOFILE=40000 TimeoutStartSec=0 EnvironmentFile=/etc/etcd.conf ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ --data-dir /var/lib/storageos-etcd \\ --quota-backend-bytes 8589934592 \\ --auto-compaction-retention 1000 \\ --auto-compaction-mode revision \\ --initial-cluster-state new \\ --initial-cluster-token etcd-token \\ --listen-client-urls http://${NODE_IP}:${CLIENT_PORT},http://127.0.0.1:2379 \\ --advertise-client-urls http://${NODE_IP}:${CLIENT_PORT} \\ --listen-peer-urls http://${NODE_IP}:${PEERS_PORT} \\ --initial-advertise-peer-urls http://${NODE_IP}:${PEERS_PORT} \\ --initial-cluster etcd-${NODE1_IP}=http://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=http://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=http://${NODE3_IP}:${PEERS_PORT} [Install] WantedBy=multi-user.target $NODE_IP is the IP address of the machine you are installing etcd on.`\n If enabling TLS, it is recomended to generate your own CA certificate and key. You will need to distribute the keys and certificates for the client auth on all etcd nodes. Moreover, the ExecStart value should look as below:\n ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ --data-dir /var/lib/storageos-etcd \\ --quota-backend-bytes 8589934592 \\ --auto-compaction-retention 1000 \\ --auto-compaction-mode revision \\ --peer-auto-tls \\ --client-cert-auth --trusted-ca-file=/path/to/client-cert.pem \\ --cert-file=/path/to/ca.pem \\ --key-file=/path/to/client-key.pem \\ --initial-cluster-state new \\ --initial-cluster-token etcd-token \\ --listen-client-urls https://${NODE_IP}:${CLIENT_PORT} \\ --advertise-client-urls https://${NODE_IP}:${CLIENT_PORT} \\ --listen-peer-urls https://${NODE_IP}:${PEERS_PORT} \\ --initial-advertise-peer-urls https://${NODE_IP}:${PEERS_PORT} \\ --initial-cluster etcd-${NODE1_IP}=https://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=https://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=https://${NODE3_IP}:${PEERS_PORT} Reload and start the etc3 systemd service\n$ systemctl daemon-reload $ systemctl enable etcd3.service $ systemctl start etcd3.service Installation Verification\n The etcdctl binary is installed at /usr/local/bin on the nodes.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:2379 member list # $NODE_IP - the IP of the node 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Read the etcd operations page for our etcd recommendations.\n Installation - Ansible For a repeatable and automated installation, use of a configuration management tool such as ansible is recommended. StorageOS provides an ansible playbook to help you deploy etcd on standalone virtual machines.\n Clone StorageOS deployment repository\ngit clone https://github.com/storageos/deploy.git cd k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd Edit the inventory file\n The inventory file targets the nodes that will run etcd. The file hosts is an example of such an inventory file.\n $ cat hosts [nodes] centos-1 ip=\u0026#34;10.64.10.228\u0026#34; fqdn=\u0026#34;ip-10-64-10-228.eu-west-2.compute.internal\u0026#34; centos-2 ip=\u0026#34;10.64.14.233\u0026#34; fqdn=\u0026#34;ip-10-64-14-233.eu-west-2.compute.internal\u0026#34; centos-3 ip=\u0026#34;10.64.12.111\u0026#34; fqdn=\u0026#34;ip-10-64-12-111.eu-west-2.compute.internal\u0026#34; # Edit the inventory file $ vi hosts # Or your own inventory file Edit the etcd configuration\n If targeting Kubernetes Master nodes, you must change etcd_port_client, etcd_port_peers\n $ cat group_vars/all etcd_version: \u0026#34;3.4.9\u0026#34; etcd_port_client: \u0026#34;2379\u0026#34; etcd_port_peers: \u0026#34;2380\u0026#34; etcd_quota_bytes: 8589934592 # 8 GB etcd_auto_compaction_mode: \u0026#34;revision\u0026#34; etcd_auto_compaction_retention: \u0026#34;1000\u0026#34; members: \u0026#34;{{ groups[\u0026#39;nodes\u0026#39;] }}\u0026#34; installation_dir: \u0026#34;/var/lib/storageos-etcd\u0026#34; advertise_format: \u0026#39;fqdn\u0026#39; # fqdn || ip backup_file: \u0026#34;/tmp/backup.db\u0026#34; tls: enabled: false ca_common_name: \u0026#34;eu-west-2.compute.internal\u0026#34; etcd_common_name: \u0026#34;*.eu-west-2.compute.internal\u0026#34; cert_dir: \u0026#34;/etc/etcdtls\u0026#34; ca_cert_file: \u0026#34;etcd-ca.pem\u0026#34; etcd_server_cert_file: \u0026#34;server.pem\u0026#34; etcd_server_key_file: \u0026#34;server-key.pem\u0026#34; etcd_client_cert_file: \u0026#34;etcd-client.crt\u0026#34; etcd_client_key_file: \u0026#34;etcd-client.key\u0026#34; $ vi group_vars/all Choose between using IP addressing or FQDN in the advertise_format parameter. It allows you to decide how Etcd advertises its address to clients. This is particularly relevant when using TLS.\n If enabling TLS, it is recomended to generate your own CA certificate and key. You can do it by generating the CA from the machine running Ansible by: ansible-playbook create_ca.yaml.\n Install\nansible-playbook -i hosts install.yaml Installation Verification\n The playbook installs the etcdctl binary on the nodes, at /usr/local/bin.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 member list 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Benefits of Running External to Kubernetes Etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. In the case that quorum is lost, an etcd node stops and marks its contents as read-only. Another peer might have a newer version that has not been committed to the database. Quorum is fundamental for etcd operations.\nIn a Kubernetes environment, applications are scheduled across and in some scenarios such as \u0026ldquo;DiskPressure\u0026rdquo; they may need to be evicted from a node, and be scheduled onto a different node. With an application such as etcd, the scenario described can result in quorum being lost, making the cluster unable to recover automatically. Usually a 3 node etcd cluster can survive losing one node and recover. However, losing a second node at the same time or even having a network partition between them will result in quorum lost.\nBind Etcd IPs to Kubernetes Service Kubernetes external services use a DNS name to reference external endpoints, making them easy to reference from inside the cluster. You can use the example from the helper github repository to deploy the external Service. Using an external service can make monitoring of etcd from Prometheus easier.\n Using Etcd with StorageOS During installation of StorageOS the kvBackend.address parameter of the StorageOS operator is used to specify the address of the etcd cluster. See the StorageOS cluster operator configuration examples for more information.\n","excerpt":"StorageOS requires an etcd cluster in order to function. For more information on why etcd is …","ref":"/v2.1/docs/prerequisites/etcd/","title":"Etcd"},{"body":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. StorageOS implements dynamic provisioning, so the creation of a PVC will automatically provision a PV (PersistentVolume) that can be used to persist data written by a Pod.\nCreate the PersistentVolumeClaim You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic PVC definition\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;fast\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC will dynamically provision a 5GB volume using the fast StorageClass. This StorageClass was created during the StorageOS install and triggers creation of a PeristentVolume by StorageOS.\nFor installations with CSI, you can create multiple StorageClasses in order to specify default labels.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels. For simplicities sake this example will use unreplicated volumes.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# Reference to the StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1labels:storageos.com/replicas:\u0026#34;1\u0026#34;spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC has the storageos.com/replicas label set. This label tells StorageOS to add a replica for the volume that is created. For the sake of keeping this example simple an unreplicated volume will be used.\n Move into the examples folder and create a PVC using the PVC definition above.\n$ # from storageos-usecases/00-basic $ kubectl create -f ./pvc-basic.yaml You can view the PVC that you have created with the command below\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-1 Bound pvc-f8ffa027-e821-11e8-bc0b-0ac77ccc61fa 5Gi RWO fast 1m Create a pod that mounts the PVC created in step 2.\n$ kubectl create -f ./pod.yaml The command above creates a Pod that uses the PVC that was created in step 1.\napiVersion:v1kind:Podmetadata:name:d1spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mntname:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-1In the Pod definition above volume v1 references the PVC created in step 2, and is mounted in the pod at /mnt. In this example a debian image is used for the container but any container image with a shell would work for this example.\n Confirm that the pod is up and running\n$ kubectl get pods NAME READY STATUS RESTARTS AGE d1 1/1 Running 0 1m Execute a shell inside the container and write some contents to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo \u0026#34;Hello World!\u0026#34; \u0026gt; /mnt/helloworld root@d1:/# cat /mnt/helloworld Hello World! By writing to /mnt inside the container, the StorageOS volume created by the PVC is being written to. If you were to kill the pod and start it again on a new node, the helloworld file would still be avaliable.\nIf you wish to see more use cases with actual applications please see our Use Cases documentation.\n ","excerpt":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. …","ref":"/v2.1/docs/operations/firstpvc/","title":"StorageOS Volume Guide"},{"body":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available Commands: cluster Fetch cluster-wide configuration details diagnostics Fetch a cluster diagnostic bundle namespace Retrieve basic details of cluster namespaces node Retrieve basic details of nodes in the cluster user Fetch user details volume Retrieve basic details of volumes Flags: -h, --help help for get Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos get [command] --help\u0026#34; for more information about a command. get cluster $ storageos get cluster --help Fetch cluster-wide configuration details Usage: storageos get cluster [flags] Examples: $ storageos get cluster Flags: -h, --help help for cluster get diagnostics $ storageos get diagnostics --help Fetch a cluster diagnostic bundle Usage: storageos get diagnostics [flags] Examples: $ storageos get diagnostics $ storageos get diagnostics --output-file ~/my-diagnostics Flags: -h, --help help for diagnostics --output-file string writes the generated diagnostic bundle to a specified file path get namespace $ storageos get namespace -h Retrieve basic details of cluster namespaces Usage: storageos get namespace [namespace names...] [flags] Aliases: namespace, namespaces Examples: $ storageos get namespaces $ storageos get namespace my-namespace-name Flags: -h, --help help for namespace -l, --selector stringArray filter returned results by a set of comma-separated label selectors get node $ storageos get node -h Retrieve basic details of nodes in the cluster Usage: storageos get node [node names...] [flags] Aliases: node, nodes Examples: $ storageos get node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors get user $ storageos get user -h Fetch user details Usage: storageos get user [user names...] [flags] Aliases: user, users Examples: $ storageos get user my-username $ storageos get user my-username-1 my-username-2 $ storageos get user --use-ids my-userid $ storageos get user --use-ids my-userid-1 my-userid-2 Flags: -h, --help help for user get volume $ storageos get volume -h Retrieve basic details of volumes Usage: storageos get volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos get volumes --all-namespaces $ storageos get volume --namespace my-namespace-name my-volume-name Flags: --all-namespaces retrieves volumes from all accessible namespaces. This option overrides the namespace configuration -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors ","excerpt":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available …","ref":"/v2.1/docs/reference/cli/get/","title":"Get"},{"body":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any of the nodes in the cluster. Initially you can log in as the default administrator, with the username and password from the storageos-api Secret. By default storageos, storageos.\n You can access the GUI by either port-forwarding with kubectl or using an Ingress rule. i.e kubectl -n kube-system port-forward svc/storageos 5705.\n Nodes Volumes You can create volumes, including replicated volumes, and view volume details:\nLicensing Cluster info Namespaces Volumes can be namespaced across different projects or teams, and you can switch namespace using the left hand panel:\n","excerpt":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any …","ref":"/v2.1/docs/reference/gui/","title":"Graphical user interface (GUI)"},{"body":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the status of nodes in the cluster.\n$ storageos get nodes NAME HEALTH AGE LABELS node1 online 44 minutes ago node2 online 44 minutes ago node3 online 44 minutes ago node4 online 44 minutes ago node5 online 44 minutes ago ","excerpt":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the …","ref":"/v2.1/docs/operations/health/","title":"Cluster health"},{"body":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Use \u0026#34;storageos [command] --help\u0026#34; for more information about a command. ","excerpt":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to …","ref":"/v2.1/docs/reference/cli/help/","title":"Help"},{"body":"StorageOS uses the storage available on the nodes where it is installed to present as available for volumes.\nIn order to mitigate against problems caused by filling the host root disk, we recommend mounting a separate device into the /var/lib/storageos directory. StorageOS is agnostic to the type of filesystem mounted in /var/lib/storageos.\nExtending Available Storage StorageOS uses subdirectories of /var/lib/storageos/data to hold user data. By default, the directory /var/lib/storageos/data/dev1 will be created when a node is bootstrapped, and used for pool data. It is possible to shard the data by creating more directories into this structure. StorageOS will save data in any directory that conforms to the pattern /var/lib/storageos/data/dev[0-9]+, such as /var/lib/storageos/data/dev2 or /var/lib/storageos/data/dev5. This functionality enables operators to mount different devices into devX directories and StorageOS will recognise them as available storage automatically.\nThere are two possible options to expand the available disk space for StorageOS to allocate:\n Mount filesystem in /var/lib/storageos/data/devX Use LVM to expand the logical volume available to StorageOS Option 1: Mount Additional Devices This option enables operators to expand the cluster\u0026rsquo;s available space at any time without having to stop applications or forcing operational downtime. The expansion of disk is transparent for applications and StorageOS Volumes. StorageOS will use the new available space to create new data files.\n Context\nWe assume that there is a disk available in our Linux system without formatting in addition to the root filesystem. StorageOS data dir dev1 (/var/lib/storageos/data/dev1) is using /dev/xvda1. We will use the device /dev/xvdf to expand StorageOS available space.\nList available block devices in the host.\nroot@node0:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Format device\nroot@node0:/var/lib/storageos/data# mkfs -t ext4 /dev/xvdf mke2fs 1.42.12 (29-Aug-2014) Creating filesystem with 26214400 4k blocks and 6553600 inodes Filesystem UUID: 380712fa-6f82-477a-81a5-d7466d4c6b7f Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 4096000, 7962624, 11239424, 20480000, 23887872 Allocating group tables: done Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done Mount filesystem\nroot@node0:~# mkdir -p /var/lib/storageos/data/dev2 root@node0:~# mount /dev/xvdf /var/lib/storageos/data/dev2 Verify available storage\nIn less than 30 seconds, StorageOS will see the new available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 244,491,013,324 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Note that the node node0 has increased the TOTAL capacity in 100Gi.\n Persist the mount at boot by adding the mount endpoint to /etc/fstab\n Option 2: Expand Existing Devices Backed by LVM This option enables operators to take advantage of LVM to manage disks.\n Context\nWe assume that /var/lib/storageos is mounted onto an LVM volume. We are using a volumegroup named storageos and logical volume called data. There is a second physical disk /dev/xvdg unused.\nList available block devices in the host.\nroot@node2:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk `-storageos-data 254:0 0 99G 0 lvm /var/lib/storageos xvdg 202:96 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 107,696,304,947 # --\u0026gt; LVM storageos/data volume Add physical disk to LVM\nroot@node2:~# vgextend storageos /dev/xvdg Volume group \u0026quot;storageos\u0026quot; successfully extended The volume group storageos must have 2 physical volumes (#PV)\nroot@node2:~# vgs VG #PV #LV #SN Attr VSize VFree storageos 2 1 0 wz--n- 199.99g 104.99g Extend logical volume data\nroot@node2:~# lvextend -L+100G /dev/storageos/data Size of logical volume storageos/data changed from 95.00 GiB (24320 extents) to 195.00 GiB (49920 extents). Logical volume data successfully resized Resize the FileSystem\n Your filesystem must support the option to be expanded, and to do so while in use. Otherwise, you need to unmount first.\n root@node2:~# resize2fs /dev/storageos/data resize2fs 1.42.12 (29-Aug-2014) Filesystem at /dev/storageos/data is mounted on /var/lib/storageos; on-line resizing required old_desc_blocks = 6, new_desc_blocks = 13 The filesystem on /dev/storageos/data is now 51118080 (4k) blocks long. Check new available space\nThe mounted file system to /var/lib/storageos has increased its size.\nroot@node2:~# df -h /dev/mapper/storageos-data Filesystem Size Used Avail Use% Mounted on /dev/mapper/storageos-data 192G 60M 183G 1% /var/lib/storageos StorageOS available storage has increased too.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 206,158,430,208 # --\u0026gt; 100G more available Persist the mount at boot by adding the mount point to /etc/fstab\n ","excerpt":"StorageOS uses the storage available on the nodes where it is installed to present as available for …","ref":"/v2.1/docs/operations/managing-host-storage/","title":"Managing Host Storage"},{"body":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init container that sets the system configuration for StorageOS. The container also manages configuration changes required when upgrading StorageOS versions.\nThe container belongs to the DaemonSet that the StorageOS Cluster Operator starts when a StorageOSCluster resource is created. The storageos-init container is executed as an initContainer as part of a Kubernetes Pod. Therefore, only successful execution of the storageos-init container processes will result in the main container starting.\nScript Framework The code responsible for fulfilling the requirements is based on a Script Framework.\nThe script framework executes a set of scripts, performing checks, verifications and other procedures needed for StorageOS to be able to start. The scripts stdout and stderr are written to the stdout and stderr of the init app. The container logs contain all the logs of the individual scripts that run. The exit statuses of the scripts are used to determine initialization failure or success. Any non-zero exit status is logged as an event in the Kubernetes Pod events.\nIf any of the scripts fail, the storageos-init container will propagate the failure to Kubernetes, showing the status of the Pod as Init:Err.\nTo view the output of all storageos-init containers the following command can be used:\nkubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init For more details, check the StorageOS init container project.\nScripts executed The storageos-init container executes the following scripts.\n enable-lio dbupgrade ","excerpt":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init …","ref":"/v2.1/docs/reference/init-container/","title":"Init container"},{"body":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a StorageOS persistent volume being mounted on /var/jenkins_home. Deploying Jenkins using StorageOS offers multiple benefits. Firstly Jenkins can spin up multiple build pods at once to allow concurrent builds of different projects. Secondly Jenkins configuration is on a PersistentVolume so even if the Jenkins pod is rescheduled the configuration will persist.\nUsing StorageOS volume replicas allows for failure of nodes holding the PersistentVolume without interrupting Jenkins. Lastly by enabling StorageOS fencing Jenkins time to recover, in case of node failures, is greatly reduced.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information\nDeploying Jenkins on Kubernetes You can find the latest files in the StorageOS example deployment repository.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./jenkins Confirm that Jenkins is up and running\n$ kubectl get pods -w -l app=jenkins NAME READY STATUS RESTARTS AGE jenkins-0 1/1 Running 0 1m Connect to the Jenkins UI through the Jenkins service.\nYou can do this by port forwarding the Jenkins Kubernetes service to your localhost and accessing the UI via your browser. Alternatively if you have network access to your Kubernetes nodes then you can create a NodePort service and access Jenkins like that. A NodePort service has been left in 10-service.yaml commented out.\nTo port-foward the Jenkins service use the following command.\n$ kubectl port-foward svc/jenkins 8080 To login to the Jenkins UI use the credentials specified in 07-config.yaml, unless these have been changed from the defaults the username/password is admin/password.\n Create a Jenkins job.\nOnce you are logged into the UI you can create a job that will be farmed out to a Kubernetes plugin build agent. Click New Item, enter a name for the project and select Freestyle project. Next add an Execute shell build step. As a proof of concept you can use the bash below to have the pod execute a sleep.\n#!/bin/bash sleep 1000 Save the project and select Schedule a build of your project. You can watch for the appearance of a build pod using kubectl get pods -l jenkins=agent -w. Once the pod is created you should see the Build Executor status in the Jenkins UI display the pod.\nTo see multiple projects being built at once create another project and try scheduling a build of both projects at the same time.\n ","excerpt":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a …","ref":"/v2.1/docs/usecases/jenkins/","title":"Jenkins"},{"body":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub and traditional queues.\nUsing StorageOS persistent volumes with Apache Kafka means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the kafka pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nKafka has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Kafka to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Apache Zookeeper is required by Kafka to function; we assume it to already exist and be accessible within the Kubernetes cluster as zookeeper, see how to run Zookeeper with StorageOS here StorageOS is assumed to have been installed; please check for the latest available version here Kafka pods require 1536 MB of memory for successful scheduling Helm To simplify the deployment of kafka, we\u0026rsquo;ve used this Kafka helm chart (incubator) (version 0.13.8, app version 5.0.1) and rendered it into the example deployment files you can find in our GitHub repo.\nClone the use cases repo You can find the latest files in the StorageOS use cases repository in /kafka/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition\n---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:kafkalabels:app:kafka...spec:serviceName:kafka-headlesspodManagementPolicy:OrderedReadyupdateStrategy:type:OnDeletereplicas:3# \u0026lt;--- number of kafa pods to runtemplate:...spec:serviceAccountName:kafkacontainers:...- name:kafka-brokerimage:\u0026#34;confluentinc/cp-kafka:5.0.1\u0026#34;imagePullPolicy:\u0026#34;IfNotPresent\u0026#34;...volumeMounts:- name:datadirmountPath:\u0026#34;/var/data\u0026#34;volumes:- name:jmx-configconfigMap:name:kafka-metricsterminationGracePeriodSeconds:60volumeClaimTemplates:- metadata:name:datadirspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]resources:requests:storage:10Gi# \u0026lt;--- storage requested for each podstorageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Create the kubernetes objects\nkubectl apply -f ./kafka/ Confirm kafka is up and running\n$ kubectl get pods -l app=kafka NAME READY STATUS RESTARTS AGE kafka-0 2/2 Running 0 10m kafka-1 2/2 Running 0 9m26s kafka-2 2/2 Running 0 7m59s Connect to kafka\nConnect to the kafka test client pod and send some test data to kafka through its service endpoint\n Connect to the pod\nkubectl exec -it kafka-test-client /bin/bash Create a topic\n/usr/bin/kafka-topics --zookeeper zookeeper:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1 Send some test data\n/usr/bin/kafka-run-class org.apache.kafka.tools.ProducerPerformance --topic test-rep-one --num-records 5000 --record-size 100 --throughput -1 --print-metrics --producer-props acks=1 bootstrap.servers=kafka:9092 buffer.memory=67108864 batch.size=8196 ","excerpt":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from …","ref":"/v2.1/docs/usecases/kafka/","title":"Kafka"},{"body":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual machines (VMs) in Kubernetes pods.\nDeploying Kubevirt using StorageOS offers multiple benefits. Kubevirt can spin up VMs as Kubernetes pods, using images on StorageOS persistent volumes. Doing this allows the VM data to persist through restarts and rescheduling. Using StorageOS volume replicas also allows for failure of nodes holding the PersistentVolume without interrupting the VM running off the PersistentVolume. Containerized Data Importer (CDI) can also be used to prepare StorageOS volumes with disk images in an automated fashion. Simply by declaring that a VirtualMachine will use a DataVolume and providing the disk image URL, a StorageOS volume can be dynamically provisioned and automatically prepared with the disk image.\nThis usecase will guide you through installing KubeVirt and CDI on your Kubernetes cluster, and create a VM. By the end of the guide you\u0026rsquo;ll be able to launch a shell inside the KubeVirt VM that\u0026rsquo;s running as a Kubernetes pod.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Please ensure you have met the Kubevirt prerequisites, please see the Kubevirt installation instructions for more information.\nAs part of this installation it is assumed that you are running a Kubernetes cluster on VMs. As such nested virtualization or hardware emulation need to be enabled.\nDeploying KubeVirt on Kubernetes For ease of installation we have enabled hardware emulation. If your VMs support nested virtualization then edit the Kubevirt ConfigMap ./kubevirt-install/10-cm.yaml, removing the line debug.useEmulation: \u0026quot;true\u0026quot;.\n In order to deploy Kubevirt you just need to clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/kubevirt $ kubectl create -f ./kubevirt-install Check that the Kubevirt pods are running.\n$ kubectl get pods -w -n kubevirt NAME READY STATUS RESTARTS AGE virt-api-57546d479b-p26d4 1/1 Running 0 1m virt-api-57546d479b-zs5dw 1/1 Running 0 1m virt-controller-56b5498854-7xsfz 1/1 Running 1 1m virt-controller-56b5498854-bz559 1/1 Running 1 1m virt-handler-6z4kq 1/1 Running 0 1m virt-handler-7szhl 1/1 Running 0 1m virt-handler-jmm6w 1/1 Running 0 1m virt-operator-79c9bdd859-8xq98 1/1 Running 0 1m virt-operator-79c9bdd859-kfjz6 1/1 Running 0 1m Once Kubevirt is running install CDI.\n$ kubectl create -f ./cdi Check that the CDI pods are running correctly.\n$ kubectl get pods -n cdi NAME READY STATUS RESTARTS AGE cdi-apiserver-8668f888df-s6pp4 1/1 Running 0 1m cdi-deployment-5cf794896b-whh4j 1/1 Running 0 1m cdi-operator-5887f96c-dz2hg 1/1 Running 0 1m cdi-uploadproxy-97fbbfcbf-6f9xs 1/1 Running 0 1m Now that CDI and Kubevirt are running, VMs can be created. In this example VMs running Cirros, a small and lightweight OS, will be created. The vm-cirros.yaml manifest creates a VirtualMachine that uses a DataVolume. This means that CDI will create a StorageOS backed PVC and download the image that the VirtualMachineInstance (VMI) will boot from onto the PVC.\n$ kubectl create -f ./vm-cirros.yaml Check that the VMI is running. Note that the VMI will only be created after CDI has downloaded the Cirros disk image onto a StorageOS persistent volume so depending on your connection speed this may take some time.\n$ kubectl get vmi NAME AGE PHASE IP NODENAME cirros 1m Running 10.244.2.12 ip-10-1-10-154.storageos.net $ kubectl get pods NAME READY STATUS RESTARTS AGE virt-launcher-cirros-drqhr 1/1 Running 0 1m Connect to the VM console.\nThis example uses the virtctl kubectl plugin in order to connect to the VMs console. The escape sequence ^] is ctrl + ]\n$ kubectl virt console cirros Successfully connected to cirros console. The escape sequence is ^] login as \u0026#39;cirros\u0026#39; user. default password: \u0026#39;gocubsgo\u0026#39;. use \u0026#39;sudo\u0026#39; for root. cirros login: cirros Password: $ Cloning Volumes CDI allows for images to be cloned using a DataVolume manifest. Verify that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n N.B. Ensure that the VMI is stopped before continuing!\n Verify that the VMI is stopped before continuing, and that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE cirros Bound pvc-f4833060-5a77-420c-927e-6bc518d9df3c 12Gi RWO fast 1m Once the PVC\u0026rsquo;s existence is confirmed then create a new DataVolume that uses the cirros PVC as its source.\n$ kubectl create -f ./cloned.yaml Watch as the CDI pods are created.\n$ kubectl get pods -w You\u0026rsquo;ll see that a cdi-upload-cloned-datavolume pod is created and then a cdi-clone-source pod is created. The cdi-source pod mounts the original cirros volume and sends the contents of the volume to the cdi-upload pod. The cdi-upload pod creates and mounts a new PVC and writes the contents of the original volume to it.\n ","excerpt":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual …","ref":"/v2.1/docs/usecases/kubevirt/","title":"Kubevirt"},{"body":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to various StorageOS artefacts. Applying specific feature labels triggers compression, replication and other storage features. No feature labels are present by default.\nStorageOS Node labels Nodes do not have any feature labels present by default. When StorageOS is run within Kubernetes, the Cluster Operator syncs any node labels set on Kubernetes nodes within StorageOS. Node labels may also be set with the CLI or UI.\n Feature Label Values Description Compute only storageos.com/computeonly true / false Specifies whether a node should be computeonly where it only acts as a client and does not host volume data locally, otherwise the node is hyperconverged (the default), where the node can operate in both client and server modes. You can set the computeonly label on the Kubernetes node and the label will be sync\u0026rsquo;d to the StorageOS node (labels take an eventual consistency reconciliation time of ~1min).\nkubectl label node $NODE storageos.com/computeonly=true StorageOS Volume labels Volumes do not have any feature labels present by default. WARNING: The caching and compression labels can only apply during provision time, they can\u0026rsquo;t be changed during execution.\n Feature Label Values Description Caching storageos.com/nocache true / false Switches off caching. Compression storageos.com/nocompress true / false Switches off compression of data at rest and in transit. Replication storageos.com/replicas integers [0, 5] Replicates entire volume across nodes. Typically 1 replica is sufficient (2 copies of the data); more than 2 replicas is not recommended. Once applied, this setting can only be changed in the StorageOS UI or CLI. To create a volume with a feature label:\n Option 1: PVC Label\nAdd the label in the PVC definition, for instance:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc-3labels:storageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;-----spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:1G Option 2: Set label in the StorageClass\nAny PVC using the StorageClass inherits the label. The PVC label takes precedence over the StorageClass parameters.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;--------provisioner:storageos# CSI driver (recommended) Option 3:\nOnce a PVC is created, you can update the Labels in StorageOS both in the UI or CLI. Those labels are going to be visible only for StorageOS and will not be synced to the Kubernetes resource.\n ","excerpt":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to …","ref":"/v2.1/docs/reference/labels/","title":"StorageOS Feature labels"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying a Free Developer licence.\nTo learn how to apply a licence to your cluster, check the operations licensing page.\nFree Developer licence The developer licence only requires the user to register and issue a licence through StorageOS.\nThe developer licence is free and grants a licence for a 5 node StorageOS cluster with 5TB of provisioned capacity. It is designed to enable basic cloud native workflows in Kubernetes that require the persistence of stateful application data. Dynamic provisioning, distributed access to data and high availability of volumes through synchronous replication and automatic failover are some of the features that are available under the developer licence.\nThe free developer licence includes community support through our public Slack channel..\nCommercial licences For information on our commercial offerings, including support, please contact sales@storageos.com.\nNote about capacity limits Some StorageOS licences have limits on capacity. StorageOS allows provisioning volumes until the limit of the licence is reached. Only the size of the volume requested by the Persistent Volume Claim counts for the licence limit, regardless whether of whether the volume has replication enabled.\nOnce the licence limit is reached, new volumes are not able to provision unless provisioned capacity is released, i.e deleting volumes. That behaviour is not tight to the capacity of the backend disks on your nodes.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2.1/docs/reference/licence/","title":"Licence"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying for a Free Developer licence.\nObtaining a Developer licence via the GUI You will need access to the StorageOS GUI on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 As an alternative, an Ingress controller may be preferred.\nOnce you have obtained access to the GUI, login using whatever credentials you used to create the cluster.\nYou can obtain and apply a free Developer licence in the StorageOS web GUI by creating or logging in with a StorageOS account on the StorageOS portal via the licence page of the GUI:\nWait a few seconds for the licence generation process to complete, at which point your licence will be visible.\nApplying a previously obtained licence via the GUI Commercial licences are delivered through contact with the StorageOS team. To apply such keys, via the web GUI, visit the licence section of the GUI and click on the tab \u0026ldquo;Upgrade\u0026rdquo;, for the specific licence level you purchased. Then paste the licence key and click on \u0026ldquo;UPLOAD KEY TO CLUSTER\u0026rdquo;. Note that you can also view your cluster ID on the same page.\nApplying a licence via the CLI Before getting a licence, you need to know the ID of your StorageOS cluster.\nThis CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence. Once, given the key, you can apply the licence by using the following command.\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos apply licence --from-stdin Read the licence CLI command reference for further information.\nObtaining an Enterprise licence Please contact sales@storageos.com to discuss pricing for commercial licences.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2.1/docs/operations/licensing/","title":"Licensing"},{"body":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on linux.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MS SQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1...spec:serviceAccountName:mssql...volumeMounts:- name:datamountPath:/var/opt/mssql...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MS SQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mssql Confirm MS SQL is up and running.\n$ kubectl get pods -w -l app=mssql NAME READY STATUS RESTARTS AGE mssql-0 1/1 Running 0 1m Connect to the MS SQL client pod and connect to the MS SQL server through the service\n$ kubectl exec -it mssql-0 -- /opt/mssql-tools/bin/sqlcmd -S mssql-0.mssql -U SA -P \u0026#39;Password15\u0026#39; 1\u0026gt; USE master; 2\u0026gt; GO Changed database context to \u0026#39;master\u0026#39;. 1\u0026gt; SELECT name, database_id, create_date FROM sys.databases ; 2\u0026gt; GO name database_id create_date --------------------------- ----------- ----------------------- master 1 2003-04-08 09:13:36.390 tempdb 2 2018-11-02 16:30:37.907 model 3 2003-04-08 09:13:36.390 msdb 4 2018-10-19 01:18:57.300 (4 rows affected) Configuration If you need custom startup options, you can edit the ConfigMap file 15-mssql-configmap.yaml with your desired MS SQL configuration settings.\n","excerpt":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on …","ref":"/v2.1/docs/usecases/mssql/","title":"MS SQL"},{"body":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With mTLS both StorageOS and etcd authenticate each other ensuring that communication only happens between mutually authenticated end points, and that all communication is encrypted.\nStorageOS uses the certificates and keys from a Secret to cypher and authenticate Etcd traffic.\nHow to create the certificates Secret The client auth certificates need the following file names, in the Secret.\n etcd-client-ca.crt - containing the etcd Certificate Authority certificate etcd-client.crt - containing the etcd Client certificate etcd-client.key - cotaining the etcd Client key kubectl create secret -n storageos-etcd generic \\ etcd-client-tls \\ --from-file=\u0026#34;etcd-client-ca.crt\u0026#34; \\ --from-file=\u0026#34;etcd-client.crt\u0026#34; \\ --from-file=\u0026#34;etcd-client.key\u0026#34; How to use the mTLS certificates Secret with StorageOS Below is an example StorageOSCluster resource that can be used to setup StorageOS with etcd using mTLS.\napiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:storageos-clusternamespace:\u0026#34;storageos-operator\u0026#34;spec:images:nodeContainer:\u0026#34;storageos/node:v2.1.0\u0026#34;secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;default\u0026#34;namespace:\u0026#34;storageos\u0026#34;csi:enable:truedeploymentStrategy:\u0026#34;deployment\u0026#34;enableControllerPublishCreds:trueenableNodePublishCreds:trueenableProvisionCreds:trueenableControllerExpandCreds:true# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;etcd\u0026#34;# Namespace of the client certificates secretkvBackend:address:\u0026#34;https://storageos-etcd-cluster-client.storagos-etcd.svc:2379\u0026#34;# Etcd client service address.backend:\u0026#34;etcd\u0026#34;# Backend typetlsEtcdSecretRefName and tlsEtcdSecretRefNamespace are used to pass a reference to the Secret.\nThe StorageOS operator uses the etcd secret that contains the client certificates, to build a secret in the StorageOS installation namespace. This secret contains the certificate filenames and certificate file contents. The StorageOS daemonset that is created by the operator mounts the secret as a volume so that the certificate files are available inside the pod. Environment variables containing the file paths are passed to the StorageOS process in order to use the files from the mounted path.\nA worked example of setting up StorageOS with external etcd using mTLS is avaliable here. For ease of use the example uses the CoreOS etcd operator and the CoreOS guide The example uses the CoreOS etcd operator and follows the CoreOS guide for Cluster TLS.\n","excerpt":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With …","ref":"/v2.1/docs/operations/etcd/storageos-secret-info/","title":"Encrypting communication with Etcd"},{"body":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular web-based applications including WordPress.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MySQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiversion:apps/v1kind:statefulsetmetadata:name:mysqlspec:selector:matchlabels:app:mysqlenv:prodservicename:mysqlreplicas:1...spec:serviceaccountname:mysql...volumemounts:- name:datamountpath:/var/lib/mysqlsubpath:mysql- name:confmountpath:/etc/mysql/mysql.conf.d...volumeclaimtemplates:- metadata:name:datalabels:env:prodspec:accessmodes:[\u0026#34;readwriteonce\u0026#34;]storageclassname:\u0026#34;fast\u0026#34;# storageos storageclass resources:requests:storage:5giThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MySQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mysql Confirm MySQL is up and running.\n$ kubectl get pods -w -l app=mysql NAME READY STATUS RESTARTS AGE mysql-0 1/1 Running 0 1m Connect to the MySQL client pod and connect to the MySQL server through the service\n$ kubectl exec client -- mysql -h mysql-0.mysql -e \u0026#34;show databases;\u0026#34; Database information_schema mysql performance_schema Configuration If you need custom startup options, you can edit the ConfigMap file 15-mysqld-configmap.yaml with your desired MySQL configuration settings.\n","excerpt":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular …","ref":"/v2.1/docs/usecases/mysql/","title":"MySQL"},{"body":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a StorageOS cluster to be used by multiple teams across multiple projects.\nIt is not necessary to create StorageOS namespaces manually, as StorageOS maps Kubernetes namespaces on a one-to-one basis when PersistentVolumeClaims using the StorageOS StorageClass are created.\nAccess to Namespaces is controlled through user or group level policies.\n","excerpt":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a …","ref":"/v2.1/docs/concepts/namespaces/","title":"Namespaces"},{"body":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is created by default.\nNamespaces apply to volumes.\nManaging Namespaces In order to create a new namespace navigate to \u0026ldquo;Namespaces\u0026rdquo; in the GUI, and select \u0026ldquo;Create Namespace\u0026rdquo;.\nWhen a Kubernetes PVC is created in a namespace, StorageOS automatically maps the Volume in the same namespace. Namespaces are created by StorageOS to fulfil the RBAC rules enforced by Kubernetes roles.\nIn order to delete a namespace, all volumes must be deleted from the namespace before the namespace can be deleted.\n","excerpt":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is …","ref":"/v2.1/docs/operations/namespaces/","title":"Namespaces"},{"body":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load balancer or even as a Kubernetes ingress controller.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Nginx on Kubernetes You can find the latest files in the StorageOS use cases repostiory\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet definition\napiVersion:apps/v1kind:StatefulSetmetadata:name:nginxspec:serviceName:nginxspec:serviceAccountName:nginxcontainers:- name:nginximage:nginxports:- containerPort:80volumeMounts:- name:nginx-datamountPath:/usr/share/nginx/htmlsubPath:html- name:nginx-configmountPath:/etc/nginx/conf.dvolumes:- name:nginx-configconfigMap:name:nginxvolumeClaimTemplates:- metadata:name:nginx-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Nginx examples folder and create the objects\ncd storageos-usecases kubectl create -f ./nginx Confirm Nginx is up and running.\n$ kubectl get pods -w -l app=nginx NAME READY STATUS RESTARTS AGE nginx-0 1/1 Running 0 1m Connect to the nginx pod and write a file to /usr/share/nginx/html that Nginx will serve.\n$ kubectl exec nginx-0 -it -- bash root@nginx-0:/# echo Hello world! \u0026gt; /usr/share/nginx/html/greetings.txt Connect to the Busybox pod and connect to the Nginx server through the service and retrieve the directory index from Nginx.\n$ kubectl exec -it busybox -- /bin/sh / # wget -q -O- nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;greetings.txt\u0026#34;\u0026gt;greetings.txt\u0026lt;/a\u0026gt; 27-Feb-2019 12:04 13 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; Retrieve and display the contents of the greetings.txt file\n/ # wget -q -O- nginx/greetings.txt Hello world! Configuration If you need custom startup options, you can edit the ConfigMap file 15-nginx-configmap.yaml with your desired Nginx configuration settings.\n","excerpt":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load …","ref":"/v2.1/docs/usecases/nginx/","title":"Nginx"},{"body":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A node must be running a daemonset pod in order to consume and/or present storage.\nBy default StorageOS nodes run in hyperconverged mode. This means that the node hosts data from StorageOS volumes and can present volumes to applications.\nAlternatively, a node can run in computeonly mode, which means no storage is consumed on the node itself and the node only presents volumes hosted by other nodes. Volumes presented to applications running on compute only nodes are therefore all remote. Compute only nodes can be very useful for topologies where nodes are ephemeral and should not host data, but the ephemeral nodes host applications that require StorageOS volumes. The nodes that are not intended to hold data, but just to present StorageOS volumes, can be set as computeonly.\nA node can be marked as compute only at any point in time by adding the label storageos.com/computeonly=true, following the labels reference.\n","excerpt":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A …","ref":"/v2.1/docs/concepts/nodes/","title":"Nodes"},{"body":"The following document lists the open source software attributions in the StorageOS Control Plane, Data Plane and CLI.\nCNats The MIT License (MIT) Copyright (c) 2015 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CppUTest Copyright (c) 2007, Michael Feathers, James Grenning and Bas Vodde All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the \u0026lt;organization\u0026gt; nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE EARLIER MENTIONED AUTHORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL \u0026lt;copyright holder\u0026gt; BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Hayai Copyright (c) 2011 - Nick Bruun. This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. If you meet (any of) the author(s), you're encouraged to buy them a beer, a drink or whatever is suited to the situation, given that you like the software. This notice may not be removed or altered from any source distribution. LZ4 LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LibreSSL LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- ==================================================================== Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit. (http://www.openssl.org/)\u0026quot; 4. The names \u0026quot;OpenSSL Toolkit\u0026quot; and \u0026quot;OpenSSL Project\u0026quot; must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact openssl-core@openssl.org. 5. Products derived from this software may not be called \u0026quot;OpenSSL\u0026quot; nor may \u0026quot;OpenSSL\u0026quot; appear in their names without prior written permission of the OpenSSL Project. 6. Redistributions of any form whatsoever must retain the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)\u0026quot; THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==================================================================== This product includes cryptographic software written by Eric Young (eay@cryptsoft.com). This product includes software written by Tim Hudson (tjh@cryptsoft.com). Original SSLeay License ----------------------- Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) All rights reserved. This package is an SSL implementation written by Eric Young (eay@cryptsoft.com). The implementation was written so as to conform with Netscapes SSL. This library is free for commercial and non-commercial use as long as the following conditions are aheared to. The following conditions apply to all code found in this distribution, be it the RC4, RSA, lhash, DES, etc., code; not just the SSL code. The SSL documentation included with this distribution is covered by the same copyright terms except that the holder is Tim Hudson (tjh@cryptsoft.com). Copyright remains Eric Young's, and as such any Copyright notices in the code are not to be removed. If this package is used in a product, Eric Young should be given attribution as the author of the parts of the library used. This can be in the form of a textual message at program startup or in documentation (online or textual) provided with the package. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: \u0026quot;This product includes cryptographic software written by Eric Young (eay@cryptsoft.com)\u0026quot; The word 'cryptographic' can be left out if the rouines from the library being used are not cryptographic related :-). 4. If you include any Windows specific code (or a derivative thereof) from the apps directory (application code) you must include an acknowledgement: \u0026quot;This product includes software written by Tim Hudson (tjh@cryptsoft.com)\u0026quot; THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The licence and distribution terms for any publically available version or derivative of this code cannot be changed. i.e. this code cannot simply be copied and put under another distribution licence [including the GNU Public Licence.] protobuf This license applies to all parts of Protocol Buffers except the following: - Atomicops support for generic gcc, located in src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Code generated by the Protocol Buffer compiler is owned by the owner of the input file used when generating it. This code is not standalone and requires a support library to be linked with it. This support library is itself covered by the above license. rocksdb BSD License For rocksdb software Copyright (c) 2011-present, Facebook, Inc. All rights reserved. --------------------------------------------------------------------- Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/armon/go-metrics The MIT License (MIT) Copyright (c) 2013 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/armon/go-radix The MIT License (MIT) Copyright (c) 2014 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/bgentry/speakeasy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \u0026quot;[]\u0026quot; replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \u0026quot;printed page\u0026quot; as the copyright notice for easier identification within third-party archives. Copyright [2013] [the CloudFoundry Authors] Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/dgrijalva/jwt-go Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgrijalva/jwt-go/request Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgryski/go-metro MetroHash This package is a mechanical translation of the reference C++ code for MetroHash, available at https://github.com/jandrewrogers/MetroHash I claim no additional copyright over the original implementation. The MIT License (MIT) Copyright (c) 2015 J. Andrew Rogers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/docker/docker/pkg/mount Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv/store Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/golang/protobuf/proto Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/context Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/handlers Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/mux Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/websocket Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/consul/api Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/errwrap Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-cleanhttp Mozilla Public License, version 2.0 1. Definitions 1.1. \u0026quot;Contributor\u0026quot; means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. \u0026quot;Contributor Version\u0026quot; means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. \u0026quot;Contribution\u0026quot; means Covered Software of a particular Contributor. 1.4. \u0026quot;Covered Software\u0026quot; means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. \u0026quot;Incompatible With Secondary Licenses\u0026quot; means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. \u0026quot;Executable Form\u0026quot; means any form of the work other than Source Code Form. 1.7. \u0026quot;Larger Work\u0026quot; means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. \u0026quot;License\u0026quot; means this document. 1.9. \u0026quot;Licensable\u0026quot; means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. \u0026quot;Modifications\u0026quot; means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. \u0026quot;Patent Claims\u0026quot; of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. \u0026quot;Secondary License\u0026quot; means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. \u0026quot;Source Code Form\u0026quot; means the form of the work preferred for making modifications. 1.14. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) means an individual or a legal entity exercising rights under this License. For legal entities, \u0026quot;You\u0026quot; includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, \u0026quot;control\u0026quot; means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an \u0026quot;as is\u0026quot; basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - \u0026quot;Incompatible With Secondary Licenses\u0026quot; Notice This Source Code Form is \u0026quot;Incompatible With Secondary Licenses\u0026quot;, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-msgpack/codec Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/go-multierror Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/memberlist Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/coordinate Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/serf Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/kelseyhightower/envconfig Copyright (c) 2013 Kelsey Hightower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/mattn/go-isatty Copyright (c) Yasuhiro MATSUMOTO \u0026lt;mattn.jp@gmail.com\u0026gt; MIT License (Expat) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/miekg/dns Extensions of the original work are copyright (c) 2011 Miek Gieben As this is fork of the official Go code the same license applies: Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/mitchellh/cli Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/conf The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/logger The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server/pse The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/util The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats/encoders/builtin The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nuid The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/pborman/uuid Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/Sirupsen/logrus The MIT License (MIT) Copyright (c) 2014 Simon Eskildsen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/thejerf/suture Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/urfave/negroni The MIT License (MIT) Copyright (c) 2014 Jeremy Saenz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang.org/x/crypto/bcrypt Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/crypto/blowfish Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/net/context Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/unix Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/registry Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/svc/eventlog Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/time/rate Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. gopkg.in/yaml.v2 Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ","excerpt":"The following document lists the open source software attributions in the StorageOS Control Plane, …","ref":"/v2.1/docs/reference/open_source_attribution/","title":"StorageOS Open Source Software Attribution Notice"},{"body":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform actions such as creation, deletion and mounting of volumes through CSI. The CSI container running in the StorageOS Daemonset creates a Linux socket that allows the communication between OpenShift and StorageOS\nCSI (Container Storage Interface) Note CSI is the standard that enables storage drivers to release on their own schedule. This allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nStorageOS v2 implements communication with the OpenShift controlplane with CSI only. The native driver has been deprecated\nStorageOS PersistentVolumeClaims The user can provide standard PVC definitions and StorageOS will dynamically provision them. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nInstallation StorageOS v2 supports OpenShift 4.0, 4.1, 4.2 and 4.3.\nTo install StorageOS on OpenShift, please follow our installation instructions page.\n","excerpt":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform …","ref":"/v2.1/docs/platforms/openshift/","title":"OpenShift"},{"body":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a user or group access to a namespace, a policy needs to be created mapping the user or group to the namespace.\n Note: Users always have access to the default namespace\n For more information on how to use policies, see the Policies operations page.\n","excerpt":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a …","ref":"/v2.1/docs/concepts/policies/","title":"Policies"},{"body":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user level so access can be controlled granularly.\nUsers can belong to one or more groups to control their namespace permissions. Additionally user specific policies can be created to grant a user access to a namespace. Users can belong to any number of groups and have any number of user level policies configured.\n Note: Users are created with access to the default namespace. Policies cannot be applied to the default namespace.\n Managing Policies To start creating policies, at least one custom namespace and user are required. For more information on how to create namespaces see our Namespace guide, for users see our Users CLI reference.\nIn order to create a policy navigate to \u0026ldquo;Policies\u0026rdquo; in the GUI and select \u0026ldquo;Create Policy\u0026rdquo;. A policy controls access to a variety of StorageOS resources and is applied to a user, by placing the user in the policies group.\nIn order to delete a policy, all users must be removed from the policy group before deletion of the policy can be completed.\n","excerpt":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user …","ref":"/v2.1/docs/operations/policies/","title":"Policies"},{"body":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational database management system (ORDBMS).\nPostgres is deployed across a wide variety of platforms with a mix of workloads ranging from small, single-node use cases to large Internet-facing clusters with many concurrent users.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. If you need to setup StorageOS on Kubernetes then please see our guide to [installing StorageOS on Kubernetes]({% link \u0026ldquo;docs/install/kubernetes.md\u0026rdquo; \u0026gt;}}).\nDeploying PostgreSQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases PersistentVolumeClaim and Pod definition excerpts\nkind:PersistentVolumeClaimmetadata:name:pg-dataannotations:volume.beta.kubernetes.io/storage-class:fast...kind:Podmetadata:name:postgressspec:securityContext:fsGroup:26containers:- name:pgimage:crunchydata/crunchy-postgres:centos7-10.4-1.8.3volumeMounts:- mountPath:/pgdataname:data...volumes:- name:datapersistentVolumeClaim:claimName:pg-dataThis excerpt is from the PersistentVolumeClaim and Pod definition. The pod definition references the pg-data VolumeClaim so storage is dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the PostgreSQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./postgres Confirm PostgreSQL is up and running.\n$ kubectl get pod postgres-0 -w NAME READY STATUS RESTARTS AGE postgres-0 1/1 Running 0 1m Connect to the PostgreSQL client pod and connect to the PostgreSQL server through the service.\n$ kubectl exec -it postgres-0 -- psql -h postgres-0.postgres -U primaryuser postgres -c \u0026#34;\\l\u0026#34; Password for user primaryuser: password List of Databases Name | Owner | Encoding | Collate | Ctype | Access privileges +=========================================================================+ postgres | postgres | SQL_ASCII | C | C | template0 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres template1 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres userdb | postgres | SQL_ASCII | C | C | =Tc/postgres + | | | | | postgres=CTc/postgres+ | | | | | testuser=CTc/postgres (4 rows) The password for the primary user is password. You can see this is set in the ConfigMap file.\n Configuration If you need custom startup options, you can edit the ConfigMap file 15-postgresd-configmap.yaml with your desired PostgreSQL configuration settings.\n","excerpt":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational …","ref":"/v2.1/docs/usecases/postgres/","title":"PostgreSQL"},{"body":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting in Kubernetes.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Prometheus on Kubernetes This is the Prometheus use case for StorageOS. Following are the steps for creating a Prometheus StatefulSet and using StorageOS to provide persistent storage.\n You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases Prometheus Custom Resource definition\napiVersion:monitoring.coreos.com/v1kind:Prometheusmetadata:name:prometheus-storageoslabels:app:prometheus-operatorspec:...storage:volumeClaimTemplate:metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:storageos-replicatedresources:requests:storage:1GiThis excerpt is from the Prometheus Custom Resource definition. This file contains the VolumeClaimTemplate that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs due to the VolumeClaimTemplate in the Prometheus StatefulSet. The Prometheus StatefulSet is created by the Prometheus Operator, triggered by the creation of the Prometheus resource.\n Move into the Prometheus examples folder and install the Prometheus Operator and create a Prometheus resource.\n$ cd storageos-usecases/prometheus $ ./install-prometheus.sh Confirm Prometheus is up and running.\n$ kubectl get pods -w -l app=prometheus NAME READY STATUS RESTARTS AGE prometheus-prometheus-storageos-0 3/3 READY 0 1m You can see the created PVC using.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-prometheus-prometheus-storageos-0 Bound pvc-b6c17c0a-e76b-4a0b-8fc6-46c0e1629210 1Gi RWO storageos-replicated 65m In the Prometheus deployment script, a service monitor is created. A Service Monitor is a special object that the Prometheus operator uses to create configuration for Endpoints for Prometheus to scrape. Although the name implies that a Service Monitor defines Kubernetes services that will be scraped, Prometheus actually targets the Endpoints that the services point to. The new Prometheus instance will use the storageos-etcd service monitor to start scraping metrics from the storageos-etcd pods. Assuming the storageos cluster was setup using ETCD as pods. For more information about service monitors, have a look at the upstream documentation.\n$ kubectl get servicemonitor NAME AGE storageos-etcd 5d1h The Prometheus web ui can be accessed by port-forwarding the Prometheus pods port to localhost.\n$ kubectl port-forward prometheus-prometheus-storageos-0 9090 Then launch a web browser and go to localhost:9090 to access the Prometheus web ui. You can confirm that the storageos-etcd target is configured there.\n Configuration In the storageos-usecases/prometheus/manifests/prometheus directory there are other example Service Monitors. For more information about Prometheus, check out the prometheus documentation.\n","excerpt":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting …","ref":"/v2.1/docs/usecases/prometheus/","title":"Prometheus"},{"body":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS transparently supports Rancher deployments using CentOS, RHEL or Debian and can support other Linux distributions detailed in the systems supported page if the appropriate kernel modules are present.\n","excerpt":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS …","ref":"/v2.1/docs/platforms/rancher/","title":"Rancher"},{"body":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional durability to disk.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Redis on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:redisspec:selector:matchLabels:app:redisenv:prodserviceName:redisreplicas:1...spec:serviceAccountName:redis...volumeMounts:- name:datamountPath:/bitnami/redis/data...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the Redis examples folder and create the objects\ncd storageos-usecases kubectl create -f ./redis Confirm Redis is up and running.\n$ kubectl get pods -w -l app=redis NAME READY STATUS RESTARTS AGE redis-0 1/1 Running 0 1m Connect to the Redis client pod and connect to the Redis server through the service\n$ kubectl exec -it redis-0 -- redis-cli -a password Warning: Using a password with \u0026#39;-a\u0026#39; option on the command line interface may not be safe. 127.0.0.1:6379\u0026gt; CONFIG GET maxmemory 1) \u0026#34;maxmemory\u0026#34; 2) \u0026#34;0\u0026#34; Configuration If you need custom startup options, you can edit the ConfigMap file 15-redis-configmap.yaml with your desired Redis configuration settings.\n","excerpt":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional …","ref":"/v2.1/docs/usecases/redis/","title":"Redis"},{"body":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than \u0026ldquo;latest\u0026rdquo;, and to perform upgrades only after reading the release notes.\nThe latest tagged release is v2.1.0. For installation instructions please see our Install page.\nThe latest CLI release is v2.1.0, available from Github\nUpgrading To upgrade from version 1.x to 2.x, please contact support for assistance.\nv2.2.0 - Released 2020-08-18 This release focuses on performance. We analysed existing performance characteristics across a variety of real-world use cases and ended up with improvements across the board. Of particular note:\n Sequential reads have improved by up to 130% Sequential writes have improved by up to 737% Random reads have improved by up to 45% Random writes have improved by up to 135% I/O for large block sizes (128K) has improved by up to 353% We are extremely proud of our performance and we love to talk about it. Have a look at the Benchmarking section of the self-evaluation guide and consider sharing your results. Our PRE engineers are available to discuss in our slack channel.\nNew Data engine revamp focused on provable consistency and performance. Key characteristics:\n Metadata is stored in an optimised index, lowering I/O latency and improving performance for all workloads. Large block reads/writes are now be handled in a single operation. Applications like Kafka will go much faster. On-disk compression is now disabled by default as in most scenarios this offers better performance. To enable on-disk compression for a specific workload, see compression.\n Improved dataplane: The number of I/O threads are now determined by the number of processing cores available. This improves scalability and performance on larger servers. ha: Improve partition tolerance behaviour when a volume master that has lost its connection to etcd rejoins. ha: Allow replicas in unhealthy states to be remediated and re-used while maintaining partition tolerance. ha: When a master fails and the new master is not yet available, introduce a back-off to the redirection logic to avoid spamming the logs with connection failure errors. ha: Ignore health advertisements for local node. Local nodes are handled directly. node delete: Only refuse to delete a node if the node health can be authoritatively verified to be in use. api: Increase HTTP server write timeout. cli/ui: Allow partial diagnostic bundle downloads. ui: Namespace dropdown can now be scrolled. ui: Add \u0026ldquo;Job title\u0026rdquo; to UI licence form. logging: Log version at startup at INFO level.authoritatively logging: Lower verbosity of SCSI warnings that do not apply to StorageOS. diagnostics: Include logs that have been rotated. diagnostics: Bundle collection across providers is now done in parallel. build: Update base image to RHEL 8.2. operator: Removed DB migration utility required for v1.3 -\u0026gt; v1.4 upgrades. operator: Automatically refreshes StorageOS API token without failing requests when the token expires. operator: Updated CSI attacher and provisioner to latest upstream. operator: Remove cluster.local suffix on Pod scheduler service address. This allows the scheduler to work in clusters with custom DNS configuration. operator: Defaults are now set for most CSI configuration options in the StorageOSCluster custom resource. Fixed csi: When unmount request is received for a volume that has already been unmounted, return success. csi: Verify volume is attached on the node before mounting it. xfs: Support older RHEL kernels which have an XFS library that does not allow reflinks/dedupe. dataplane: Reserve 1GiB of capacity on the target disk to allow manual cleanup operations, rather than filling target disk to capacity. operator: In some cases /var/lib/storageos could fail to unmount cleanly after a restart. This resulted in multiple entries in /proc/mounts. v2.1.0 - Released 2020-06-26 New csi: Volume expansion now supported in offline mode. To expand a volume, stop any workloads accessing the volume, then edit the PVC to increase the capacity. For more information, see our Volume Resize operations page and the CSI Volume Expansion page. api: Volume configuration including replica count can now be updated while the volume is in use. Other updateable fields include labels and description. failover: Before determining that a node is offline and performing recovery operations, the I/O path is also verified. This provides more robust failure detection and ensures that nodes that are still responding to I/O do not get replaced. This I/O path verification is in addition to the gossip-based failure detection. operator: Default tolerations are now set for the StorageOS node container. This helps ensure that the StorageOS node container does not get evicted when the node is running low on resources. Improved api: Added checks to prevent deletion of a node with active volumes, or if it is the master of at least one volume. This helps prevent orphaned volumes. cli: Add an --offline-delete flag to allow removal of volumes whose master and replica nodes are offline. This allows cleanup of orphaned volumes. ui: Add an offline volume delete option. ui: Volumes can now be detached from the UI. cli: Labels are no longer truncated. api: When a new node is added to the cluster, its capacity is available to use immediately. Fixed ui: Favicon was missing. ui: Duplicate volumes could be shown on the node details page. operator: During uninstall a ClusterRoleBinding was not removed. v2.0.0 - Released 2020-05-05 New operator: StorageOS containers now run in the kube-system namespace by default to allow the system-node-critical priority class to be set. This instructs Kubernetes to start StorageOS before application Pods, and to evict StorageOS only after application Pods have finished. This setting was previously recommended in documentation; it is now the default. operator: StorageOS CSI helper containers now run as privileged. This ensures that the CSI endpoint can be seen on systems with SELinux enabled. ui: replication progress for new or re-joining replicas is now displayed. ui: show warning for unlicensed clusters. cli: new commands: licence management get policy create namespace create policy describe user describe namespace describe policy delete user delete namespace delete policy licence: removed the default licence expiry date added for v2.0.0-rc.1. Improved dataplane: improved retry behaviour for network I/O. cli: \u0026ldquo;get volumes\u0026rdquo; for all namespaces should be done in parallel. cli: help text document config file ui: link node name and get to node details on the volume details page. ui: node details add available capacity spinner. ui: node list remove capacity values / address port. ui: node list show master/replica counts. ui: node list remove edit action. ui: format entity labels. ui: node details link volumes. ui: align buttons for licences. ui: k8s warning in \u0026ldquo;create volume\u0026rdquo; modal. ui: node list remove \u0026ldquo;API\u0026rdquo; from \u0026ldquo;API Address\u0026rdquo; ui: add some details about the Licence on the licence page. api: include valid for duration in login response. licence: restrict nodes which are unregistered after 24 hours. scheduler: return error for namespace/volume not found dataplane: start gRPC threads separately from rest of the supervisor. Fixed ui: centre licence types. ui: capacity in ui is per namespace. cli: fail gracefully if missing some output details (i.e. no node exists for id). v2.0.0-rc.1 - Released 2020-03-31 Initial release of version 2.x. See StorageOS v2.0 Release Blog for details.\n","excerpt":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than …","ref":"/v2.1/docs/reference/release_notes/","title":"Release notes"},{"body":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous replication ensures strong consistency for applications such as databases and Elasticsearch, incurring one network round trip on writes.\nThe basic model for StorageOS replication is of a master volume with distributed replicas. Each volume can be replicated between 0 and 5 times, which are provisioned to 0 to 5 nodes, up to the number of remaining nodes in the cluster.\nIn this diagram, the master volume D was created on node 1, and two replicas, D2 and D3 on nodes 3 and 5.\nWrites that come into D (step 1) are written in parallel to D2 and D3 (step 2). When both replicas and the master acknowledge that the data has been written (step 3), the write operation return successfully to the application (step 4).\nFor most applications, one replica is sufficient (storageos.com/replicas=1).\nAll replication traffic on the wire is compressed using the lz4 algorithm, then streamed over tcp/ip to target port tcp/5703.\nIf the master volume is lost, a replica is promoted to master (D2 or D3 above) and a new replica is created and synced on an available node (Node 2 or 4). This is transparent to the application and does not cause downtime.\nIf a replica volume is lost and there are enough remaining nodes, a new replica is created and synced on an available node. While a new replica is created and being synced, the volume\u0026rsquo;s health will be marked as degraded.\nIf the lost replica comes back online before the new replica has finished synchronizing, then StorageOS will calculate which of the two synchronizing replicas has the smallest difference compared to the master volume and keep that replica. The same holds true if a master volume is lost and a replica is promoted to be the new master. If possible, a new replica will be created and begin to sync. Should the former master come back online it will be demoted to a replica and the replica will the smallest difference to the current master will be kept.\nWhile the replica count is controllable on a per-volume basis, some environments may prefer to set default labels on the StorageClass.\nDelta Sync StorageOS implements a delta sync between a volume master and its replicas. This means that if a replica for a volume goes offline, that when the replica comes back online only the regions with changed blocks need to be synchronized. This optimization reduces the time it takes for replicas to catch up, improving volume resilience. Additionally, it reduces network and IO bandwidth which can reduce costs when running in public clouds.\n","excerpt":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous …","ref":"/v2.1/docs/concepts/replication/","title":"Replication"},{"body":"Managing Resources for StorageOS containers Kubernetes resource requests and limits are two optional Pod properties that allow you to specify how much of a resource a container in a Pod needs or can use. They are two main resources that you can specify requests and limits for, CPU and Memory.\nAs StorageOS is an infrastructure component, the health of other applications depends on being able to write to the StorageOS volumes. As such it is of paramount importance to avoid restarts of the StorageOS DaemonSet Pods. Restarting a StorageOS Pod results in the volumes of the node the StorageOS Pod is running on being marked as Read Only, and causes the failover of primary volumes on that node to their replicas. After a StorageOS Pod restart, once the StorageOS DaemonSet Pod is \u0026ldquo;READY\u0026rdquo;, the application Pods running on the node need to be restarted in order to trigger a mount of the filesystem hosted on the StorageOS volume and resume normal operations. To avoid restarts of the StorageOS main container by Kubernetes due to resource limits being reached, it is recommended to not set resource limits on the StorageOS DaemonSet. In addition to avoiding resource limits, StorageOS uses a high priority class when the DaemonSet is installed in the \u0026lsquo;kube-system\u0026rsquo; namespace. That avoids the DaemonSet Pods of being evicted.\nFor more information about managing resources for containers please see the Kubernetes documentation\nDefining Pod resource requests and reservations To add resource requests and reservations to the StorageOS DaemonSet configure them in the StorageOSCluster resource.\n","excerpt":"Managing Resources for StorageOS containers Kubernetes resource requests and limits are two optional …","ref":"/v2.1/docs/reference/resource_requests_and_limits/","title":"Resource Requests and Limits"},{"body":"","excerpt":"","ref":"/v2.1/search/","title":"Search Results"},{"body":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for instance, StorageOS. A StorageClass defines parameters to pass to the provisioner, which in the case of StorageOS can be translated into behaviour applied to the Volumes. Many StorageClasses can be provisioned to apply different feature labels to the StorageOS Volumes.\nBy default the StorageOS Cluster Operator installs the fast StorageClass at bootstrap of StorageOS. You can define its name in the StorageOS Cluster Resource.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:fastlabels:app:storageosprovisioner:csi.storageos.com# CSI DriverallowVolumeExpansion:trueparameters:pool:defaultcsi.storage.k8s.io/fstype:ext4# Change the NameSpace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/controller-expand-secret-name:csi-controller-expand-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-expand-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# NameSpace that runs StorageOS DaemonsetStorageClasses can be created to define default labels for StorageOS volumes, but also to map to any semantic aggregation of volumes that suits your use case, whether there are different roles (dev, staging, prod), or a StorageClass maps to a team or customer using the cluster.\nExamples You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic StorageClass definition in v2-storageclass-replicated.yaml.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIallowVolumeExpansion:trueparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the NameSpace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/controller-expand-secret-name:csi-controller-expand-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-expand-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# NameSpace that runs StorageOS DaemonsetThat StorageClass can be used by a PVC:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels.\nYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\n","excerpt":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for …","ref":"/v2.1/docs/operations/storageclasses/","title":"Kubernetes StorageClasses"},{"body":"","excerpt":"","ref":"/v2.1/","title":"StorageOS"},{"body":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for two different purposes.\nTelemetry Telemetry is made up a DNS version check query and a once per day report. Error reporting is the sentry.io crash dump reporting. sentry.io StorageOS sends crash reports to sentry.io. This information helps our developers monitor and fix crashes. Information is sent to sentry.io when a process inside the StorageOS container crashes.\n The crash report contains the signal that triggered the shutdown (e.g. SIGSEGV), the exit code and whether or not the crash generated a core dump. All StorageOS clusters with a routable connection to the internet will send crash reports to sentry.io over tcp/443. StorageOS respects environment variables that ProxyFromEnvironment uses.\nAn exhaustive list of information included in the crash report is below:\n StorageOS version Crash description string Anonymized Cluster ID Anonymized Node ID DNS Query StorageOS will perform a \u0026ldquo;latest version check\u0026rdquo; using a DNS query in order to inform administrators that a new version is available. StorageOS will also send anonymized node ids, cluster id and StorageOS version information to StorageOS using a DNS query. The information that we send in the query is encoded as well as being anonymized. This query allows us to inform Cluster admins when StorageOS upgrades are available in the StorageOS GUI and in the logs.\nThe DNS query includes:\n Anonymized StorageOS Cluster ID Anonymized StorageOS node ID StorageOS version number Once Per Day Report The once per day report contains information about the StorageOS cluster and Kubernetes versions to help StorageOS focus our development efforts on the most popular platforms. The once per day data is encrypted and sent to a StorageOS telemetry server so it is never processed outside of StorageOS assets.\nAn exhaustive list of information included in the once per day report is below:\n api_call_metrics cluster_disable_crash_reporting cluster_disable_version_check cluster_log_format cluster_log_level cluster_tls_provided k8s_distribution k8s_in_k8s k8s_scheduler_extender_enabled k8s_version node_available_bytes node_capacity node_crash_files_on_disk node_created_at_time node_etcd_config node_etcd_namespacing_enabled node_etcd_tls_enabled node_free_bytes node_health node_http_tls_enabled node_id node_labels node_status node_storageos_version node_system_clock_time node_total_bytes node_version volume_fs_type volume_id volume_labels volume_master_attach volume_master_delete_deployment volume_master_detach volume_master_failover_deployment volume_master_promote volume_master_provision volume_master_recover_replica volume_master_trigger_rejoin volume_metrics volume_placement_strategy volume_provision_source_user_agent volume_replica_delete_deployment volume_replica_failover_deployment volume_replica_promote volume_replica_provision volume_replicas volume_replica_trigger_rejoin volume_size_bytes Disable Telemetry It is possible to disable telemetry using the GUI, CLI, API, environment variables or the StorageOS Cluster Spec.\nStorageOS Cluster Spec Disable telemetry explicitly through the configurable spec parameters of the StorageOSCluster custom resource.\nEnvironment Variables You can use the following environmental variables to disable or enable telemetry.\nDISABLE_VERSION_CHECK # Disable the DNS version check DISABLE_TELEMETRY # Disable the once per day reporting DISABLE_ERROR_REPORTING # Disable sentry.io crash reports ","excerpt":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for …","ref":"/v2.1/docs/reference/telemetry/","title":"StorageOS Telemetry"},{"body":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The bundle packages the information needed for the engineering team to understand the context of your cluster and begin troubleshooting any issues you may be experiencing.\nGenerate Bundle CLI The StorageOS CLI can generate the bundle.\n$ storageos get diagnostics The file generated is in the form of diagnostics-${TIME_STAMP}.gz\n GUI Or you can use the StoregeOS GUI.\n Go to section \u0026ldquo;Cluster\u0026rdquo; Press the button \u0026ldquo;DOWNLOAD DIAGNOSTICS\u0026rdquo;. Data collected in the bundle Most of the data collected in the bundle is regarding the state of the StorageOS cluster, however some other information regarding the infrastructure is also gathered. The information is used to have a clear view of the cluster where StorageOS is running.\nThe bundle incorporates for each node:\n StorageOS Daemonset Pod logs lshw dmesg (kernel logs) StorageOS metadata for the ControlPlane and DataPlane StorageOS metadata collected cluster metadata namespaces metadata nodes metadata volumes metadata capacity stats environment variables health Privacy StorageOS can only obtain the bundle if it is downloaded by the user and given to our engineering team, or uploaded for analysis. The data received by StorageOS is private and never leaves nor will leave StorageOS Inc.\nThe data contained in the cluster diagnostic bundle has the sole purpose of helping customers troubleshoot their issues.\n","excerpt":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The …","ref":"/v2.1/docs/reference/bundle/","title":"Support bundle"},{"body":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base directory for storing configuration and blob files. Recommended host filesystem types are ext4 or xfs. If you require a specific filesystem please contact StorageOS.\nPersistent Volume Filesystems StorageOS provides a block device on which a file system can be created. The creation of the filesystem is either handled by StorageOS or by Kubernetes which affects what filesystems can be created.\nCSI Driver When using StorageOS with the CSI driver, StorageOS is responsible for running mkfs against the block device that pods mount. StorageOS is able to create ext2, ext3, ext4 and xfs file systems.\n","excerpt":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base …","ref":"/v2.1/docs/reference/filesystems/","title":"Supported File systems"},{"body":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain node taints. Taints can be thought of as the opposite of Node affinity in that taints repel pods. Node taints are automatically applied by Kubernetes in response to node resources coming under contention. As StorageOS provides storage to pods it should not be evicted during periods of resource contention, as any pods using StorageOS volumes on the same node would need to be restarted.\nAs StorageOS runs as a daemonset some Kubernetes tolerations are added by Kubernetes while others are automatically added by the StorageOS operator.\nFor more information about tolerations please see the Kubernetes documentation.\ntolerations:# The unreachable and not-ready tolerations are added by Kubernetes to daemonsets automatically- key:\u0026#34;node.kubernetes.io/unreachable\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;- key:\u0026#34;node.kubernetes.io/not-ready\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;# The following tolerations are added to the StorageOS daemonset by the StorageOS operator- key:node.kubernetes.io/disk-pressureoperator:Exists- key:node.kubernetes.io/memory-pressureoperator:Exists- key:node.kubernetes.io/network-unavailableoperator:Exists- key:node.kubernetes.io/out-of-diskoperator:Exists- key:node.kubernetes.io/pid-pressureoperator:Exists- key:node.kubernetes.io/unschedulableoperator:ExistsAdding Custom Tolerations To add custom tolerations to the StorageOS daemonset configure them in the StorageOSCluster resource.\n","excerpt":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain …","ref":"/v2.1/docs/reference/tolerations/","title":"Tolerations"},{"body":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to the StorageOS installation, integration with orchestrators or common misconfigurations.\nTools To be able to troubleshoot issues the StorageOS cli is required.\nPod in pending because of mount error Issue: The output of kubectl describe pod $POD_ID contains no such file or directory and references the StorageOS volume device file.\nroot@node1:~# kubectl -n kube-system describe $POD_ID (...) Events: (...) Normal Scheduled 11s default-scheduler Successfully assigned default/d1 to node3 Warning FailedMount 4s (x4 over 9s) kubelet, node3 MountVolume.SetUp failed for volume \u0026#34;pvc-f2a49198-c00c-11e8-ba01-0800278dc04d\u0026#34; : stat /var/lib/storageos/volumes/d9df3549-26c0-4cfc-62b4-724b443069a1: no such file or directory Reason: There are two main reasons this issue may arise:\n The StorageOS DEVICE_DIR location is wrongly configured when using Kubelet as a container Mount Propagation is not enabled (Option 1) Misconfiguration of the DeviceDir/SharedDir\nSome Kubernetes distributions such as Rancher, DockerEE or some installations of OpenShift deploy the Kubelet as a container, because of this, the device files that StorageOS creates to mount into the containers need to be visible to the kubelet. StorageOS can be configured to share the device directory.\nModern installations use CSI, which handles the complexity internally.\nAssert: root@node1:~# kubectl -n default describe stos | grep \u0026#34;Shared Dir\u0026#34; Shared Dir: # \u0026amp;lt;-- Shouldn\u0026#39;t be blank Solution: The Cluster Operator Custom Definition should specify the SharedDir option as follows.\nspec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container ... See example on how to configure the StorageOS Custom Resource.\n (Option 2) Mount propagation is not enabled.\n Applies only if Option 1 is configured properly.\n Assert: If not using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/storageos/volumes is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/storageos/volumes/ root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/storageos/volumes bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nIf using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/kubelet/plugins/kubernetes.io~storageos/devices is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/kubelet/plugins/kubernetes.io~storageos/devices root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/kubelet/plugins/kubernetes.io~storageos/devices bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nSolution: Older versions of Kubernetes need to enable mount propagation as it is not enabled by default. Most Kubernetes distributions allow MountPropagation to be enabled using FeatureGates. Rancher specifically, needs to enable it in the \u0026ldquo;View in API\u0026rdquo; section of your cluster. You need to edit the section \u0026ldquo;rancherKubernetesEngineConfig\u0026rdquo; to enable the Kubelet feature gate.\nPVC pending state - Failed to dial StorageOS A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: root@node1:~/# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE vol-1 Pending fast 7s kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 7s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: Get http://storageos-cluster/version: failed to dial all known cluster members, (10.233.59.206:5705) Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount volume can\u0026rsquo;t be transmitted to StorageOS, hence the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nIn this case, the Event message indicates that StorageOS API is not responding, implying that StorageOS is not running. For Kubernetes to define StorageOS pods ready, the health check must pass.\nAssert: Check the status of StorageOS pods.\nroot@node1:~/# kubectl -n kube-system get pod --selector app=storageos # for CSI add --selector kind=daemonset NAME READY STATUS RESTARTS AGE storageos-qrqkj 0/1 Running 0 1m storageos-s4bfv 0/1 Running 0 1m storageos-vcpfx 0/1 Running 0 1m storageos-w98f5 0/1 Running 0 1m If the pods are not READY, the service will not forward traffic to the API they serve hence PVC will remain in pending state until StorageOS pods are available.\n Kubernetes keeps trying to execute the action until it succeeds. If a PVC is created before StorageOS finish starting, the PVC will be created eventually.\n Solution: StorageOS health check takes 60 seconds of grace before reporting as READY. If StorageOS is starting properly after that period, the volume will be created when StorageOS finishes its bootstrap. If StorageOS is not running or is not starting properly, the solution would be to troubleshoot the installation. PVC pending state - Secret Missing A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 13s (x2 over 28s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: failed to get secret from [\u0026#34;storageos\u0026#34;/\u0026#34;storageos-api\u0026#34;] Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount a volume can\u0026rsquo;t be transmitted to StorageOS, and the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nThe StorageClass provisioned for StorageOS references a Secret from where it retrieves the API endpoint and the authentication parameters. If that secret is incorrect or missing, the connections won\u0026rsquo;t be established. It is common to see that the Secret has been deployed in a different namespace where the StorageClass expects it or that is has been deployed with a different name.\nAssert: Check the StorageClass parameters to know where the Secret is expected to be found.\nkubectl get storageclass fast -o yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: 2018-09-25T08:44:57Z labels: app: storageos name: fast resourceVersion: \u0026#34;108853\u0026#34; selfLink: /apis/storage.k8s.io/v1/storageclasses/fast uid: 48490a9b-c09f-11e8-ba01-0800278dc04d parameters: adminSecretName: storageos-api adminSecretNamespace: storageos description: Kubernetes volume fsType: ext4 pool: default provisioner: kubernetes.io/storageos reclaimPolicy: Delete Note that the parameters specify adminSecretName and adminSecretNamespace.\n Check if the secret exists according to those parameters\nkubectl -n kube-system get secret storageos-api No resources found. Error from server (NotFound): secrets \u0026#34;storageos-api\u0026#34; not found If no resources are found, it is clear that the Secret doesn\u0026rsquo;t exist or it is not deployed in the right location.\n Solution: Deploy StorageOS following the installation procedures. If you are using the manifests provided for Kubernetes to deploy StorageOS rather than using automated provisioners, make sure that the StorageClass parameters and the Secret reference match.\nNode name different from Hostname Issue: StorageOS nodes can\u0026rsquo;t join the cluster showing the following log entries.\ntime=\u0026#34;2018-09-24T13:47:02Z\u0026#34; level=error msg=\u0026#34;failed to start api\u0026#34; error=\u0026#34;error verifying UUID: UUID aed3275f-846b-1f75-43a1-adbfec8bf974 has already been registered and has hostname \u0026#39;debian-4\u0026#39;, not \u0026#39;node4\u0026#39;\u0026#34; module=command Reason: The StorageOS registration process to start the cluster uses the hostname of the node where the StorageOS container is running, provided by the Kubelet. However, StorageOS verifies the network hostname of the OS as a prestart check to make sure it can communicate with other nodes. If those names don\u0026rsquo;t match, StorageOS will be unable to start.\nSolution: Make sure the hostnames match with the Kubernetes advertised names. If you have changed the hostname of your nodes, make sure that you restart the nodes to apply the change.\nPeer discovery - Networking Issue: StorageOS nodes can\u0026rsquo;t join the cluster showing the following logs after one minute of container uptime.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover nodes in the cluster. When StorageOS starts, one or more nodes can be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nAssert: It is likely that ports are block by a firewall.\nSSH into one of your nodes and check connectivity to the rest of the nodes.\n# Successfull execution: [root@node06 ~]# nc -zv node04 5705 Ncat: Version 7.50 ( https://nmap.org/ncat ) Ncat: Connected to 10.0.1.166:5705. Ncat: 0 bytes sent, 0 bytes received in 0.01 seconds. StorageOS exposes network diagnostics in its API, viewable from the CLI. To use this feature, the CLI must query the API of a running node. The diagnostics show information from all known cluster members. If all the ports are blocked during the first bootstrap of the cluster, the diagnostics won\u0026rsquo;t show any data as nodes couldn\u0026rsquo;t register.\n StorageOS networks diagnostics are available for storageos-rc5 and storageos-cli-rc3 and above.\n # Example: root@node1:~# storageos cluster connectivity SOURCE NAME ADDRESS LATENCY STATUS MESSAGE node4 node2.nats 172.28.128.4:5708 1.949275ms OK node4 node3.api 172.28.128.5:5705 3.070574ms OK node4 node3.nats 172.28.128.5:5708 2.989238ms OK node4 node2.directfs 172.28.128.4:5703 2.925707ms OK node4 node3.etcd 172.28.128.5:5707 2.854726ms OK node4 node3.directfs 172.28.128.5:5703 2.833371ms OK node4 node1.api 172.28.128.3:5705 2.714467ms OK node4 node1.nats 172.28.128.3:5708 2.613752ms OK node4 node1.etcd 172.28.128.3:5707 2.594159ms OK node4 node1.directfs 172.28.128.3:5703 2.601834ms OK node4 node2.api 172.28.128.4:5705 2.598236ms OK node4 node2.etcd 172.28.128.4:5707 16.650625ms OK node3 node4.nats 172.28.128.6:5708 1.304126ms OK node3 node4.api 172.28.128.6:5705 1.515218ms OK node3 node2.directfs 172.28.128.4:5703 1.359827ms OK node3 node1.api 172.28.128.3:5705 1.185535ms OK node3 node4.directfs 172.28.128.6:5703 1.379765ms OK node3 node1.etcd 172.28.128.3:5707 1.221176ms OK node3 node1.nats 172.28.128.3:5708 1.330122ms OK node3 node2.api 172.28.128.4:5705 1.238541ms OK node3 node1.directfs 172.28.128.3:5703 1.413574ms OK node3 node2.etcd 172.28.128.4:5707 1.214273ms OK node3 node2.nats 172.28.128.4:5708 1.321145ms OK node1 node4.directfs 172.28.128.6:5703 1.140797ms OK node1 node3.api 172.28.128.5:5705 1.089252ms OK node1 node4.api 172.28.128.6:5705 1.178439ms OK node1 node4.nats 172.28.128.6:5708 1.176648ms OK node1 node2.directfs 172.28.128.4:5703 1.529612ms OK node1 node2.etcd 172.28.128.4:5707 1.165681ms OK node1 node2.api 172.28.128.4:5705 1.29602ms OK node1 node2.nats 172.28.128.4:5708 1.267454ms OK node1 node3.nats 172.28.128.5:5708 1.485657ms OK node1 node3.etcd 172.28.128.5:5707 1.469429ms OK node1 node3.directfs 172.28.128.5:5703 1.503015ms OK node2 node4.directfs 172.28.128.6:5703 1.484ms OK node2 node1.directfs 172.28.128.3:5703 1.275304ms OK node2 node4.nats 172.28.128.6:5708 1.261422ms OK node2 node4.api 172.28.128.6:5705 1.465532ms OK node2 node3.api 172.28.128.5:5705 1.252768ms OK node2 node3.nats 172.28.128.5:5708 1.212332ms OK node2 node3.directfs 172.28.128.5:5703 1.192792ms OK node2 node3.etcd 172.28.128.5:5707 1.270076ms OK node2 node1.etcd 172.28.128.3:5707 1.218522ms OK node2 node1.api 172.28.128.3:5705 1.363071ms OK node2 node1.nats 172.28.128.3:5708 1.349383ms OK Solution: Open ports following the prerequisites page.\nPeer discovery - Pod allocation Issue: StorageOS nodes can\u0026rsquo;t join the cluster and show the following log entries.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover the nodes in the cluster. When StorageOS starts, one or more active nodes must be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nIf there are no active StorageOS nodes, the bootstrap process will elect the first node in the JOIN variable as master, and the rest will try to discover from it. In case of that node not starting, the whole cluster will remain unable to bootstrap.\nInstallations of StorageOS use a DaemonSet, and by default do not schedule StorageOS pods to master nodes, due to the presence of the node-role.kubernetes.io/master:NoSchedule taint that is typically present. In such cases the JOIN variable must not contain master nodes or the StorageOS cluster will remain unable to start.\nAssert: Check that the first node of the JOIN variable started properly.\nroot@node1:~/# kubectl -n kube-system describe ds/storageos | grep JOIN JOIN: 172.28.128.3,172.28.128.4,172.28.128.5 root@node1:~/# kubectl -n kube-system get pod -o wide | grep 172.28.128.3 storageos-8zqxl 1/1 Running 0 2m 172.28.128.3 node1 Solution: Make sure that the JOIN variable doesn\u0026rsquo;t specify the master nodes. In case you are using the discovery service, it is necessary to ensure that the DaemonSet won\u0026rsquo;t allocate Pods on the masters. This can be achieved with taints, node selectors or labels.\nFor installations with the StorageOS operator you can specify which nodes to deploy StorageOS on using nodeSelectors. See examples in the Cluster Operator Examples page.\nFor more advanced installations using compute-only and storage nodes, check the storageos.com/deployment=computeonly label that can be added to the nodes through Kubernetes node labels, or StorageOS in the Labels page.\nLIO Init:Error Issue: StorageOS pods not starting with Init:Error\nkubectl -n kube-system get pod NAME READY STATUS RESTARTS AGE storageos-2kwqx 0/3 Init:Err 0 6s storageos-cffcr 0/3 Init:Err 0 6s storageos-d4f69 0/3 Init:Err 0 6s storageos-nhq7m 0/3 Init:Err 0 6s Reason: This indicates that since the Linux open source SCSI drivers are not enabled, StorageOS cannot start. The StorageOS DaemonSet enables the required kernel modules on the host system. If you are seeing these errors it is because that container couldn\u0026rsquo;t load the modules.\nAssert Check the logs of the init container.\nkubectl -n kube-system logs $ANY_STORAGEOS_POD -c storageos-init In case of failure, it will show the following output, indicating which kernel modules couldn\u0026rsquo;t be loaded or that they are not properly configured:\nChecking configfs configfs mounted on sys/kernel/config Module target_core_mod is not running executing modprobe -b target_core_mod Module tcm_loop is not running executing modprobe -b tcm_loop modprobe: FATAL: Module tcm_loop not found. Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and delete StorageOS pods, allowing the DaemonSet to create the pods again.\nLIO not enabled Issue: StorageOS node can\u0026rsquo;t start and shows the following log entries.\ntime=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;liocheck returned error\u0026#34; category=liocheck error=\u0026#34;exit status 1\u0026#34; module=dataplane stderr=\u0026#34;Sysfs root \u0026#39;/sys/kernel/config/target\u0026#39; is missing, is kernel configfs present and target_core_mod loaded? category=fslio level=warn\\nRuntime error checking stage \u0026#39;target_core_mod\u0026#39;: SysFs root missing category=fslio level=warn\\nliocheck: FAIL (lio_capable_system() returns failure) category=fslio level=fatal\\n\u0026#34; stdout= time=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;failed to start dataplane services\u0026#34; error=\u0026#34;system dependency check failed: exit status 1\u0026#34; module=command Reason: This indicates that one or more kernel modules required for StorageOS are not loaded.\nAssert The following kernel modules must be enabled in the host.\nlsmod | egrep \u0026#34;^tcm_loop|^target_core_mod|^target_core_file|^configfs\u0026#34; Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and restart the container.\n(OpenShift) StorageOS pods missing \u0026ndash; DaemonSet error StorageOS DaemonSet doesn\u0026rsquo;t have any pod replicas. The DaemonSet couldn\u0026rsquo;t allocate any Pod due to security issues.\nIssue: [root@master02 standard]# oc get pod No resources found. [root@master02 standard]# oc describe daemonset storageos (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedCreate 0s (x12 over 10s) daemonset-controller Error creating: pods \u0026#34;storageos-\u0026#34; is forbidden: unable to validate against any security context constraint: [provider restricted: .spec.securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used provider restricted: .spec.securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.volumes[0]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[1]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[2]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[3]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.initContainers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.initContainers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.initContainers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.initContainers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.containers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.containers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.containers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used] Reason: The OpenShift cluster has security context constraint policies enabled that forbid any pod, without an explicitly set policy for the service account, to be allocated.\nAssert: Check if the StorageOS ServiceAccount can create pods with enough permissions\noc get scc privileged -o yaml # Or custom scc with enough privileges (...) users: - system:admin - system:serviceaccount:openshift-infra:build-controller - system:serviceaccount:management-infra:management-admin - system:serviceaccount:management-infra:inspector-admin - system:serviceaccount:storageos:storageos \u0026amp;lt;-- - system:serviceaccount:tiller:tiller If the StorageOS sa system:serviceaccount:storageos:storageos is in the privileged scc it will be able to create pods.\nSolution: Add the ServiceAccount system:serviceaccount:storageos:storageos to a scc with enough privileges.\noc adm policy add-scc-to-user privileged system:serviceaccount:storageos:storageos Getting Help If our troubleshooting guides do not help resolve your issue, please see our support section for details on how to get in touch with us.\n","excerpt":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to …","ref":"/v2.1/docs/operations/troubleshooting/","title":"Troubleshooting"},{"body":"This document details a step-by-step procedure to remove StorageOS from a Kubernetes cluster.\nRemember that StorageOS enables the stateful applications within your cluster. It is very important to remove any applications that rely on StorageOS before you remove StorageOS itself, or those applications will suffer unrecoverable errors.\nRemove Stateful Workloads and Data Delete any resources using StorageOS volumes\nDelete any statefulsets, deployments or pods that are using StorageOS Volumes.\n Delete PVCs using StorageOS\nDelete any Persistent Volume Claims that are using StorageOS.\n$ kubectl -n $NS delete pvc $PVC This will delete data held by StorageOS and won\u0026rsquo;t be recoverable.\n Remove StorageOS Cluster Delete StorageOS Cluster\n$ kubectl get storageoscluster --all-namespaces # Find the namespace where the Custom Resource runs $ kubectl -n $NS delete storageoscluster --all # Usually to be found in storageos-operator Wait until the StorageOS resources are gone\n$ kubectl -n kube-system get pod # NS: Namespace where StorageOS Daemonset is running, usually \u0026#39;kube-system\u0026#39; Uninstall the StorageOS Operator Delete the Cluster Operator once the StorageOS Pods are terminated\n Delete the StorageOS Operator deployment\n$ kubectl delete -f https://github.com/storageos/cluster-operator/releases/download/v2.1.0/storageos-operator.yaml Procedure is finished. StorageOS is now uninstalled.\nRemove StorageOS contents and metadata (unrecoverable) The steps up until now have been recoverable - as long as the etcd backing StorageOS and the contents of /var/lib/storageos on your nodes are safe then StorageOS can be reinstalled. For complete removal and recovery of disk space, you can use the following procedure.\n The following steps will delete all data held by StorageOS and won\u0026rsquo;t be recoverable.\n Remove the StorageOS data directory\nThere are two ways to remove the StorageOS data directory:\n (Option 1) Login in to the hosts and execute the following commands\n$ sudo rm -rf /var/lib/storageos $ sudo umount /var/lib/kubelet/plugins_registry/storageos (Option 2) Execute the following command to deploy a DaemonSet that removes the StorageOS data directory.\n N.B This step is irreversible and once the data is removed it cannot be recovered.\n Run the following command where kubectl is installed and with the context set for your Kubernetes cluster.\n $ curl -s https://docs.storageos.com/v2.1/sh/permanently-delete-storageos-data.sh | bash Flush Etcd Data\nThis will remove any keys written by StorageOS.\n$ export ETCDCTL_API=3 $ etcdctl --endpoints=http://$ETCD_IP:2379 del --prefix \u0026#34;storageos\u0026#34; If running Etcd with mTLS, you can set the certificates location with the following command.\n$ export ETCDCTL_API=3 $ etcdctl --endpoints=https://$ETCD_IP:2379 \\ --cacert=/path/to/ca.pem \\ --cert=/path/to/client-cert.pem \\ --key=/path/to/client-key.pem \\ del --prefix \u0026#34;storageos\u0026#34; ","excerpt":"This document details a step-by-step procedure to remove StorageOS from a Kubernetes cluster. …","ref":"/v2.1/docs/operations/uninstall/","title":"Uninstall StorageOS"},{"body":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces using policies.\n Note: Users are created with access to the default namespace. This access is only revoked when a policy is created for the user or their group.\n Managing users Users can be created and deleted by navigating to the \u0026ldquo;Users\u0026rdquo; section of the GUI.\nAlternatively users can also be managed using the CLI.\nTo create a user with the CLI, run:\n$ storageos user create jim The above command will create a user named jim. The command will also prompt you to enter a password for the newly created user.\n","excerpt":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces …","ref":"/v2.1/docs/operations/users/","title":"User Management"},{"body":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version [flags] Flags: -h, --help help for version Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version …","ref":"/v2.1/docs/reference/cli/version/","title":"Version"},{"body":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating the volume config via the CLI or UI. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure the volume is not in use. Please note that StorageOS only supports increasing volume size. For more information about how the resize works please see our Resize concepts page.\nResizing a Volume In order to resize a PVC the storage request field must be updated.\napiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: fast accessModes: - ReadWriteOnce resources: requests: storage: 5Gi In order to edit a PVC you can use kubectl edit or kubectl apply to make changes.\nStorageOS supports offline resize of volumes through editing a PVC storage request. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This requires that any pods using a volume must be scaled down for the resize to take place.\n N.B. Resizing a volume without updating the PVC directly will NOT result in the PVC being updated. The methods below are included for completeness, in Kubernetes environments editing the PVC is the preferred method for resizing a volume.\n To resize a volume using the StorageOS CLI use the volume update command\n$ storageos update volume size pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa 10GiB Name: pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa ID: 925e667f-91d3-465a-9391-8fdb56d0c9ff Size: 11 GB Description: AttachedOn: Replicas: 1x ready Labels: - csi.storage.k8s.io/pv/name pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa - csi.storage.k8s.io/pvc/name pvc-1 - csi.storage.k8s.io/pvc/namespace default - foo bar - pool default - storageos.com/replicas 1 Volume pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa (925e667f-91d3-465a-9391-8fdb56d0c9ff) updated. Size changed. To resize a volume using the StorageOS UI, navigate to the volumes section and click the edit pencil in order to update the volume config.\n","excerpt":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating …","ref":"/v2.1/docs/operations/resize/","title":"Volume Resize"},{"body":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard POSIX semantics. StorageOS presents volumes as mounts into containers via the Linux LIO subsystem.\nConceptually, StorageOS volumes have a frontend presentation, which is the what the application sees, and a backend presentation, which is the actual on-disk format. Depending on the configuration, frontend and backend components may be on the same or different hosts.\nVolumes are formatted using the linux standard ext4 filesystem by default. Kubernetes users may change the default filesystem type to ext2, ext3, ext4, or xfs by setting the fsType parameter in their StorageClass (See Supported Filesystems for more information). Different filesystems may be supported in the future.\nStorageOS volumes are represented on disk in two parts. Actual volume data is written to blob files in /var/lib/storageos/data/dev[\\d+]. Inside these directories, each StorageOS block device gets two blob files of the form vol.xxxxxx.y.blob, where x is the inode number for the device, and y is an index between 0 and 1. We provide two blob files in order to ensure that certain operations which require locking do not impede in-flight writes to the volume.\nIn systems which have multiple /var/lib/storageos/data/dev[\\d+] directories, two blob files are created per block device. This allows us to load-balance writes across multiple devices. In cases where dev directories are added after a period of run time, later directories are favoured for writes until the data is distributed evenly across the blob files.\nMetadata is kept in directories named /var/lib/storageos/data/db[\\d+]. We maintain an index of all blocks written to the blob file inside the metadata store, including checksums. These checksums allow us to detect bitrot, and return errors on reads, rather than serve bad data. In future versions we may implement recovery from replicas for volumes with one or more replicas defined.\nStorageOS metadata requires approximately 2.7GB of storage per 1TB of allocated blocks in the associated volume. This size is consistent irrespective of data compression defined on the volume.\nTo ensure deterministic performance, individual StorageOS volumes must fit on a single node.\nVolume Resize StorageOS v2.1 supports offline resize of volumes. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure that the volume is not in use.\nThis means that if a Kubernetes pod is currently consuming a volume that a resize request has been issued for, the resize will not be actioned until the pod is terminated and the volume is detached from the node. The StorageOS controlplane will then attach the volume to the node that holds the master deployment and resize the underlying block device and then run resize2fs to expand the filesystem.\nFor a walk through of how to resize a volume please see the Volume Resize operations page.\n","excerpt":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard …","ref":"/v2.1/docs/concepts/volumes/","title":"Volumes"},{"body":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.\nUsing StorageOS persistent volumes with Apache Zookeeper means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Zookeeper pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nAs Zookeeper has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Zookeeper to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Zookeeper on Kubernetes Pre-requisites StorageOS is assumed to have been installed; please check for the latest available version here. Helm To simplify the deployment of Zookeeper, we\u0026rsquo;ve used this Zookeeper helm chart (incubator) (version 1.2.2, app version 3.4.10) and rendered it into the example deployment files you can find in our GitHub repo.\nDeployment Clone the use cases repo You can find the latest files in the StorageOS use cases repository in /zookeeper/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion ---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:zookeeper...spec:replicas:3# \u0026lt;--- number of zookeeper pods...containers:- name:zookeeperimage:\u0026#34;gcr.io/google_samples/k8szk:v3\u0026#34;imagePullPolicy:IfNotPresent...volumeMounts:- name:datamountPath:/var/lib/zookeepervolumeClaimTemplates:- metadata:name:dataspec:accessModes:- \u0026#34;ReadWriteOnce\u0026#34;storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useresources:requests:storage:\u0026#34;5Gi\u0026#34;# \u0026lt;--- storage requested per podThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nCreate the kubernetes objects cd storageos-usecases kubectl apply -f ./zookeeper/ Confirm Zookeeper is up and running $ kubectl get pods NAME READY STATUS RESTARTS AGE zookeeper-0 1/1 Running 0 2m30s zookeeper-1 1/1 Running 0 112s zookeeper-2 1/1 Running 0 56s zookeeper-test-client 1/1 Running 0 2m30s Connect to Zookeeper Connect to the zookeeper client pod and list existing topics using the service endpoint\nkubectl exec -it zookeeper-test-client /bin/bash and issue a command to the zookeeper service\nzkCli.sh -server zookeeper ls /zookeeper ","excerpt":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration …","ref":"/v2.1/docs/usecases/zookeeper/","title":"Zookeeper"}]
\ No newline at end of file
diff --git a/v2.1/search/index.html b/v2.1/search/index.html
index 70c6d9e71..f2fc3858b 100644
--- a/v2.1/search/index.html
+++ b/v2.1/search/index.html
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.1/offline-search-index.7d11f35697e7f6052a6b8dd6860863a4868f2a4632dcb63e8c531ecdd95790f3cfa0e2afbb33400aa296c44a33fb50c5697b3afcdc3682bc937ad8ab1e9ef25f.json"
+ data-offline-search-index-json-src="/v2.1/offline-search-index.56338aa2a1ca30aa2b62664211ebdfa8012dbf16709bc48768553aa3f38aecb54d8e05d00ecd62e525765858103d790a97eb33d485430c36824999a7486a835d.json"
data-offline-search-base-href="/"
>
diff --git a/v2.1/yaml/bundle-configuration.yaml b/v2.1/yaml/bundle-configuration.yaml
index f1c1004a0..5b21e3ed3 100644
--- a/v2.1/yaml/bundle-configuration.yaml
+++ b/v2.1/yaml/bundle-configuration.yaml
@@ -40,11 +40,18 @@ spec:
# InternalAPIPort = 5710 # Grpc API
# GossipPort = 5711 # Gossip+Healthcheck
- echo \"Source node for the test:\";
+ date;
+ echo \"Source node:\";
hostname -f -I; echo;
+ echo \"TCP:\";
parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
- ::: 5703 5704 5705 5710 5711
+ ::: 5703 5704 5705 5710 5711;
+ echo;
+
+ echo \"UDP:\";
+ parallel -j2 nc -vunz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
+ ::: 5711;
"
timeout: 90s
- exec:
diff --git a/v2.2/404.html b/v2.2/404.html
index 4242cbee0..cfb25509b 100644
--- a/v2.2/404.html
+++ b/v2.2/404.html
@@ -181,7 +181,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/best-practices/index.html b/v2.2/docs/best-practices/index.html
index 040e11898..4a0ecb219 100644
--- a/v2.2/docs/best-practices/index.html
+++ b/v2.2/docs/best-practices/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/architecture/index.html b/v2.2/docs/concepts/architecture/index.html
index 85e705b5b..71ef06d13 100644
--- a/v2.2/docs/concepts/architecture/index.html
+++ b/v2.2/docs/concepts/architecture/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/clusters/index.html b/v2.2/docs/concepts/clusters/index.html
index 4aba1abb9..88acbe26b 100644
--- a/v2.2/docs/concepts/clusters/index.html
+++ b/v2.2/docs/concepts/clusters/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/compression/index.html b/v2.2/docs/concepts/compression/index.html
index 5c262a83b..d110c9a6c 100644
--- a/v2.2/docs/concepts/compression/index.html
+++ b/v2.2/docs/concepts/compression/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/etcd/index.html b/v2.2/docs/concepts/etcd/index.html
index 13758f790..27f4d4c45 100644
--- a/v2.2/docs/concepts/etcd/index.html
+++ b/v2.2/docs/concepts/etcd/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/index.html b/v2.2/docs/concepts/index.html
index 4e9be5fed..66f3110bb 100644
--- a/v2.2/docs/concepts/index.html
+++ b/v2.2/docs/concepts/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/namespaces/index.html b/v2.2/docs/concepts/namespaces/index.html
index 68b6ff744..b9e8d9996 100644
--- a/v2.2/docs/concepts/namespaces/index.html
+++ b/v2.2/docs/concepts/namespaces/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/nodes/index.html b/v2.2/docs/concepts/nodes/index.html
index 697318759..17f453889 100644
--- a/v2.2/docs/concepts/nodes/index.html
+++ b/v2.2/docs/concepts/nodes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/policies/index.html b/v2.2/docs/concepts/policies/index.html
index 3bbb0a8bd..e2fbb683c 100644
--- a/v2.2/docs/concepts/policies/index.html
+++ b/v2.2/docs/concepts/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/replication/index.html b/v2.2/docs/concepts/replication/index.html
index 23b033b64..d73a75442 100644
--- a/v2.2/docs/concepts/replication/index.html
+++ b/v2.2/docs/concepts/replication/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/concepts/volumes/index.html b/v2.2/docs/concepts/volumes/index.html
index 13e8bfd2a..3d806939b 100644
--- a/v2.2/docs/concepts/volumes/index.html
+++ b/v2.2/docs/concepts/volumes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/index.html b/v2.2/docs/index.html
index 7a144f1b4..f93ed2577 100644
--- a/v2.2/docs/index.html
+++ b/v2.2/docs/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/index.xml b/v2.2/docs/index.xml
index da368dcb9..cbcf9bc17 100644
--- a/v2.2/docs/index.xml
+++ b/v2.2/docs/index.xml
@@ -2136,6 +2136,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -2168,7 +2173,7 @@ the init container, perform the following steps:</p>
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -2281,6 +2286,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2.2/docs/install/index.html b/v2.2/docs/install/index.html
index a119f1b26..e5c836892 100644
--- a/v2.2/docs/install/index.html
+++ b/v2.2/docs/install/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/install/kubernetes/index.html b/v2.2/docs/install/kubernetes/index.html
index aa31cd8e7..668292d8f 100644
--- a/v2.2/docs/install/kubernetes/index.html
+++ b/v2.2/docs/install/kubernetes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/install/openshift/index.html b/v2.2/docs/install/openshift/index.html
index 1fccc356e..0a1013be5 100644
--- a/v2.2/docs/install/openshift/index.html
+++ b/v2.2/docs/install/openshift/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/install/rancher/index.html b/v2.2/docs/install/rancher/index.html
index 7bd999b96..89232b056 100644
--- a/v2.2/docs/install/rancher/index.html
+++ b/v2.2/docs/install/rancher/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/introduction/index.html b/v2.2/docs/introduction/index.html
index a62224457..7ace461db 100644
--- a/v2.2/docs/introduction/index.html
+++ b/v2.2/docs/introduction/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/introduction/overview/index.html b/v2.2/docs/introduction/overview/index.html
index cb4a887cb..020e918ed 100644
--- a/v2.2/docs/introduction/overview/index.html
+++ b/v2.2/docs/introduction/overview/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/introduction/platforms/index.html b/v2.2/docs/introduction/platforms/index.html
index 40bfb1376..75ffe2f66 100644
--- a/v2.2/docs/introduction/platforms/index.html
+++ b/v2.2/docs/introduction/platforms/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/introduction/quickstart/index.html b/v2.2/docs/introduction/quickstart/index.html
index 908adbaf1..dbab72229 100644
--- a/v2.2/docs/introduction/quickstart/index.html
+++ b/v2.2/docs/introduction/quickstart/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/cluster-id/index.html b/v2.2/docs/operations/cluster-id/index.html
index 5ef7acaaf..16124ff7f 100644
--- a/v2.2/docs/operations/cluster-id/index.html
+++ b/v2.2/docs/operations/cluster-id/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/etcd/index.html b/v2.2/docs/operations/etcd/index.html
index 033d8687b..b0b1dd38d 100644
--- a/v2.2/docs/operations/etcd/index.html
+++ b/v2.2/docs/operations/etcd/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/etcd/storageos-secret-info/index.html b/v2.2/docs/operations/etcd/storageos-secret-info/index.html
index 57bc58575..488e410a8 100644
--- a/v2.2/docs/operations/etcd/storageos-secret-info/index.html
+++ b/v2.2/docs/operations/etcd/storageos-secret-info/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/firstpvc/index.html b/v2.2/docs/operations/firstpvc/index.html
index 908fc93cd..d6cf423b4 100644
--- a/v2.2/docs/operations/firstpvc/index.html
+++ b/v2.2/docs/operations/firstpvc/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/health/index.html b/v2.2/docs/operations/health/index.html
index da36a5378..406424d75 100644
--- a/v2.2/docs/operations/health/index.html
+++ b/v2.2/docs/operations/health/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/index.html b/v2.2/docs/operations/index.html
index 9f3769314..bf7b617a2 100644
--- a/v2.2/docs/operations/index.html
+++ b/v2.2/docs/operations/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/licensing/index.html b/v2.2/docs/operations/licensing/index.html
index 9204b90a5..6095ac190 100644
--- a/v2.2/docs/operations/licensing/index.html
+++ b/v2.2/docs/operations/licensing/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/managing-host-storage/index.html b/v2.2/docs/operations/managing-host-storage/index.html
index 8071579b4..60abdbd98 100644
--- a/v2.2/docs/operations/managing-host-storage/index.html
+++ b/v2.2/docs/operations/managing-host-storage/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/namespaces/index.html b/v2.2/docs/operations/namespaces/index.html
index a070aa422..88f3edc9d 100644
--- a/v2.2/docs/operations/namespaces/index.html
+++ b/v2.2/docs/operations/namespaces/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/policies/index.html b/v2.2/docs/operations/policies/index.html
index 08b469a63..dcf79af55 100644
--- a/v2.2/docs/operations/policies/index.html
+++ b/v2.2/docs/operations/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/resize/index.html b/v2.2/docs/operations/resize/index.html
index 210843681..2e5e40db2 100644
--- a/v2.2/docs/operations/resize/index.html
+++ b/v2.2/docs/operations/resize/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/storageclasses/index.html b/v2.2/docs/operations/storageclasses/index.html
index dfb38daca..d7c083e13 100644
--- a/v2.2/docs/operations/storageclasses/index.html
+++ b/v2.2/docs/operations/storageclasses/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/troubleshooting/index.html b/v2.2/docs/operations/troubleshooting/index.html
index 2460f946c..065779c3e 100644
--- a/v2.2/docs/operations/troubleshooting/index.html
+++ b/v2.2/docs/operations/troubleshooting/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/uninstall/index.html b/v2.2/docs/operations/uninstall/index.html
index 3970e131f..1c78346c3 100644
--- a/v2.2/docs/operations/uninstall/index.html
+++ b/v2.2/docs/operations/uninstall/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/upgrade/index.html b/v2.2/docs/operations/upgrade/index.html
index e5ad23980..f972820ca 100644
--- a/v2.2/docs/operations/upgrade/index.html
+++ b/v2.2/docs/operations/upgrade/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/operations/users/index.html b/v2.2/docs/operations/users/index.html
index 87db6482c..ac638d94b 100644
--- a/v2.2/docs/operations/users/index.html
+++ b/v2.2/docs/operations/users/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/platforms/aws-eks/index.html b/v2.2/docs/platforms/aws-eks/index.html
index f1c03dde3..d8379ce9e 100644
--- a/v2.2/docs/platforms/aws-eks/index.html
+++ b/v2.2/docs/platforms/aws-eks/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/platforms/azure-aks/index.html b/v2.2/docs/platforms/azure-aks/index.html
index a281a805e..a3c0f85f3 100644
--- a/v2.2/docs/platforms/azure-aks/index.html
+++ b/v2.2/docs/platforms/azure-aks/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/platforms/dockeree/index.html b/v2.2/docs/platforms/dockeree/index.html
index 0fd449761..dc9e0bbbf 100644
--- a/v2.2/docs/platforms/dockeree/index.html
+++ b/v2.2/docs/platforms/dockeree/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/platforms/index.html b/v2.2/docs/platforms/index.html
index 5cf262633..be7478d73 100644
--- a/v2.2/docs/platforms/index.html
+++ b/v2.2/docs/platforms/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/platforms/openshift/index.html b/v2.2/docs/platforms/openshift/index.html
index 844f17e0a..d61180902 100644
--- a/v2.2/docs/platforms/openshift/index.html
+++ b/v2.2/docs/platforms/openshift/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/platforms/rancher/index.html b/v2.2/docs/platforms/rancher/index.html
index f5098eba0..0f1562db9 100644
--- a/v2.2/docs/platforms/rancher/index.html
+++ b/v2.2/docs/platforms/rancher/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/prerequisites/etcd/index.html b/v2.2/docs/prerequisites/etcd/index.html
index 8ebb887a4..d93f1a91a 100644
--- a/v2.2/docs/prerequisites/etcd/index.html
+++ b/v2.2/docs/prerequisites/etcd/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/prerequisites/firewalls/index.html b/v2.2/docs/prerequisites/firewalls/index.html
index 16435eadc..da958ffbf 100644
--- a/v2.2/docs/prerequisites/firewalls/index.html
+++ b/v2.2/docs/prerequisites/firewalls/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/prerequisites/index.html b/v2.2/docs/prerequisites/index.html
index a42e0bbab..7ba3e373b 100644
--- a/v2.2/docs/prerequisites/index.html
+++ b/v2.2/docs/prerequisites/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/prerequisites/mountpropagation/index.html b/v2.2/docs/prerequisites/mountpropagation/index.html
index e3bfe6814..c4f01ff5d 100644
--- a/v2.2/docs/prerequisites/mountpropagation/index.html
+++ b/v2.2/docs/prerequisites/mountpropagation/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/prerequisites/systemconfiguration/index.html b/v2.2/docs/prerequisites/systemconfiguration/index.html
index 888c957cd..5154710f5 100644
--- a/v2.2/docs/prerequisites/systemconfiguration/index.html
+++ b/v2.2/docs/prerequisites/systemconfiguration/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/api/index.html b/v2.2/docs/reference/api/index.html
index 0a2153247..9148664e0 100644
--- a/v2.2/docs/reference/api/index.html
+++ b/v2.2/docs/reference/api/index.html
@@ -189,7 +189,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/bundle/index.html b/v2.2/docs/reference/bundle/index.html
index 54879982a..eafe991dd 100644
--- a/v2.2/docs/reference/bundle/index.html
+++ b/v2.2/docs/reference/bundle/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/apply/index.html b/v2.2/docs/reference/cli/apply/index.html
index 08475eee3..3ec858629 100644
--- a/v2.2/docs/reference/cli/apply/index.html
+++ b/v2.2/docs/reference/cli/apply/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/attach/index.html b/v2.2/docs/reference/cli/attach/index.html
index a05ed5915..99182d2ab 100644
--- a/v2.2/docs/reference/cli/attach/index.html
+++ b/v2.2/docs/reference/cli/attach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/create/index.html b/v2.2/docs/reference/cli/create/index.html
index 950b1e10b..1d17df7d6 100644
--- a/v2.2/docs/reference/cli/create/index.html
+++ b/v2.2/docs/reference/cli/create/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/delete/index.html b/v2.2/docs/reference/cli/delete/index.html
index dc1268cf9..2494e40f0 100644
--- a/v2.2/docs/reference/cli/delete/index.html
+++ b/v2.2/docs/reference/cli/delete/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/describe/index.html b/v2.2/docs/reference/cli/describe/index.html
index 876b2b68f..c7bcf12f5 100644
--- a/v2.2/docs/reference/cli/describe/index.html
+++ b/v2.2/docs/reference/cli/describe/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/detach/index.html b/v2.2/docs/reference/cli/detach/index.html
index 99980392b..a1ec9d438 100644
--- a/v2.2/docs/reference/cli/detach/index.html
+++ b/v2.2/docs/reference/cli/detach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/get/index.html b/v2.2/docs/reference/cli/get/index.html
index b15ac581e..070bb7c1a 100644
--- a/v2.2/docs/reference/cli/get/index.html
+++ b/v2.2/docs/reference/cli/get/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/help/index.html b/v2.2/docs/reference/cli/help/index.html
index 5025267bc..66dbeadca 100644
--- a/v2.2/docs/reference/cli/help/index.html
+++ b/v2.2/docs/reference/cli/help/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/index.html b/v2.2/docs/reference/cli/index.html
index d7693db3e..05cbe3140 100644
--- a/v2.2/docs/reference/cli/index.html
+++ b/v2.2/docs/reference/cli/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cli/version/index.html b/v2.2/docs/reference/cli/version/index.html
index 7f64cb5ac..70a070a7a 100644
--- a/v2.2/docs/reference/cli/version/index.html
+++ b/v2.2/docs/reference/cli/version/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cluster-operator/configuration/index.html b/v2.2/docs/reference/cluster-operator/configuration/index.html
index cc26e4508..80c1c2e8c 100644
--- a/v2.2/docs/reference/cluster-operator/configuration/index.html
+++ b/v2.2/docs/reference/cluster-operator/configuration/index.html
@@ -30,7 +30,7 @@
-
+
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -1151,6 +1151,11 @@ StorageOSCluster Resource Confi
false |
+images.apiManagerContainer |
+StorageOS API Manager container image |
+storageos/api-manager:v1.0.0 |
+
+
images.csiClusterDriverRegistrarContainer |
CSI Cluster Driver Registrar Container image |
quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 |
@@ -1183,7 +1188,7 @@ StorageOSCluster Resource Confi
images.initContainer |
StorageOS init container image |
-storageos/init:1.0.0 |
+storageos/init:2.1.0 |
images.kubeSchedulerContainer |
@@ -1296,6 +1301,11 @@ StorageOSCluster Resource Confi
/var/lib/kubelet/plugins/kubernetes.io~storageos |
+storageClassName |
+The name of the default StorageClass created for StorageOS volumes |
+fast |
+
+
tlsEtcdSecretRefName |
Secret containing etcd client certificates |
|
diff --git a/v2.2/docs/reference/cluster-operator/examples/index.html b/v2.2/docs/reference/cluster-operator/examples/index.html
index 8320744d7..6d66b228a 100644
--- a/v2.2/docs/reference/cluster-operator/examples/index.html
+++ b/v2.2/docs/reference/cluster-operator/examples/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cluster-operator/index.html b/v2.2/docs/reference/cluster-operator/index.html
index 2a221a4d0..36559017a 100644
--- a/v2.2/docs/reference/cluster-operator/index.html
+++ b/v2.2/docs/reference/cluster-operator/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cluster-operator/index.xml b/v2.2/docs/reference/cluster-operator/index.xml
index da707b9ed..89c89387d 100644
--- a/v2.2/docs/reference/cluster-operator/index.xml
+++ b/v2.2/docs/reference/cluster-operator/index.xml
@@ -108,6 +108,11 @@
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -140,7 +145,7 @@
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -253,6 +258,11 @@
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2.2/docs/reference/cluster-operator/install/index.html b/v2.2/docs/reference/cluster-operator/install/index.html
index 313e9dbe1..1436af3a2 100644
--- a/v2.2/docs/reference/cluster-operator/install/index.html
+++ b/v2.2/docs/reference/cluster-operator/install/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/cluster-operator/upgrade/index.html b/v2.2/docs/reference/cluster-operator/upgrade/index.html
index 9e83f647b..e9d0be158 100644
--- a/v2.2/docs/reference/cluster-operator/upgrade/index.html
+++ b/v2.2/docs/reference/cluster-operator/upgrade/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/contributing/index.html b/v2.2/docs/reference/contributing/index.html
index 3e8b81f12..66e81a53c 100644
--- a/v2.2/docs/reference/contributing/index.html
+++ b/v2.2/docs/reference/contributing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/filesystems/index.html b/v2.2/docs/reference/filesystems/index.html
index b3bbdf948..055d259b5 100644
--- a/v2.2/docs/reference/filesystems/index.html
+++ b/v2.2/docs/reference/filesystems/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/gui/index.html b/v2.2/docs/reference/gui/index.html
index 1a729e743..c5e8f94e3 100644
--- a/v2.2/docs/reference/gui/index.html
+++ b/v2.2/docs/reference/gui/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/index.html b/v2.2/docs/reference/index.html
index cc31bc119..9839fc88b 100644
--- a/v2.2/docs/reference/index.html
+++ b/v2.2/docs/reference/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/init-container/index.html b/v2.2/docs/reference/init-container/index.html
index 4b499e24e..9fcb9d76b 100644
--- a/v2.2/docs/reference/init-container/index.html
+++ b/v2.2/docs/reference/init-container/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/labels/index.html b/v2.2/docs/reference/labels/index.html
index aabe8eaca..13b0e668c 100644
--- a/v2.2/docs/reference/labels/index.html
+++ b/v2.2/docs/reference/labels/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/licence/index.html b/v2.2/docs/reference/licence/index.html
index fec43d37e..a584f8d88 100644
--- a/v2.2/docs/reference/licence/index.html
+++ b/v2.2/docs/reference/licence/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/open_source_attribution/index.html b/v2.2/docs/reference/open_source_attribution/index.html
index b36d66a77..124056cab 100644
--- a/v2.2/docs/reference/open_source_attribution/index.html
+++ b/v2.2/docs/reference/open_source_attribution/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/release_notes/index.html b/v2.2/docs/reference/release_notes/index.html
index 461ba94cd..f47d282dc 100644
--- a/v2.2/docs/reference/release_notes/index.html
+++ b/v2.2/docs/reference/release_notes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/resource_requests_and_limits/index.html b/v2.2/docs/reference/resource_requests_and_limits/index.html
index d50c4dfce..9ca7d40ac 100644
--- a/v2.2/docs/reference/resource_requests_and_limits/index.html
+++ b/v2.2/docs/reference/resource_requests_and_limits/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/telemetry/index.html b/v2.2/docs/reference/telemetry/index.html
index 1e3857239..696f402f6 100644
--- a/v2.2/docs/reference/telemetry/index.html
+++ b/v2.2/docs/reference/telemetry/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/reference/tolerations/index.html b/v2.2/docs/reference/tolerations/index.html
index c8885d2bb..f4f3e9bfb 100644
--- a/v2.2/docs/reference/tolerations/index.html
+++ b/v2.2/docs/reference/tolerations/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/self-eval/index.html b/v2.2/docs/self-eval/index.html
index d99d57d28..e009aefd8 100644
--- a/v2.2/docs/self-eval/index.html
+++ b/v2.2/docs/self-eval/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/support/index.html b/v2.2/docs/support/index.html
index 5e45eacda..1c2bf61af 100644
--- a/v2.2/docs/support/index.html
+++ b/v2.2/docs/support/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/cassandra/index.html b/v2.2/docs/usecases/cassandra/index.html
index b4eb92f25..622262903 100644
--- a/v2.2/docs/usecases/cassandra/index.html
+++ b/v2.2/docs/usecases/cassandra/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/elasticsearch/index.html b/v2.2/docs/usecases/elasticsearch/index.html
index 5b71664e3..13f7f7c5b 100644
--- a/v2.2/docs/usecases/elasticsearch/index.html
+++ b/v2.2/docs/usecases/elasticsearch/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/index.html b/v2.2/docs/usecases/index.html
index 456d9829f..84dccae4d 100644
--- a/v2.2/docs/usecases/index.html
+++ b/v2.2/docs/usecases/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/influxdb/index.html b/v2.2/docs/usecases/influxdb/index.html
index 246a29647..7a146fe81 100644
--- a/v2.2/docs/usecases/influxdb/index.html
+++ b/v2.2/docs/usecases/influxdb/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/jenkins/index.html b/v2.2/docs/usecases/jenkins/index.html
index 56e5cab99..349a28e6e 100644
--- a/v2.2/docs/usecases/jenkins/index.html
+++ b/v2.2/docs/usecases/jenkins/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/kafka/index.html b/v2.2/docs/usecases/kafka/index.html
index 6d438097c..94650e654 100644
--- a/v2.2/docs/usecases/kafka/index.html
+++ b/v2.2/docs/usecases/kafka/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/kubevirt/index.html b/v2.2/docs/usecases/kubevirt/index.html
index 6b4481c2f..b62eb5464 100644
--- a/v2.2/docs/usecases/kubevirt/index.html
+++ b/v2.2/docs/usecases/kubevirt/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/mssql/index.html b/v2.2/docs/usecases/mssql/index.html
index 6840b4665..031763dbc 100644
--- a/v2.2/docs/usecases/mssql/index.html
+++ b/v2.2/docs/usecases/mssql/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/mysql/index.html b/v2.2/docs/usecases/mysql/index.html
index 44bf9afb3..3583017f4 100644
--- a/v2.2/docs/usecases/mysql/index.html
+++ b/v2.2/docs/usecases/mysql/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/nginx/index.html b/v2.2/docs/usecases/nginx/index.html
index 41a9d6505..8d3c934b3 100644
--- a/v2.2/docs/usecases/nginx/index.html
+++ b/v2.2/docs/usecases/nginx/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/postgres/index.html b/v2.2/docs/usecases/postgres/index.html
index b2791427a..91566a232 100644
--- a/v2.2/docs/usecases/postgres/index.html
+++ b/v2.2/docs/usecases/postgres/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/prometheus/index.html b/v2.2/docs/usecases/prometheus/index.html
index 6f9f2cc7f..5a6eb9ca8 100644
--- a/v2.2/docs/usecases/prometheus/index.html
+++ b/v2.2/docs/usecases/prometheus/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/redis/index.html b/v2.2/docs/usecases/redis/index.html
index 4fef679a4..3020244b7 100644
--- a/v2.2/docs/usecases/redis/index.html
+++ b/v2.2/docs/usecases/redis/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/sidecar-backups/index.html b/v2.2/docs/usecases/sidecar-backups/index.html
index 301f740dc..7e4ae6da0 100644
--- a/v2.2/docs/usecases/sidecar-backups/index.html
+++ b/v2.2/docs/usecases/sidecar-backups/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/velero-backups/index.html b/v2.2/docs/usecases/velero-backups/index.html
index 287744ba3..7bdbc84ca 100644
--- a/v2.2/docs/usecases/velero-backups/index.html
+++ b/v2.2/docs/usecases/velero-backups/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/docs/usecases/zookeeper/index.html b/v2.2/docs/usecases/zookeeper/index.html
index 3c89896bc..e29dd13e3 100644
--- a/v2.2/docs/usecases/zookeeper/index.html
+++ b/v2.2/docs/usecases/zookeeper/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/index.html b/v2.2/index.html
index c5d936ea1..3a7a5e180 100644
--- a/v2.2/index.html
+++ b/v2.2/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/index.json b/v2.2/index.json
index dcc527985..ea6838966 100644
--- a/v2.2/index.json
+++ b/v2.2/index.json
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json b/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json
new file mode 100644
index 000000000..aecee68cd
--- /dev/null
+++ b/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json
@@ -0,0 +1 @@
+[{"body":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher OpenShift ","excerpt":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher …","ref":"/v2.2/docs/reference/cluster-operator/install/","title":"Install"},{"body":"Current Version The current StorageOS version is v2.2.0\nSections Our documentation is arranged into sections, accessible from the navigation bar on the left.\nIntroduction - Quickstart and support information.\nConcepts - Architectural and deep technical information.\nPrerequisites - We require certain prerequisites to be met for the product to function correctly. Please read these carefully and ensure they are implemented.\nPlatforms - Due to differences in the various orchestrators that StorageOS can run under, we list install guides and other platform specific operations here.\nOperations - Platform agnostic operations.\nUse Cases - A set of examples to get up and running with StorageOS quickly.\nReference - Information on our GUI, CLI, and other important information.\n","excerpt":"Current Version The current StorageOS version is v2.2.0\nSections Our documentation is arranged into …","ref":"/v2.2/docs/introduction/","title":"Introduction"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed service such as EKS, AKS, GKE, DO or DockerEE platform can use the following Kubernetes guide to install StorageOS.\n StorageOS v2 uses CSI only. The native driver has been deprecated.\n Make sure to add a StorageOS licence after installing.\n 1.19 1.18 1.17 1.16 1.15 1.14 1.13 Install StorageOS on Kubernetes 1.19 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.18 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.17 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.16 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.15 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.14 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.13 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed …","ref":"/v2.2/docs/install/kubernetes/","title":"Kubernetes"},{"body":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some significant enhancements over our v1 product. We\u0026rsquo;ve built V2 based on our observations of trends in the industry, as well as our own experience.\nMany of our customers want to run big clusters - in the tens or hundreds of nodes. In these sorts of big environments, the challenges multiply. Not only do we need to scale well, but we also need to be more failure tolerant. Bigger environments typically suffer higher failure rates (more nodes = greater chance of something failing), but are also subject to all sorts of transient conditions such as network partitions.\nThe second trend we\u0026rsquo;ve seen become increasingly common is the desire to run multiple clusters, and consume storage between them in some way - sometimes to implement novel topologies such as a centralised storage cluster with satellites consuming the storage, and sometimes to replicate data between those clusters for HA or DR purposes.\nWe\u0026rsquo;ve built V2 with these architectures and design patterns in mind. Not only does it scale well, but it contains the foundations we need to implement a rich set of multi-cluster functionality.\nUpgraded Control Plane At the heart of the V2 release is an upgraded control plane. We\u0026rsquo;ve changed a lot here. Firstly, our usage of etcd is vastly improved. We\u0026rsquo;ve learnt a lot about the subtleties of distributed consensus in the last year, particularly in noisy or unpredictable environments. Not only is StorageOS V2 much lighter on your etcd cluster, but it\u0026rsquo;s a lot more tolerant of transient failure conditions that are often found in cloud environments, or clusters under heavy load.\nWe spent some time describing and testing our internal state machine using the TLA+ formal verification language. This allows us to have a much higher degree of confidence that our algorithms will behave correctly, particularly under hard-to-test edge cases and failure conditions.\nAdditionally, we\u0026rsquo;ve changed the way volumes behave with respect to centralised scheduling. Each volume group (consisting of a master and 0 or more replicas) now behaves as a strongly consistent unit allowing it to take action independent of the activities of the rest of the cluster. Other state can be distributed around the cluster using eventually consistent mechanisms. This approach inherently scales better and allows StorageOS V2 to effectively manage many more nodes and volumes than before.\nWe\u0026rsquo;ve implemented TLS on all endpoints. Not only does this give you encrypted traffic between nodes in your storage cluster, it also protects all endpoints with strong, public key based authentication. Today\u0026rsquo;s IT environments can\u0026rsquo;t rely on firewalls to keep bad actors out - they must implement security at all layers within the stack - defense in depth. While we recognise that this brings a welcome relief to many security conscious administrators, we also know that managing certificate authorities (CAs) can be an unwelcome source of complexity. For this reason, StorageOS V2 implements an internal CA by default, to manage this complexity for you. If you\u0026rsquo;d prefer to integrate your own CA, we support that too - it\u0026rsquo;s up to you.\nFinally - our logging has undergone a complete transformation in this edition. We know that systems engineers and operators don\u0026rsquo;t just value headline features, but that observability and diagnostics are equally important. All logs are now decorated with rich context to help you understand what is happening within your cluster, and we\u0026rsquo;ll output in json by default, for easy ingestion into log aggregators such as Elasticsearch.\nUpgraded Data Plane Not to be outdone, our data plane contains some significant improvements.\nFirstly, we\u0026rsquo;ve completely re-written our sync algorithm (see Delta Sync, used when seeding or catching up replicas that have been offline or partitioned. Our new algorithm uses a Hash List to sync only changed sections of a volume (similar in some ways to what rsync does). StorageOS maintains these hashes during normal operation, meaning that when resyncing a failed replica, for example after a node reboot, we can very quickly and efficiently catch this replica up, rather than needing to promote and build a new one from scratch. This improves resiliency within your cluster, and prevents using excessive network bandwidth during failover conditions - at a time when it might be needed the most.\nSecondly, a new threading model, with dynamic pool sizing, means that StorageOS is faster, a lot faster. In our tests we observed improvements across the board, with improvements in throughput of up to 135% for some scenarios.\n","excerpt":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some …","ref":"/v2.2/docs/introduction/overview/","title":"What is StorageOS V2"},{"body":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source implementation of the SCSI target.\nWe require the following modules to be loaded:\n target_core_mod tcm_loop target_core_file configfs target_core_user uio N.B. Other applications utilising TCMU cannot be run concurrently with StorageOS. Doing so may result in corruption of data. On startup, StorageOS will detect if other applications are using TCMU and fall back to FUSE. However if StorageOS is started first there is no mechanism for StorageOS to fallback to FUSE if another application begins to use TCMU. TCMU can be disabled using the DISABLE_TCMU StorageOSCluster spec parameter.\n Depending on the distribution, the modules are shipped as part of the base kernel package or as part of a kernel extras package which needs to be installed.\nDistribution Specifics The following distributions are supported by default:\n RHEL 7.5 CentOS 7 Debian 9 Ubuntu Azure RancherOS - Note CSI is not supported on RancherOS Ubuntu 16.04/18.04 requires the installation of additional packages.\n N.B. Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not provide the necessary linux-image-extra package - see below for more information\n Ubuntu Package Installation Ubuntu 16.04/18.04 Generic and Ubuntu 16.04 GCE require extra packages:\nUbuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Ubuntu With AWS Or GCE Kernels Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not yet provide the linux-image-extra package. As such you should either use Debian, CentOS or RHEL, or install the non-cloud-provider optimised Ubuntu kernel.\nInstalling the non-cloud-provider optimised Ubuntu kernel is something that should only be done with full understanding of potential ramifications.\nsudo apt -y update sudo apt install -y linux-virtual linux-image-extra-virtual sudo apt purge -y linux*aws # Reboot the machine sudo shutdown -r now Automatic Configuration Once required kernel modules are installed on the system, for convenience we provide a container which will ensure the appropriate modules are loaded and ready for use at runtime. On Docker installations, you will need to run the init container prior to starting StorageOS. Our installation guides for Kubernetes and OpenShift include this step.\n# Load the required kernel modules. The Kubernetes and OpenShift installations include this step. docker run --name enable_lio \\ --privileged \\ --rm \\ --cap-add=SYS_ADMIN \\ -v /lib/modules:/lib/modules \\ -v /sys:/sys:rshared \\ storageos/init:0.2 Manual Configuration For those wishing to manage their own kernel configuration, rather than using the init container, perform the following steps:\n Ensure kernel modules are all loaded per list above Ensure configfs is loaded and mounted at /sys/kernel/config ","excerpt":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source …","ref":"/v2.2/docs/prerequisites/systemconfiguration/","title":"System Configuration"},{"body":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters of the StorageOSCluster custom resource and their default values.\n Parameter Description Default csi.deploymentStrategy CSI helper deployment strategy (statefulset or deployment) statefulset csi.enable Enable CSI setup false csi.enableControllerPublishCreds Enable CSI controller publish credentials false csi.enableNodePublishCreds Enable CSI node publish credentials false csi.enableProvisionCreds Enable CSI provision credentials false debug Enable debug mode for all the cluster nodes false disableFencing Disable Pod fencing false disableScheduler Disable StorageOS scheduler false disableTCMU Disable TCMU to allow co-existence with other TCMU users. Disabling TCMU degrades performance false disableTelemetry Disable telemetry reports false forceTCMU Forces TCMU to be enabled or causes StorageOS to abort startup false images.apiManagerContainer StorageOS API Manager container image storageos/api-manager:v1.0.0 images.csiClusterDriverRegistrarContainer CSI Cluster Driver Registrar Container image quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 images.csiExternalAttacherContainer CSI External Attacher Container image quay.io/k8scsi/csi-attacher:v1.0.1 images.csiExternalProvisionerContainer CSI External Provisioner Container image storageos/csi-provisioner:v1.0.1 ìmages.csiLivenessProbeContainer CSI Liveness Probe Container Image quay.io/k8scsi/livenessprobe:v1.0.1 images.csiNodeDriverRegistrarContainer CSI Node Driver Registrar Container image quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 images.hyperkubeContainer Deprecated field - HyperKube Container image Default dependent on Scheduler version images.initContainer StorageOS init container image storageos/init:2.1.0 images.kubeSchedulerContainer Kube scheduler container image Default dependent on Scheduler version images.nfsContainer StorageOS nfs container image storageos/nfs:1.0.0 images.nodeContainer StorageOS node container image storageos/node:v2.2.0 ingress.annotations Annotations of the ingress used by the cluster ingress.enable Enable ingress for the cluster false ingress.hostname Hostname to be used in cluster ingress storageos.local ingress.tls Enable TLS for the ingress false k8sDistro The name of the Kubernetes distribution is use, e.g. rancher or eks kvBackend.address Comma-separated list of addresses of external key-value store. (1.2.3.4:2379,2.3.4.5:2379) kvBackend.backend (v2 deprecated) Name of the key-value store to use. Set to etcd for external key-value store. embedded namespace Namespace where storageos cluster resources are created kube-system nodeSelectorTerms Set node selector for storageos pod placement pause Pause the operator for cluster maintenance false resources Set resource requirements for the containers secretRefName Reference name of storageos secret secretRefNamespace Namespace of storageos secret service.annotations Annotations of the Service used by the cluster service.externalPort External port of the Service used by the cluster 5705 service.internalPort Internal port of the Service used by the cluster 5705 service.name Name of the Service used by the cluster storageos service.type Type of the Service used by the cluster ClusterIP sharedDir Path to be shared with kubelet container when deployed as a pod /var/lib/kubelet/plugins/kubernetes.io~storageos storageClassName The name of the default StorageClass created for StorageOS volumes fast tlsEtcdSecretRefName Secret containing etcd client certificates tlsEtcdSecretRefNamespace Namespace of the tlsEtcdSecretRefName tolerations Set pod tolerations for storageos pod placement ","excerpt":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters …","ref":"/v2.2/docs/reference/cluster-operator/configuration/","title":"Configuration"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment of an etcd cluster.\n For OpenShift upgrades, refer to the OpenShift platform page.\n If you have installed OpenShift in AWS ensure that the requisite ports are opened for the worker nodes\u0026rsquo; security group.\n Make sure to add a StorageOS licence after installing.\n StorageOS v2 supports OpenShift v4. For more information check the OpenShift platform page.\n OperatorHub Manual OperatorHub Select the OperatorHub from the Catalog sub menu and search for StorageOS\n Choose between using the RedHat Market Place or the Community Operators installation.\n Select StorageOS and click install\n Select the install options\n Make sure the Approval Strategy is set to Manual. So the StorageOS Operator doesn\u0026rsquo;t upgrade versions without explicit approval.\n Start the approval procedure\n Follow the approval link\n Approve the installation\n The StorageOS Cluster Operator is installed along the required CRDs\n Create a Secret in the openshift-operators project\n Use the YAML options to create a secret containing the apiUsername and an apiPassword key. The username and password defined in the secret will be used to authenticate when using the StorageOS CLI and GUI. For the communication between StorageOS and OpenShift, the CSI credentials csiProvisionUsername, csiProvisionPassword, csiControllerPublishUsername, csiControllerPublishPassword, csiNodePublishUsername, csiNodePublishPassword are needed. Take note of which project you created the secret in.\nInput the Secret as YAML for simplicity.\napiVersion:v1kind:Secretmetadata:name:storageos-apinamespace:openshift-operatorstype:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9z Go to the \u0026ldquo;Installed Operators\u0026rdquo;\n Verify that the StorageOS Cluster Operator is installed\n Go to the \u0026ldquo;StorageOS Cluster\u0026rdquo; section\n Create the StorageOS Cluster\n A StorageOS Cluster is defined using a Custom Resource Definition\n Create the Custom Resource\nThe StorageOS cluster resource describes the StorageOS cluster that will be created. Parameters such as the secretRefName, the secretRefNamespace and the kvBackend.address are mandatory.\nAdditional spec parameters are available on the Cluster Operator configuration page.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: storageos namespace: openshift-operators spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference the Secret created in the previous step secretRefNamespace: \u0026#34;openshift-operators\u0026#34; # Namespace of the Secret created in the previous step k8sDistro: \u0026#34;openshift\u0026#34; kvBackend: address: \u0026#39;storageos-etcd-client.etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Verify that the StorageOS Cluster Resource enters a running state.\n It can take up to a minute to report the StorageOS Pods ready\n Check the StorageOS Pods in the kube-system project\n A Status of 3/3 for the Daemonset Pods indicates that StorageOS is bootstrapped successfully.\n License cluster\n A newly installed StorageOS cluster does not include a licence. A cluster must be licensed within 24 hours of the installation. For more information, check the reference licence page.\n You can apply a Free Developer licence following the operations licensing page, or purchase a licence contacting sales@storageos.com.\n Manual install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment …","ref":"/v2.2/docs/install/openshift/","title":"OpenShift"},{"body":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all nodes in the cluster:\n Port Number TCP/UDP Use 5701 TCP gRPC 5703 TCP DirectFS 5704 TCP Dataplane Supervisor 5705 TCP REST API 5711 TCP \u0026amp; UDP Gossip service StorageOS also uses ephemeral ports to dial-out to these ports on other StorageOS nodes. For this reason, outgoing traffic should be enabled.\nFirewalls and VPS providers Some VPS providers (such as Digital Ocean) ship default firewall rulesets which must be updated to allow StorageOS to run. Some example rules are shown below - modify to taste.\nUFW For distributions using UFW, such as RHEL and derivatives:\nufw default allow outgoing ufw allow 5701:5711/tcp ufw allow 5711/udp Firewalld For distributions that enable firewalld to control iptables such as some installations of OpenShift.\nfirewall-cmd --permanent --new-service=storageos firewall-cmd --permanent --service=storageos --add-port=5700-5800/tcp firewall-cmd --add-service=storageos --zone=public --permanent firewall-cmd --reload Iptables For those using plain iptables:\n# Inbound traffic iptables -I INPUT -i lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I INPUT -m state --state ESTABLISHED,RELATED -m comment --comment \u0026#39;Permit established traffic\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 5701:5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p udp --dport 5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT # Outbound traffic iptables -I OUTPUT -o lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I OUTPUT -d 0.0.0.0/0 -m comment --comment \u0026#39;Permit outbound traffic\u0026#39; -j ACCEPT Please ensure that the iptables rules you have added above come before any default DROP or REJECT rules.\n","excerpt":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all …","ref":"/v2.2/docs/prerequisites/firewalls/","title":"Firewalls"},{"body":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, OpenShift or Rancher.\n Dive into our Use Case documentation for Workloads deployed in Kubernetes.\n ","excerpt":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, …","ref":"/v2.2/docs/introduction/quickstart/","title":"Quickstart"},{"body":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and Password in base64 encoding.\nkubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;default\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z END This example contains a default password, for production installations, use a unique, strong password.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Create a cluster-config.yaml according to your needs from the examples below.\nkubectl create -f cluster-config.yaml Note that StorageOS will be deployed in spec.namespace (storageos by default), irrespective of what NameSpace the CR is defined in.\n Examples You can checkout all the parameters configurable in the configuration page.\n All examples must reference the storageos-api Secret.\nspec:secretRefName:\u0026#34;storageos-api\u0026#34;# Reference to the Secret created in the previous stepsecretRefNamespace:\u0026#34;storageos-operator\u0026#34;# Namespace of the SecretCheck out Cluster Definition examples for full CR files.\nInstalling with an external etcd spec:kvBackend:address:\u0026#39;10.43.93.95:2379\u0026#39;# IP of the SVC that exposes ETCD# address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can specify individual IPs of the etcd serversIf using Etcd with mTLS, you need to specify the secret that hold the certificates with the following parameters:\nspec:# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;storageos-etcd\u0026#34;# Namespace of the client certificates secretFollow the etcd operations page to setup the secret with the Etcd client certificate, client key and CA.\nInstalling to a subset of nodes In this case we select nodes that are workers. To make sure that StorageOS doesn\u0026rsquo;t start in Master nodes.\nYou can see the labels in the nodes by kubectl get node --show-labels.\nspec:nodeSelectorTerms:- matchExpressions:- key:\u0026#34;node-role.kubernetes.io/worker\u0026#34;operator:Invalues:- \u0026#34;true\u0026#34;# OpenShift uses \u0026#34;node-role.kubernetes.io/compute=true\u0026#34;# Rancher uses \u0026#34;node-role.kubernetes.io/worker=true\u0026#34;# Kops uses \u0026#34;node-role.kubernetes.io/node=\u0026#34; Different provisioners and Kubernetes distributions use node labels differently to specify master vs workers. Node Taints are not enough to make sure StorageOS doesn\u0026rsquo;t start in a node. The JOIN variable is defined by the operator by selecting all the nodes that match the nodeSelectorTerms.\n Enabling CSI spec:csi:enable:truedeploymentStrategy:deploymentenableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:trueenableControllerExpandCreds:trueThe credentials must be defined in the storageos-api Secret\napiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# Add base64 encoded creds below for CSI credentials.csiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9zSpecifying a shared directory for use with kubelet as a container spec:sharedDir:\u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39;Defining pod resource requests and reservations spec:resources:requests:memory:\u0026#34;512Mi\u0026#34;# cpu: \u0026#34;1\u0026#34;# limits:# memory: \u0026#34;4Gi\u0026#34;Limiting StorageOS can cause malfunction for IO to StorageOS volumes, therefore we do not currently recommend applying upper limits to resources for StorageOS pods.\nSpecifying custom Tolerations spec:tolerations:- key:\u0026#34;key1\u0026#34;operator:\u0026#34;Equal\u0026#34;value:\u0026#34;value1\u0026#34;effect:\u0026#34;EffectToTolerate\u0026#34;- key:\u0026#34;key2\u0026#34;operator:\u0026#34;Exists\u0026#34;Custom tolerations specified in the StorageOSCluster definition are added to all StorageOS components; the StorageOS daemonset, CSI helper and scheduler.\nIn the above example a toleration key1=value1:EffectToTolerate would be tolerated and key2 would be tolerated regardless of the value and effect. For more information about tolerations please see the Kubernetes documentation.\n","excerpt":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and …","ref":"/v2.2/docs/reference/cluster-operator/examples/","title":"Cluster Operator examples"},{"body":" Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by default.\n StorageOS requires mount propagation enabled to present devices as volumes for containers (see linux kernel documentation here).\nCertain versions of docker ship with a systemd manifest with MountFlags set to \u0026lsquo;slave\u0026rsquo;, thus preventing StorageOS from working. This can be removed or set to \u0026lsquo;shared\u0026rsquo; with a systemd drop in:\nmkdir -p /etc/systemd/system/docker.service.d/ cat \u0026lt;\u0026lt;EOF \u0026gt; /etc/systemd/system/docker.service.d/mount_propagation_flags.conf [Service] MountFlags=shared EOF # systemctl daemon-reload # systemctl restart docker.service To confirm behaviour, the following command should run without error.\ndocker run -it --rm -v /mnt:/mnt:shared busybox sh -c /bin/date Orchestrators such as Kubernetes or OpenShift have their own ways of exposing this setting. Kubernetes 1.10 and OpenShift 3.10 have mount propagation enabled by default. Previous versions require that feature gates are enabled on the Kubernetes master\u0026rsquo;s controller-manager and apiserver services and in the kubelet service on each node.\nInstallations of orchestrators using Docker require that mount propagation is enabled for both.\nRefer to our installation pages for the orchestrators to see details on how to check and enable mount propagation where appropriate.\n","excerpt":"Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by …","ref":"/v2.2/docs/prerequisites/mountpropagation/","title":"Mount Propagation"},{"body":"StorageOS is a certified Rancher application. We offer two installation methods:\n Rancher Catalogue - this is the easiest and requires just a few clicks Manual - allowing more control and visibility Before proceeding, ensure that you have followed our prerequisites. On Rancher, pay particular attention to the OS version and image used - some platforms require extra mainline kernel modules to be enabled.\n Catalog Manual Catalog Install StorageOS is a Certified application in the Rancher Catalog. You can install StorageOS using the Rancher application install.\nBefore completing the steps below, you will need an etcd cluster. For evaluations it is sufficient to use our simple test recipe. For production installations, please follow our production recipe. Make a note of the etcd endpoint URL in either case.\n Select the System project of your cluster\n Select the Apps tab and click Launch\n Search for StorageOS and click on the App\nThis will install the StorageOS operator, which manages the StorageOS DaemonSet.\n Check and ammend installation options\nA generic configuration for StorageOS is preset using the default values in the form. Be sure to check the etcd address and ensure it matches the value you noted at the beginning of this guide.\nThe catalog form exposes several useful parameters - documented below.\nFor further customization, you can opt to set the option to \u0026lsquo;Install StorageOS Cluster\u0026rsquo; to false and install a custom CR. See below for this.\n Launch the StorageOS cluster\n Verify the cluster bootstrap has successfully completed\n License the newly installed cluster\nNewly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nYou will need access to the StorageOS API on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 Now follow the instructions on our licensing operations page to obtain and apply a license.\nInstallation of StorageOS is now complete.\n Simple Customization - Modify Catalog Form The following options are exposed by the catalog form to allow some simple customization of the StorageOS installation.\n Cluster Operator namespace : The Kubernetes namespace where the StorageOS Cluster Operator and other resources will be created. Container Images : By default images are pulled from DockerHub, you can specify the image URLs when using private registries. Install StorageOS cluster : Controls the automatic deployment of StorageOS after installing the Cluster Operator. If set to false, the Operator will be created, but a Custom Resource will not be applied to the cluster. Launch the operator and proceed to the section Advanced Customization below. Namespace : The Kubernetes namespace where StorageOS will be installed. By default, StorageOS installs into the kube-system namespace, which will add a priority class to ensure high priority resource allocation. Installing StorageOS with the priority class prevents StorageOS from being evicted during periods of resource contention. It is inadvisable to modify this under normal circumstances. Username/Password : Default Username and Password for the admin account to be created at StorageOS bootstrap. A random password will be generated by leaving the field empty or clicking the Generate button. External etcd address(es) : Connection and configuration details for an external Etcd cluster.See our documentation here. Node Selectors and Tolerations : Control placement of StorageOS DaemonSet Pods. StorageOS will only be installed on the selected nodes. Tolerations : Define any tolerations you wish the DaemonSet to observe. Advanced Customization - Apply Custom CR If Install StorageOS Cluster was set to false, StorageOS will not be bootstrapped automatically. After the StorageOS Operator is installed, you can now create a Custom Resource that describes the StorageOS cluster.\n Select the System Workloads and Import YAML Create the Secret and CustomResource This is an example.\n--- apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z --- apiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container resources: requests: memory: \u0026#34;512Mi\u0026#34; nodeSelectorTerms: - matchExpressions: - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation operator: In values: - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Manual Installation The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.2.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"StorageOS is a certified Rancher application. We offer two installation methods:\n Rancher Catalogue …","ref":"/v2.2/docs/install/rancher/","title":"Rancher"},{"body":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following yaml manifest.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n Upgrade StorageOS Operator using Helm If you have installed the StorageOS Operator using the Helm Chart, then you can upgrade the operator using the following commands.\n$ helm list NAME REVISION STATUS CHART APP VERSION NAMESPACE storageos-v1 4 DEPLOYED storageos-operator-0.2.11 1.3.0 storageos-operator $ helm repo update $ helm upgrade $NAME storageos/storageos-operator When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n ","excerpt":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following …","ref":"/v2.2/docs/reference/cluster-operator/upgrade/","title":"Cluster Operator Upgrade"},{"body":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 active volumes per node 4.x kernels have a limitation of 4096 active volumes per node We are distribution agnostic as long as our prerequisites are met Orchestrators Kubernetes 1.13+ OpenShift 4.0+ ","excerpt":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 …","ref":"/v2.2/docs/introduction/platforms/","title":"Supported Platforms and Orchestrators"},{"body":"","excerpt":"","ref":"/v2.2/docs/concepts/","title":"Concepts"},{"body":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a 64-bit architecture. Kubernetes 1.9+ with CSI. Container Runtime Engine: CRI-O, Containerd or Docker 1.10+ with mount propagation enabled. The necessary ports should be open. See the ports and firewall settings. Etcd cluster for StorageOS A mechanism for device presentation. Recommended: At least three nodes for replication and high availability. Kubernetes 1.13+. Install the storageos CLI. If using Helm2, make sure the tiller ServiceAccount has enough privileges to create resources such as Namespaces, ClusterRoles, etc. For instance, following this installation procedure. System clocks synchronized using NTP or similar methods. While our distributed consensus algorithm does not require synchronised clocks, it does help to more easily correlate logs across multiple nodes. ","excerpt":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a …","ref":"/v2.2/docs/prerequisites/","title":"Prerequisites"},{"body":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different distributions such as OpenShift, Rancher, EKS, AKS, GKE, etc. The user can provide standard PVC definitions and StorageOS will dynamically provision matching volumes. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nKubernetes and StorageOS communicate with each other to perform actions such as creation, deletion or mounting of volumes. The CSI (Container Storage Interface) driver is the standard method of communication. Using CSI, Kubernetes and StorageOS communicate over a Unix domain socket.\nIt is recommended to use the CSI StorageOS installation from Kubernetes 1.13 onwards. Although the former communication procedure, the native driver, is still in use and StorageOS maintains support for it, its use is discouraged as it is a deprecated method, to be removed in newer versions of Kubernetes.\nCSI (Container Storage Interface) Note CSI is the standard method of communication that enables storage drivers for Kubernetes to release on their own schedule. The CSI standard allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI was released GA from Kubernetes 1.13. StorageOS v2 only supports the use of CSI as a storage driver. In addition, the StorageOS Cluster Operator handles the configuration of the CSI driver and its complexity by detecting the version of the Kubernetes installed.\nCheck out the status of the CSI release cycle in relation with Kubernetes in the CSI project page.\nCSI communication is fully supported by StorageOS if the cluster is deployed with any supported Linux Distribution.\nKubernetes Upgrades on Managed Services Managed services that support in place upgrades are fully supported. However, upgrading Kubernetes using green/blue deployments is not supported. This is because nodes are replaced rather than being upgraded. Any data stored on the nodes is lost when new nodes replace the previous ones.\nDocker Some managed Kubernetes platforms such as Azure AKS, enable the \u0026lsquo;Live-Restore\u0026rsquo; Docker feature, enabling containers to continue running while Docker is stopped or upgraded. This feature can cause nodes to hang while shutting-down or rebooting, as rather than going through an orderly shutdown, StorageOS (and other processes) are killed before the disks are synced and unmounted. Devices in this inaccessible state will log a warning similar to:\nTransport endpoint not connected To prevent this behaviour, we advise disabling this feature by setting\n{ ... \u0026quot;live-restore\u0026quot;: false } in /etc/docker/daemon.json.\nHere\u0026rsquo;s an example Ansible snippet that might be used to achieve this\n... - name: configure /etc/docker/daemon.json lineinfile: path: /etc/docker/daemon.json regexp: '^.*\u0026quot;live-restore\u0026quot;: true,$' line: ' \u0026quot;live-restore\u0026quot;: false,' backrefs: yes notify: restart docker ... Note: Use at own risk; you may need to adapt the example to work in your environment\n ","excerpt":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different …","ref":"/v2.2/docs/install/","title":"Install"},{"body":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to a specific platform.\nStorageOS can be used in any Kubernetes distribution when the prerequisites are met. That includes Kubernetes manged services.\n","excerpt":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to …","ref":"/v2.2/docs/platforms/","title":"Platforms"},{"body":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential cluster metadata and manage distributed configuration state. An embedded etcd instance is included in the StorageOS container, but for production environments and testing of production workloads, we recommend deploying an external etcd cluster. For more details about, and an example of, how to run etcd, see the External etcd Operations page.\nIt is highly recommended to use external etcd for cloud environments and place the etcd cluster on stable nodes. Placing the etcd on nodes that are recycled often might affect the normal operations of StorageOS.\nEtcd low latency IO It is recommended to run etcd on low-latency disks and keep other IO-intensive applications separate from the etcd nodes. Etcd is very sensitive to IO latency. Thus, the effect of disk contention can cause etcd downtime.\nBatch jobs such as backups, builds or application bundling can easily cause a high usage of disks making etcd unstable. It is recommended to run such workloads apart from the etcd servers.\nSetup of storage on the hosts We recommend creating a separate filesystem for StorageOS to mitigate the risk of filling the root filesystem on nodes. This has to be done for each node in the cluster.\nFollow the managing host storage best practices page for more details.\nResource reservations StorageOS resource consumption depends on the workloads and the StorageOS features in use.\nThe recommended minimum memory reservation for the StorageOS Pods is 512MB for non-production environments. However it is recommended to prepare nodes so StorageOS can operate with at least with 1-2GB of memory. StorageOS frees memory when possible.\nFor production environments, we recommend 4GB of Memory and 1 CPU as a minimum and to test StorageOS using realistic workloads and tune resources accordingly.\nStorageOS Pods resource allocation will impact directly on the availability of volumes in case of eviction or resource limit triggered restart. It is recommended to not limit StorageOS Pods.\nStorageOS implements a storage engine, therefore limiting CPU consumption might affect the I/O throughput of your volumes.\nMaintain a sufficient number of nodes for replicas to be created To ensure that a new replica can always be created, an additional node should be available. To guarantee high availability, clusters using Volumes with 1 replica must have at least 3 storage nodes. When using Volumes with 2 replicas, at least 4 storage nodes, 3 replicas, 5 nodes, etc.\nMinimum number of storage nodes = 1 (primary) + N (replicas) + 1\nFor more information, see the section on replication.\nStorageOS API username/password The API grants full access to StorageOS functionality, therefore we recommend that the default administrative password of \u0026lsquo;storageos\u0026rsquo; is reset to something unique and strong.\nYou can change the default parameters by encoding the apiUsername and apiPassword values (in base64) into the storageos-api secret.\nTo generate a unique password, a technique such as the following, which generates a pseudo-random 24 character string, may be used:\n# Generate strong password PASSWORD=$(cat -e /dev/urandom | tr -dc \u0026#39;a-zA-Z0-9-!@#$%^\u0026amp;*()_+~\u0026#39; | fold -w 24 | head -n 1) # Convert password to base64 representation for embedding in a K8S secret BASE64PASSWORD=$(echo -n $PASSWORD | base64) Note that the Kubernetes secret containing a strong password must be created before bootstrapping the cluster. Multiple installation procedures use this Secret to create a StorageOS account when the cluster first starts.\nStorageOS Pod placement StorageOS must run on all nodes that will contribute storage capacity to the cluster or that will host Pods which use StorageOS volumes. For production environments, it is recommended to avoid placing StorageOS Pods on Master nodes.\nStorageOS is deployed with a DaemonSet controller, and therefore tolerates the standard unschedulable (:NoSchedule) action. If that is the only taint placed on master or cordoned nodes StorageOS pods might start on them (see the Kubernetes docs for more details). To avoid scheduling StorageOS pods on master nodes, you can add an arbitrary taint to them for which the StorageOS DaemonSet won\u0026rsquo;t have a toleration.\nDedicated instance groups Cloud environments give users the ability to quickly scale the number of nodes in a cluster in response to their needs. Because of the ephemeral nature of the cloud, StorageOS recommends setting conservative downscaling policies.\nFor production clusters, it recommended to use dedicated instance groups for Stateful applications that allow the user to set different scaling policies and define StorageOS pools based on node selectors to collocate volumes.\nLosing a few nodes at the same time could cause the loss of data even when volume replicas are being used.\nPort blocking StorageOS exposes ports to operate. It is recommended that the ports are not accessible from outside the scope of your cluster.\nStorageOS in Docker EE StorageOS does not support running on Swarm nodes nor on mixed (Kubernetes and Swarm) nodes. StorageOS volumes have to be provisioned and used from Kubernetes nodes.\n","excerpt":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential …","ref":"/v2.2/docs/best-practices/","title":"StorageOS Best Practices"},{"body":"","excerpt":"","ref":"/v2.2/docs/operations/","title":"Operations"},{"body":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or other Orchestrators that are derived from Kubernetes such as OpenShift or Rancher. This is useful for running stateful applications, such as databases or CI/CD applications, under the control of Kubernetes as Kubernetes can make scheduling decisions without the application data being lost.\nWhat we have outlined in the cookbooks below are some quick deployments of stateful applications into a Kubernetes cluster. These examples are not production ready but have been provided to give you some insight into how to use StorageOS with stateful applications.\nStatefulSets The examples we have provided use StatefulSets as a way to deploy applications into Kubernetes. The reason for this is that the StatefulSet controller is designed to manage stateful applications and it \u0026ldquo;provides guarantees about the ordering and uniqueness\u0026rdquo; of sets of pods.\nPractically this means that when a StatefulSet scales, pods are created in order from 0-(N-1), if a StatefulSet scales down then pods are deleted in reverse order from (N-1)-0.\nSecondly, it means that the behaviour of the StatefulSet controller differs from that of Deployment and ReplicaSet controllers. For Deployment and ReplicaSet controllers \u0026ldquo;\u0026hellip; at many points in the lifetime of a replica set there will be 2 copies of a pod\u0026rsquo;s processes running concurrently\u0026rdquo;. Having two different pods mount a volume at the same time can cause corruption of data. Currently Kubernetes accessModes only apply restrictions to nodes mounting volumes rather than pods, so it is important that StatefulSets are used with StorageOS volumes so the necessary pod uniqueness guarantees are maintained.\nStatefulSet Manifests The StorageOS specific part of the Kubernetes manifests for these examples lies in the VolumeClaimTemplate that\u0026rsquo;s part of the statefulset definition.\nVolumeClaimTemplate\napiVersion:apps/v1kind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1template:metadata:labels:app:mssqlenv:prodspec:serviceAccountName:mssqlcontainers:- name:fooimage:barvolumeMounts:- name:bazmountPath:/var/opt/barenvFrom:- configMapRef:name:mssqlvolumeClaimTemplates:- metadata:name:bazlabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiIn the StatefulSet definition above the container has a volume mount defined called baz. The definition for this volume is found in the VolumeClaimTemplate where the fast storageClass will be used to dynamically provision storage if the persistent volume does not already exist.\n","excerpt":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or …","ref":"/v2.2/docs/usecases/","title":"Use Cases: Install applications on Kubernetes with StorageOS"},{"body":"","excerpt":"","ref":"/v2.2/docs/reference/","title":"Reference"},{"body":"When you need support, please raise a ticket via our Help Desk portal. It is important to select the priority of your ticket in accordance with the severity:\n Severity 1 - Urgent Severity 2 - High Severity 3 - Medium Severity 4 - Low This helps us route and prioritise the ticket accordingly. You can find definitions of our support severity levels on our website.\nResponses to tickets will be cc\u0026rsquo;d via email.\nFor community support and general enquiries, please join our public Slack channel. \nInformation to include in tickets To help us provide effective support, we request that you provide as much information as possible when contacting us. The list below is a suggested starting point. Additionally, please include anything specific, such as log entries, that may help us debug your issue.\nInformation about the cluster can be automatically sent to StorageOS engineers as mentioned in the section StorageOS Diagnostic Bundle.\nPlatform Cloud provider/Bare metal OS distribution and version Kernel version docker version and installation procedure (distro packages or docker install) StorageOS Version of StorageOS storageos get nodes storageos get volumes storageos describe volume VOL_ID # in case of issues with a specific volume Orchestrator related (Kubernetes, OpenShift, etc) Version and installation method Managed or self managed? kubectl -n kube-system get pod kubectl -n kube-system logs -lapp=storageos -c storageos kubectl -n kube-system get storageclass Specific for your namespaces: kubectl describe pvc PVC_NAME Specific for your namespaces: kubectl describe pod POD_NAME Environment Changes Details of any recent changes to your environment such as planned maintenance, node reboots, network failures, etcd outage, etc.. This can help speed up ticket triage and resolution considerably StorageOS Cluster Diagnostic Bundle StorageOS has a cluster diagnostic function that aggregates cluster information.\nThe cluster diagnostic bundle is created only when a user chooses to do so. For convenience the cluster diagnostic bundle can either be uploaded to StorageOS or downloaded to the machine running the browser or CLI. The cluster diagnostic bundle is uploaded from a StorageOS node to a StorageOS GCP encrypted bucket using a TLS encrypted connection. The upload takes place only after user confirmation.\nAs a general outline the following information is sent to us:\n StorageOS logs StorageOS cluster state - node config, volumes and namespaces output of storageos describe node output of lshw output of dmesg See Support Bundle for an exhaustive list of what is included in the cluster diagnostic bundle.\nStorageOS engineers might ask for a cluster diagnostic bundle to be generated during support cases.\nThe information given in the cluster cluster diagnostic bundle is only used for support purposes and it will be removed once the data is no longer needed for such purposes. In case the information is sensitive and can\u0026rsquo;t be given to StorageOS, please make sure that support engineers have as much information about your environment as possible.\nYou can generate a cluster diagnostic bundle through the StorageOS GUI by navigating to the Cluster menu and downloading the StorageOS support Bundle. The cluster cluster diagnostic bundle can also be generated using the CLI command below.\n$ storageos get diagnostics -o storageos-diagnostic-bundle ","excerpt":"When you need support, please raise a ticket via our Help Desk portal. It is important to select the …","ref":"/v2.2/docs/support/","title":"Support"},{"body":"Our self-evaluation guide is a step by step recipe for installing and testing StorageOS. This guide is divided into three sections:\n Installation - install StorageOS with a single command Feature Testing - short walkthrough of some of our features Benchmarking - a recipe to benchmark StorageOS on your infrastructure For more comprehensive documentation including installation advice for complex setups, operational guides, and use-cases, please see our main documentation site.\nSupport for Self Evaluations Should you have questions or require support, there are several ways to get in touch with us. The fastest way to get in touch is to join our public Slack channel. You can also get in touch via email to info@storageos.com.\nFurthermore you can fill out the form below and we will get in touch.\n hbspt.forms.create({ portalId: \"3402546\", formId: \"a07fecd3-ce5b-4835-b136-51a94a35632b\", sfdcCampaignId: \"70158000000BAZzAAO\" }); Installation In this document we detail a simple installation suitable for evaluation purposes. The etcd we install uses a 3 node cluster with ephemeral storage, and as such is not suitable for production workloads. However, for evaluation purposes it should be sufficient. For production deployments, please see our main documentation pages.\nA standard StorageOS installation uses the StorageOS operator, which performs most platform-specific configuration for you. The StorageOS operator has been certified by Red Hat and is open source.\nThe basic installation steps are:\n Check prerequisites Install an etcd cluster Install the StorageOS Operator Install StorageOS by applying a CR for the Operator to action Prerequisites While we do not require custom kernel modules or additional userspace tooling, StorageOS does have a few basic prerequisites that are met by default by most modern distributions:\n At least 1 CPU core, 2GB RAM free TCP ports 5701-5710 and TCP \u0026amp; UDP 5711 open between all nodes in the cluster A 64bit supported operating system - StorageOS can run without additional packages in Debian 9, RancherOS, RHEL7.5,8 and CentOS7,8 and need the package linux-image-extra for Ubuntu. Mainline kernel modules target_core_mod, tcp_loop, target_core_file, target_core_user, configfs, and ui. These are present by default on most modern linux distributions, and can be installed with standard package managers. See our system configuration page for instructions. Quick Installation For those feeling brave and/or lucky, we have encapsulated the installation instructions in this guide into a short bash script. As with similar instructions elsewhere on the internet, caution is advised here. The script is liberally commented and we recommend that you inspect it in your text editor of choice before running.\n Run the following command where kubectl is installed and with the context set for your Kubernetes cluster\n curl -sL https://storageos.run | bash Note that this script deploys a cluster with etcd on transient storage, suitable for evaluation purposes only. Do not use this cluster for production workloads.\nYour new cluster can be used straight away. You can now jump to the Provision a Volume section below to provision your first volume and license the cluster.\nInstalling StorageOS Step By Step This section describes the installation steps in some more detail.\nInstall etcd Here we install a 3 node etcd cluster using the CoreOS operator to install on ephemeral storage within your Kubernetes cluster. This is suitable for evaluation purposes. For production deployment instructions, please see our production etcd page.\n Download repo\n$ git clone https://github.com/coreos/etcd-operator.git Configure NS, Role and RoleBinding\n$ export ROLE_NAME=etcd-operator $ export ROLE_BINDING_NAME=etcd-operator $ export NAMESPACE=etcd Create Namespace\n$ kubectl create namespace $NAMESPACE Deploy Operator\n$ ./etcd-operator/example/rbac/create_role.sh $ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END Create the EtcdCluster resource\n$ kubectl -n etcd create -f - \u0026lt;\u0026lt;YAML apiVersion: \u0026#34;etcd.database.coreos.com/v1beta2\u0026#34; kind: \u0026#34;EtcdCluster\u0026#34; metadata: name: \u0026#34;storageos-etcd\u0026#34; spec: size: 3 # Set to 1 if insufficient nodes version: \u0026#34;3.4.7\u0026#34; pod: etcdEnv: - name: ETCD_QUOTA_BACKEND_BYTES value: \u0026#34;2147483648\u0026#34; # 2 GB - name: ETCD_AUTO_COMPACTION_RETENTION value: \u0026#34;1000\u0026#34; # Keep 1000 revisions - name: ETCD_AUTO_COMPACTION_MODE value: \u0026#34;revision\u0026#34; # Set the revision mode resources: requests: cpu: 200m memory: 300Mi securityContext: runAsNonRoot: true runAsUser: 9000 fsGroup: 9000 tolerations: - operator: \u0026#34;Exists\u0026#34; # add tolerations to match all taints on the master nodes where etcd will be deployed affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster operator: In values: - storageos-etcd topologyKey: kubernetes.io/hostname YAML This creates a 3 node etcd cluster and associated service in Kubernetes. We can use the service endpoint when creating the StorageOS cluster.\nInstall StorageOS Operator In order to install the StorageOS operator download the requisite yaml manifests or apply them with kubectl.\n$ kubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml You can verify the operator is running using the following command\n$ kubectl get pods -n storageos-operator Install StorageOS Once the StorageOS operator has been installed a StorageOS cluster can be generated by creating a StorageOSCluster resource.\nA StorageOSCluster resource describes the state of the StorageOS cluster that is desired and the StorageOS operator will create the desired StorageOS cluster. For examples of StorageOSCluster resources please see our examples page here. For a full list of the configurable spec parameters of the StorageOSCluster resource please see here.\n Create a secret defining the API username and password\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zcsiControllerExpandUsername:c3RvcmFnZW9zcsiControllerExpandPassword:c3RvcmFnZW9zEND Create a StorageOSCluster resource\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:example-storageosclusternamespace:\u0026#34;storageos-operator\u0026#34;spec:# StorageOS Pods are in kube-system by defaultsecretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;storageos-operator\u0026#34;k8sDistro:\u0026#34;upstream\u0026#34;# Set the Kubernetes distribution for your cluster (upstream, eks, aks, gke, rancher, dockeree)# storageClassName: fast # The storage class creates by the StorageOS operator is configurablekvBackend:address:\u0026#34;storageos-etcd-client.etcd.svc.cluster.local:2379\u0026#34;END Confirm that the cluster has been created and that StorageOS pods are running\n$ kubectl -n kube-system get pods StorageOS pods enter a ready state after a minimum of 45s has passed.\n Deploy the StorageOS CLI as a container\n$ kubectl -n kube-system run \\ --image storageos/cli:v2.2.0 \\ --restart=Never \\ --env STORAGEOS_ENDPOINTS=storageos:5705 \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sh -c \u0026#34;while true; do sleep 999999; done\u0026#34; Provision a StorageOS Volume Now that we have a working StorageOS cluster, we can provision a volume to test everything is working as expected.\n Create a PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Verify that the volume was created with the CLI\n pvc-1 should be listed in the CLI output\n $ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that consumes the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Check that the pod starts successfully. If the pod starts successfully then the StorageOS cluster is working correctly\n$ kubectl get pod d1 -w The pod mounts a StorageOS volume under /mnt so any files written there will persist beyond the lifetime of the pod. This can be demonstrated using the following commands.\n Execute a shell inside the pod and write some data to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo Hello World! \u0026gt; /mnt/hello root@d1:/# cat /mnt/hello Hello World! should be printed out.\n Delete the pod\n$ kubectl delete pod d1 Recreate the pod\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Open a shell inside the pod and check the contents of /mnt/hello\n$ kubectl exec -it d1 -- cat /mnt/hello Hello World! should be printed out.\n License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nYou will need access to the StorageOS API on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 Now follow the instructions on our operations licensing page to obtain and apply a license.\nInstallation of StorageOS is now complete.\nStorageOS Features Now that you have a fully functional StorageOS cluster we will explain some of our features that may be of use to you as you complete application and synthetic benchmarks.\nStorageOS features are all enabled/disabled by applying labels to volumes. These labels can be passed to StorageOS via persistent volume claims (PVCs) or can be applied to volumes using the StorageOS CLI or GUI.\nThe following is not an exhaustive feature list but outlines features which are commonly of use during a self-evaluation.\nVolume Replication StorageOS enables synchronous replication of volumes using the storageos.com/replicas label.\nThe volume that is active is referred to as the master volume. The master volume and its replicas are always placed on separate nodes. In fact if a replica cannot be placed on a node without a replica of the same volume, the volume will fail to be created. For example, in a three node StorageOS cluster a volume with 3 replicas cannot be created as the third replica cannot be placed on a node that doesn\u0026rsquo;t already contain a replica of the same volume.\nSee our replication documentation for more information on volume replication.\n To test volume replication create the following PersistentVolumeClaim\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-replicated labels: storageos.com/replicas: \u0026#34;1\u0026#34; spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Note that volume replication is enabled by setting the storageos.com/replicas label on the volume.\n Confirm that a replicated volume has been created by using the StorageOS CLI or UI\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that uses the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: replicated-pod spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-replicated END Write data to the volume\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# echo Hello World! \u0026gt; /mnt/hello root@replicated-pod:/# cat /mnt/hello Hello World! should be printed out.\n Find the location of the master volume and shutdown the node\nShutting down a node causes all volumes, with online replicas, on the node to be evicted. For replicated volumes this immediately promotes a replica to become the new master.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-replicated Bound pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 5Gi RWO fast 1m $ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker (online) kind-worker2 1/1 26 seconds ago Check the location of the master volume and notice that it is on a new node. If the pod that mounted the volume was located on the same node that was shutdown then the pod will need to be recreated.\n$ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker2 (online) kind-worker2 1/1 46 seconds ago Check that the data is still accessible to the pod\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# cat /mnt/hello Hello World! should be printed out.\n Benchmarking Benchmarking storage is a complex topic. Considering the many books and papers that have been written about benchmarking, we could write many paragraphs here and only begin to scratch the surface.\nTaking this complexity into account we present recipes for both synthetic benchmarks using FIO, and a sample application benchmark to test PostgreSQL using pgbench. The approaches are complementary - synthetic benchmarks allow us to strictly control the parameters of the IO we put through the system in order to stress various aspects of it. Application benchmarks allow us to get a sense of the performance of the system when running an actual representative workload - which of course is the ultimate arbiter of performance.\nDespite the inherent complexity of benchmarking storage there are a few general considerations to keep in mind.\nConsiderations Local vs. Remote Volumes When a workload is placed on the same node as a volume, there is no network round trip for IO, and performance is consequently improved. When considering the performance of StorageOS it is important to evaluate both local and remote volumes; since for certain workloads we want the high performance of a local attachment, but we also desire the flexibility of knowing that our less performance-sensitive workloads can run from any node in the cluster.\nThe Effect of Synchronous Replication Synchronous replication does not impact the read performance of a volume, but it can have a significant impact on the write performance of the volume, since all writes must be propagated to replicas before being acked to the application. The impact varies in proportion to the inter-node latency. For an inter-node latency of 1ms, we have a max ceiling of 1000 write IOPS, and in reality a little less than that to allow for processing time on the nodes. This is less concerning then it may first appear, since many applications will issue multiple writes in parallel (known as increasing the queue depth).\nSynthetic Benchmarks Prerequisites Kubernetes cluster with a minimum of 3 nodes and a minimum of 30 Gb available capacity StorageOS CLI running as a pod in the cluster Synthetic benchmarks using tools such as FIO are a useful way to begin measuring StorageOS performance. While not fully representative of application performance, they allow us to reason about the performance of storage devices without the added complexity of simulating real world workloads, and provide results easily comparable across platforms.\nFIO has a number of parameters that can be adjusted to simulate a variety of workloads and configurations. Parameters that are particularly important are:\n Block Size - the size of the IO units performed. Queue Depth - the amount of concurrent IOs in flight IO Pattern - access can be random across the disk, or sequentially. IO subsystems typically perform better with sequential IO, because of the effect of read ahead caches, and other factors For this self-evaluation we will run a set of tests based on the excellent DBench, which distills the numerous FIO options into a series of well-crafted scenarios:\n Random Read/Write IOPS and BW - these tests measure the IOPS ceiling (with a 4k block size) and bandwidth ceiling (with a 128K block size) of the volume using a random IO pattern and high queue depth Average Latency - these tests measure the IO latency of the volume under favourable conditions using a random access pattern, low queue depth and low block size Sequential Read/Write - these tests measure the sequential read/write throughput of the volume with a high queue depth and high block size Mixed Random Read/Write IOPS - these tests measure the performance of the volume under a 60/40 read/write workload using a random access pattern and low blocksize For convenience we present a single script to run the scenarios using local and remote volumes both with and without a replica. Deploy the FIO tests for the four scenarios using the following command:\ncurl -sL https://docs.storageos.com/v2.2/sh/deploy-synthetic-benchmarks.sh | bash The script will take approximately 20 minutes to complete, and will print the results to STDOUT.\nThe exact results observed will depend on the particular platform and environment, but the following trends should be observable:\n local volumes perform faster than remote volumes read performance is independent of the number of replicas write performance is dependent on the number of replicas Application Benchmarks While synthetic benchmarks are useful for examining the behaviour of StorageOS with very specific workloads, in order to get a realistic picture of StorageOS performance actual applications should be tested.\nMany applications come with test suites which provide standard workloads. For best results, test using your application of choice with a representative configuration and real world data.\nAs an example of benchmarking an application the following steps lay out how to benchmark a Postgres database backed by a StorageOS volume.\n Start by cloning the StorageOS use cases repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/pgbench.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the Postgres examples folder\n$ cd storageos-usecases/pgbench Decide which node you want the pgbench pod and volume to be located on. The node needs to be labelled app=postgres\n$ kubectl label node \u0026lt;NODE\u0026gt; app=postgres Then set the storageos.com/hint.master label in 20-postgres-statefulset.yaml file to match the StorageOS nodeID for the node you have chosen before creating all the files. The StorageOS nodeID can be obtained using the cli and doing a describe node\n$ kubectl create -f . Confirm that Postgres is up and running\n$ kubectl get pods -w -l app=postgres Use the StorageOS CLI or the GUI to check the master volume location and the mount location. They should match\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Exec into the pgbench container and run pgbench\n$ kubectl exec -it pgbench -- bash -c \u0026#39;/opt/cpm/bin/start.sh\u0026#39; Conclusion After completing these steps you will have benchmark scores for StorageOS. Please keep in mind that benchmarks are only part of the story and that there is no replacement for testing actual production or production like workloads.\nStorageOS invites you to provide feedback on your self-evaluation to the slack channel or by directly emailing us at info@storageos.com]\n","excerpt":"Our self-evaluation guide is a step by step recipe for installing and testing StorageOS. This guide …","ref":"/v2.2/docs/self-eval/","title":"Self Evaluation Guide"},{"body":"","excerpt":"","ref":"/v2.2/index.json","title":""},{"body":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: '#ohpen_swagger_ui', presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ] }) window.ui = ui } \t","excerpt":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: …","ref":"/v2.2/docs/reference/api/","title":"API Reference"},{"body":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply licence --from-file \u0026lt;path-to-licence-file\u0026gt; $ echo \u0026#34;\u0026lt;licence file contents\u0026gt;\u0026#34; | storageos apply licence --from-stdin Flags: --from-file string reads a StorageOS product licence from a specified file path --from-stdin reads a StorageOS product licence from the standard input -h, --help help for licence Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply …","ref":"/v2.2/docs/reference/cli/apply/","title":"Apply licence"},{"body":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes.\nFundamentally, StorageOS uses the storage attached to the nodes in the StorageOS cluster to create and present virtual volumes into containers. Space on the host is consumed from the mount point /var/lib/storageos/data, so it is therefore recommended that disk devices are used exclusively for StorageOS, as described in Managing Host Storage \nStorageOS is agnostic to the underlying storage and runs equally well on bare metal, in virtual machines or on cloud providers.\nRead about the cloud native storage principles behind StorageOS.\nStorageOS on Kubernetes StorageOS is architected as a series of containers that fulfil separate, discrete functions.\n StorageOS Controlplane\nResponsible for monitoring and maintaining the state of volumes and nodes in the cluster\n StorageOS Dataplane\nResponsible for all I/O path related tasks; reading, writing, compression and caching\n StorageOS Scheduler\nResponsible for scheduling applications on the same node as applications volume\n CSI helper\nResponsible for registering StorageOS with Kubernetes as a CSI driver\n StorageOS Operator\nResponsible for the creation and maintenance of the StorageOS cluster\n StorageOS is deployed by the StorageOS operator. In Kubernetes, the StorageOS Controlplane and Dataplane are deployed in a single pod managed by a daemonset. This daemonset runs on every node in the cluster that will consume or present storage. The scheduler, CSI helper and Operator run as separate pods and are controlled as deployments.\nStorageOS is designed to feel familiar to Kubernetes and Docker users. Storage is managed through standard StorageClasses and PersistentVolumeClaims, and features are controlled by Kubernetes-style labels and selectors, prefixed with storageos.com/. By default, volumes are cached to improve read performance and compressed to reduce network traffic.\nAny pod may mount a StorageOS virtual volume from any node that is also running StorageOS, regardless of whether the pod and volume are collocated on the same node. Therefore, applications may be started or restarted on any node and access volumes transparently.\n","excerpt":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes. …","ref":"/v2.2/docs/concepts/architecture/","title":"Architecture"},{"body":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos attach --namespace my-namespace-name my-volume my-node Flags: -h, --help help for attach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos …","ref":"/v2.2/docs/reference/cli/attach/","title":"Attach"},{"body":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our Kubernetes installation instructions page.\nAn EKS deployment of Kubernetes uses AWS Linux by default with an optimized kernel. As the requisite kernel modules are not available for StorageOS to use TCMU, FUSE will be used as a fallback. Using FUSE instead of TCMU has performance implications.\nFor more details about the OS Distributions check the System Configuration page.\n","excerpt":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our …","ref":"/v2.2/docs/platforms/aws-eks/","title":"AWS EKS"},{"body":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our Kubernetes installation instructions page.\nAn AKS deployment of Kubernetes uses Ubuntu by default with an optimized kernel. All versions of Ubuntu with a kernel version later than4.15.0-1029-azure meet the StorageOS prerequisites.\n","excerpt":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our …","ref":"/v2.2/docs/platforms/azure-aks/","title":"Azure AKS"},{"body":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing StorageOS persistent volumes with Cassandra means that if a Cassandra pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up the pod data is immediately available. Should Kubernetes schedule the Cassandra pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not a StorageOS master is located on the same node.\nAs Cassandra has features to allow it to handle replication careful consideration of whether to allow StorageOS or Cassandra to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Cassandra on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiVersion:apps/v1kind:StatefulSetmetadata:name:cassandraspec:selector:matchLabels:app:cassandraserviceName:cassandrareplicas:3...spec:...volumeMounts:- name:cassandra-datamountPath:/var/lib/cassandra...volumeClaimTemplates:- metadata:name:cassandra-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Move into the Cassandra examples folder and create the objects\ncd storageos-usecases kubectl create -f ./cassandra Confirm Cassandra is up and running.\n$ kubectl get pods -w -l app=cassandra NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 8m32s cassandra-1 1/1 Running 0 7m51s cassandra-2 1/1 Running 0 6m36s Connect to the Cassandra client pod and connect to the Cassandra server through the service\n$ kubectl exec -it cassandra-0 -- cqlsh cassandra-0.cassandra Connected to K8Demo at cassandra-0.cassandra:9042. [cqlsh 5.0.1 | Cassandra 3.11.3 | CQL spec 3.4.4 | Native protocol v4] Use HELP for help. cqlsh\u0026gt; SELECT cluster_name, listen_address FROM system.local; cluster_name | listen_address --------------+---------------- K8Demo | 100.96.7.124 (1 rows) ","excerpt":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing …","ref":"/v2.2/docs/usecases/cassandra/","title":""},{"body":"The storageos command line interface (CLI) is used to manage cluster-wide configuration.\nInstallation # linux/amd64 curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.2.0/storageos_linux_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ # MacOS curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.2.0/storageos_darwin_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ You will need to provide the correct credentials to connect to the API. The default installation uses the storageos-api Secret to generate the first admin user. By default, it creates a single user with username storageos and password storageos:\nexport STORAGEOS_USERNAME=storageos export STORAGEOS_PASSWORD=storageos export STORAGEOS_ENDPOINTS=10.1.5.249:5705 Usage $ storageos Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Formatting CLI Output StorageOS CLI output can be formatted using the --output option. The strings that are passed to --output are \u0026lsquo;json\u0026rsquo;, \u0026lsquo;yaml\u0026rsquo; or \u0026lsquo;text\u0026rsquo;.\nCheatsheet Command Subcommand Description apply licence Apply licence to cluster. attach Attach Volume to a node. create user, volume Create resources such as users or volumes. delete volume Delete resources. describe node, volume Show detailed view of resources. detach volume Detach volume from a node. get cluster, diagnostics, namespace, node, user, volume List resources. help Help version Show CLI version. Source is available on Github.\n","excerpt":"The storageos command line interface (CLI) is used to manage cluster-wide configuration. …","ref":"/v2.2/docs/reference/cli/","title":"Command line interface"},{"body":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator acts as a Kubernetes controller that watches the StorageOSCluster CR (Custom Resource). Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\nYou can find the source code in the cluster-operator repository.\nInstall the operator following orchestrator specific procedure.\nTo deploy a StorageOS cluster you will need to fulfil the following steps.\n Install Etcd Install operator Create storageos-api Secret Create a CR (Custom Resource) to deploy StorageOS ","excerpt":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS …","ref":"/v2.2/docs/reference/cluster-operator/","title":"Cluster Operator"},{"body":"Every StorageOS cluster has a unique ClusterID generated at bootstrap. A StorageOS Licence is specific for a ClusterID.\nHow to obtain the ClusterID You can obtain the ClusterID using either the CLI or GUI.\nGUI You will need access to the StorageOS GUI on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 As an alternative, an Ingress controller may be preferred.\nOnce you have obtained access to the GUI, login using whatever credentials you used to create the cluster and go to the \u0026ldquo;Licence\u0026rdquo; section on the left navigation menu.\nCLI This CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence.\n","excerpt":"Every StorageOS cluster has a unique ClusterID generated at bootstrap. A StorageOS Licence is …","ref":"/v2.2/docs/operations/cluster-id/","title":"Find ClusterID"},{"body":"StorageOS clusters represent groups of nodes which run a common distributed control plane.\nTypically, a StorageOS cluster maps one-to-one to a Kubernetes (or similar orchestrator) cluster, and we expect our daemonset to run on all worker nodes within the cluster that will consume or present storage.\nClusters use etcd to maintain state and manage distributed consensus between nodes.\n","excerpt":"StorageOS clusters represent groups of nodes which run a common distributed control plane. …","ref":"/v2.2/docs/concepts/clusters/","title":"Clusters"},{"body":"StorageOS compression is handled on a per volume basis and is disabled by default in v2.2+, as performance is generally increased when compression is disabled due to block alignment. This means that there is a trade off between volume performance and the space the volume occupies on the backend device.\nCompression can be enabled by setting the label storageos.com/nocompress=false on a volume at volume creation time.\nStorageOS utilises the lz4 compression algorithm when writing to the backend store and when compressing replication traffic before it is sent across the network.\nStorageOS detects whether a block can be compressed or not by creating a heuristic that predicts the size of a compressed block. If the heuristic indicates that the compressed block is likely to be larger than the original block then the uncompressed block is stored. Block size increases post compression if the compression dictionary is added to a block that cannot be compressed. By verifying whether blocks can be compressed, disk efficiency is increased and CPU resources are not wasted on attempts to compress uncompressible blocks. StorageOS\u0026rsquo; patented on disk format is used to tell whether individual blocks are compressed without overhead. As such volume compression can be dynamically enabled/disabled even while a volume is in use.\nWhen compression and encryption are both enabled for a volume, blocks are compressed then encrypted.\n","excerpt":"StorageOS compression is handled on a per volume basis and is disabled by default in v2.2+, as …","ref":"/v2.2/docs/concepts/compression/","title":"Compression"},{"body":"We are always looking to improve our documentation. If you like to help people and can write, read on for the process for submitting your contributions. If your guide is published, you\u0026rsquo;ll receive $250 per article by PayPal.\nContent guidelines A guide contains step by step instructions for how to accomplish a specific task using StorageOS. To be accepted, guides must be:\n Written in English. Relevant, accurate and complete. Technically correct and thoroughly tested. Follow the writing style guide. Guides should avoid:\n Duplicating an existing guide or other sources, such as blogs or forum posts. Including irrelevant material. Submission and review You should submit your guide as a pull request to the GitHub repo.\nYour submission will be left open for community review for two weeks. After this, your submission will be reviewed internally for about another week.\nIf accepted, your pull request will be approved and you will have 36 hours to send your submission title and PayPal account information. Non-response will be taken as a go-ahead to publish.\nLegal COPYRIGHT OWNERSHIP. The StorageOS Guides \u0026amp; Tutorials repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license.\nCREDIT. Nothing contained in this Agreement shall be deeded to require StorageOS to use the Work, or any part thereof, in connection with StorageOS Guides \u0026amp; Tutorials or otherwise. Credit for the Work shall read, “Contributed by writer’s name.”\nPAYMENT. Upon publication of a submission to the StorageOS Guides \u0026amp; Tutorials Repository, the writer will be paid the sum of USD $250.00 as an electronic payment.\n","excerpt":"We are always looking to improve our documentation. If you like to help people and can write, read …","ref":"/v2.2/docs/reference/contributing/","title":"Contributing to the docs"},{"body":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: user Create a new user account volume Provision a new volume Flags: -h, --help help for create Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos create [command] --help\u0026#34; for more information about a command. create volume $ storageos create volume --help Provision a new volume Usage: storageos create volume [flags] Examples: $ storageos create volume --description \u0026#34;This volume contains the data for my app\u0026#34; --fs-type \u0026#34;ext4\u0026#34; --labels env=prod,rack=db-1 --size 10GiB --namespace my-namespace-name my-app $ storageos create volume --replicas 1 --namespace my-namespace-name my-replicated-app Flags: --async perform the operation asynchronously, using the configured timeout duration --cache caches volume data (default true) --compress compress data stored by the volume at rest and during transit (default true) -d, --description string a human-friendly description to give the volume -f, --fs-type string the filesystem to format the new volume with once provisioned (default \u0026#34;ext4\u0026#34;) -h, --help help for volume -l, --labels strings an optional set of labels to assign to the new volume, provided as a comma-separated list of key=value pairs -r, --replicas uint the number of replicated copies of the volume to maintain -s, --size string the capacity to provision the volume with (default \u0026#34;5GiB\u0026#34;) --throttle deprioritises the volume\u0026#39;s traffic by reducing the rate of disk I/O Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) create user $ storageos create user --help Create a new user account Usage: storageos create user [flags] Examples: $ storageos create user --with-username=alice --with-admin=true Flags: -h, --help help for user --with-admin control whether the user is given administrative priviledges --with-groups stringArray the list of policy groups to assign to the user --with-password string the password to assign to the user. If not specified, this will be prompted for. --with-username string the username to assign Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: …","ref":"/v2.2/docs/reference/cli/create/","title":"Create"},{"body":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available Commands: volume Delete a volume Flags: -h, --help help for delete Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos delete [command] --help\u0026#34; for more information about a command. delete volume $ storageos delete volume --help Delete a volume Usage: storageos delete volume [volume name] [flags] Examples: $ storageos delete volume my-test-volume my-unneeded-volume $ storagoes delete volume --namespace my-namespace my-old-volume Flags: --async perform the operation asynchronously, using the configured timeout duration --cas string make changes to a resource conditional upon matching a provided version -h, --help help for volume Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available …","ref":"/v2.2/docs/reference/cli/delete/","title":"Delete"},{"body":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] Available Commands: node Retrieve detailed information for nodes in the cluster volume Show detailed information for volumes Flags: -h, --help help for describe Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos describe [command] --help\u0026#34; for more information about a command. describe node $ storageos describe node --help Retrieve detailed information for nodes in the cluster Usage: storageos describe node [node names...] [flags] Aliases: node, nodes Examples: $ storageos describe node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) describe volume $ storageos describe volume --help Show detailed information for volumes Usage: storageos describe volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos describe volumes $ storageos describe volume --namespace my-namespace-name my-volume-name Flags: -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] …","ref":"/v2.2/docs/reference/cli/describe/","title":"Describe"},{"body":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] Examples: $ storageos detach volume --namespace my-namespace-name my-volume Flags: --cas string make changes to a resource conditional upon matching a provided version -h, --help help for detach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] …","ref":"/v2.2/docs/reference/cli/detach/","title":"Detach Volume"},{"body":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page.\nDocker EE and the Universal Control Plane can run on different Linux distributions. StorageOS supports RHEL, CentOS, Debian, and selected Ubuntu images. For more details, check out the supported OSs in the prerequisites page.\nStorageOS only supports Kubernetes nodes managed by Docker Enterprise Edition, not those nodes running Swarm. Mixed nodes (those running Kubernetes and Swarm workloads) are not supported. As a consequence, StorageOS volumes can only be provisioned on Kubernetes nodes, and only these nodes should be used for stateful workloads.\n","excerpt":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page. …","ref":"/v2.2/docs/platforms/dockeree/","title":"Docker Enterprise Edition"},{"body":"","excerpt":"","ref":"/v2.2/docs/","title":"Documentation"},{"body":"ElasticSearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, most popularly used to aggregate logs, but also to serve as a search backend to a number of different applications.\nUsing StorageOS persistent volumes with ElasticSearch (ES) means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Elasticsearch pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nElasticsearch has features to allow it to handle data replication, and as such careful consideration of whether to allow StorageOS or Elasticsearch to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Elasticsearch on Kubernetes Prerequisites Some OS tuning is required, which is done automatically when using our example from the use cases repository.\nElasticsearch requires vm.max_map_count to be increased to a minimum of 262144, which is a system wide setting. One way to achieve this is to run sysctl -w vm.max_map_count=262144 and update /etc/sysctl.conf to ensure it persists over a reboot. See ElasicSearch reference here.\nAdministrators should be aware that this impacts the behaviour of nodes and that there may be collisions with other application settings. Administrators are advised to centrally collate sysctl settings using the tooling of their choice.\nDeployment of the application StatefulSet defintion ---apiVersion:apps/v1kind:StatefulSetmetadata:name:esdata[...]spec:serviceAccountName:elasticsearchcontainers:- name:dataimage:elasticsearch:6.7.0imagePullPolicy:IfNotPresent[...]volumeMounts:- name:datamountPath:/usr/share/elasticsearch/data/data[...]volumeClaimTemplates:- metadata:name:\u0026#34;data\u0026#34;spec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- default StorageOS storage class nameresources:requests:storage:10Gi# \u0026lt;--- change this to the appropriate valueThis excerpt is from the StatefulSet definition (/elasticsearch/10-es-data.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nInstallation Clone the use cases repo You can find the latest files in the StorageOS use cases repostiory in /elasticsearch/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases Create the kubernetes objects\nThis will install an ES cluster with 3 master, 3 data and 3 coordinator nodes. Combined they will require ~ 14 GiB of available memory in your cluster, however, more may be used as the application is being used\nkubectl apply -f ./elasticsearch/ Once completed, an internal service object will have been created making the cluster available as http://elasticsearch:9200/ which is the default Kibana (when installed via Helm) will be using.\n Confirm Elasticsearch is up and running\nkubectl get pods -l component=elasticsearch NAME READY STATUS RESTARTS AGE elasticsearch-exporter-d86ffd94-zw45l 1/1 Running 0 5m44s es-coordinator-b7b984dd4-7wlz5 1/1 Running 0 5m44s es-coordinator-b7b984dd4-89w26 1/1 Running 0 5m44s es-coordinator-b7b984dd4-b4t6j 1/1 Running 0 5m44s es-master-78dfd5b49f-9gf5c 1/1 Running 0 5m44s es-master-78dfd5b49f-smsbw 1/1 Running 0 5m44s es-master-78dfd5b49f-z4qpj 1/1 Running 0 5m44s esdata-0 1/1 Running 0 5m44s esdata-1 1/1 Running 0 4m34s esdata-2 1/1 Running 0 3m22s Connect to ElasticSearch\nTo connect to ES directly, you can use the following port-forward command\nkubectl port-forward svc/elasticsearch 9200 and then access it via http://localhost:9200\n Kibana (optional) One of the most popular uses of ES is to use it for log aggregation and indexing, Kibana helps us visualize the data in these indices and can be easily used when installed via its Helm chart\n Install the helm chart.\nhelm install stable/kibana Once installed, use a port-foward to Kibana instead of directly to ES\nkubectl port-forward --namespace default $(kubectl get pods --namespace default -l \u0026#34;app=kibana\u0026#34; -o jsonpath=\u0026#34;{.items[0].metadata.name}\u0026#34;) 5601 and then access it via http://localhost:5601\n Monitoring (optional) As part of the example deployment, ES metrics are exposed and can be scraped by Prometheus on port 9108 (see 77-es-exporter.yaml). This is enabled by default, and should work with the default Prometheus install via Helm. If you\u0026rsquo;re using the Prometheus service monitors, you can monitor this installation by creating a monitor for the es-exporter service. For an example of how this is done to monitor StorageOS, please see prometheus-setup.\n","excerpt":"ElasticSearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, …","ref":"/v2.2/docs/usecases/elasticsearch/","title":""},{"body":"Etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to durably persist the StorageOS cluster state. As the backing store for Kubernetes, StorageOS uses etcd for many of the same reasons.\nStorageOS uses etcd as the single source of truth for all StorageOS objects. Whenever a request is made to create, update or delete an object the result is written to etcd before the request is completed. Using etcd as a configuration store allows nodes to retrieve the current cluster state after being offlined, allowing offlined nodes to rejoin the cluster.\n N.B. StorageOS v2.0 does not provide an embedded etcd server as previous versions did. You will need to setup an etcd server for StorageOS to use prior to installation of StorageOS. Please see our etcd prerequisites page for more information on how to install and configure etcd.\n ","excerpt":"Etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to …","ref":"/v2.2/docs/concepts/etcd/","title":"Etcd"},{"body":"Check the Etcd prerequisites page for a step by step installation of Etcd.\nBest practices StorageOS uses etcd as a service, whether it is deployed following the step by step instructions or as a custom installation. It is expected that the user maintains the availability and integrity of the etcd cluster.\nIt is highly recommended to keep the cluster backed up and ensure high availability of its data.\nNetwork low latency It is important to keep the latency between StorageOS nodes and the etcd replicas low. Deploying an etcd cluster in a different data center or region can make StorageOS detect etcd nodes as unavailable due to latency. 10ms latency between StorageOS and etcd would be the maximum threshold for proper functioning of the system.\nDisk low latency Etcd is very sensitive to disk latency. Because of that, it is recommended to run etcd away from other IO-intensive workloads. Operations such as backups, builds or application bundling cause a heavy usage of disks. If such operations run alongside the etcd nodes, they will cause etcd to become unstable and suffer downtime. It is best to run etcd nodes isolated from other IO workloads.\nEtcd advertise urls The Etcd startup parameters advertise-client-urls and initial-advertise-peer-urls specify the addresses etcd clients or other etcd members should use to contact the etcd server. The advertised addresses must be reachable from the remote machines - i.e. where StorageOS is running - so it can connect successfully. Do not advertise addresses like localhost or 0.0.0.0 for a production setup since these addresses are unreachable from remote machines.\nMonitoring It is highly recommended to add monitoring to the etcd cluster. Etcd serves Prometheus metrics on the client port http://etcd-url:2379/metrics.\nYou can use StorageOS developed Grafana Dashboards for etcd. When using etcd for production, you can use the etcd-cluster-as-service, while the etcd-cluster-as-pod can be used when using etcd from the operator.\nDefragmentation Etcd uses revisions to store multiple versions of keys. Compaction removes all key revision prior to a certain revision from etcd. Typically the etcd configuration enables the automatic compaction of keys to prevent performance degradation and limit the storage required. Compaction of revisions can create fragmentation that means space on disk is available for use by etcd but is unavailable for use by the file system. In order to reclaim this space, etcd can be defragmented.\nReclaiming space is important because when the etcd database file grows over the \u0026ldquo;DB_BACKEND_BYTES\u0026rdquo; parameter, the cluster triggers an alarm and sets itself read only and only allows reads and deletes. To avoid hitting the db backend bytes limit, compaction and defragmentation are required. How often defragmentation is required depends on the churn of key revisions in etcd.\nThe Grafana Dashboards mentioned above indicate when nodes require defragmentation. Be aware that defragmentation is a blocking operation that is performed per node, hence the etcd node will be locked for the duration of the defragmentation. Defragmentation usually takes a few milliseconds to complete.\nYou can also set cronjobs that execute the following defragmentation script. It will run a defrag when the DB is at 80% full. A defragmentation operation has to be executed per etcd node and it is a blocking operation. It is recommended to not execute the degragmentation in all etcd members at the same time. If using a cronjob, set them up for different times.\ncurl -sSLo defrag-etcd.sh https://raw.githubusercontent.com/storageos/deploy/master/k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd/roles/install_etcd/templates/defrag-etcd.sh.j2 chmod +x defrag-etcd.sh Known CoreOS Etcd Operator issues This topology is only recommended for deployments where isolated nodes cannot be used.\nEtcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd in pods it is therefore important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster causes the loss of all the etcd keystore and make StorageOS unable to perform metadata changes.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. A restore of the etcd keyspace from a backup might cause issues due to the disparity between the cluster state and its metadata in a different point in time. If you need to restore from a backup after a failure of etcd, contact the StorageOS support team.\n","excerpt":"Check the Etcd prerequisites page for a step by step installation of Etcd.\nBest practices StorageOS …","ref":"/v2.2/docs/operations/etcd/","title":"Etcd"},{"body":"StorageOS requires an etcd cluster in order to function. For more information on why etcd is required please see our etcd concepts page.\nWe do not support using the Kubernetes etcd for StorageOS installations.\nWe provide two methods for installing etcd. For those looking for a quick route to evaluating StorageOS, our first method installs etcd into your Kubernetes cluster using the CoreOS operator. Due to limitations in the CoreOS operator, this installation is not backed by any persistent storage, and therefore is unsuitable for production installations.\nFor production installations, there is currently no satisfactory way of installing a production-grade etcd cluster inside Kubernetes (although the landscape is changing rapidly, watch this space), and our production guidance remains to install etcd on separate machines outside of your Kubernetes cluster. This method is the best way to ensure a stable StorageOS cluster. Please see our etcd operations page for additional information on deployment best practices and concerns.\n Ephemeral pods within Kubernetes (Testing) External Virtual Machines (Production) Click the tabs below to select the installation method of your choice.\n Testing Production Testing - Installing Etcd Into Your Kubernetes Cluster This fast and convenient method is useful for quickly creating an etcd cluster in order to evaluate StorageOS. Do not use it for production installations.\nThis method uses the CoreOS etcd-operator to install a 3 node etcd cluster within your Kubernetes cluster, in the storageos-etcd namespace. We then install a Kubernetes service in that same namespace.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. A restore of the etcd keyspace from a backup might cause issues due to the disparity between the cluster state and its metadata in a different point in time. If you need to restore from a backup after a failure of etcd, contact the StorageOS support team.\nQuick Install For a one command install, the following script uses kubectl to create an etcd cluster in the storageos-etcd namespace. It requires kubectl in the system path, and the context set to the appropriate cluster.\ncurl -s https://docs.storageos.com/v2.2/sh/deploy-etcd.sh | bash Installation Step by Step For those who would prefer to execute the steps by themselves, they are as follows:\n Configure Namespace\nexport NAMESPACE=storageos-etcd Create Namespace\nkubectl create namespace $NAMESPACE If running in Openshift, an SCC is needed to start Pods\noc adm policy add-scc-to-user anyuid system:serviceaccount:$NAMESPACE:default Create ClusterRole and ClusterRoleBinding\n$ kubectl -n $NAMESPACE create -f-\u0026lt;\u0026lt;END apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: etcd-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: etcd-operator subjects: - kind: ServiceAccount name: default namespace: $NAMESPACE END $ kubectl -n $NAMESPACE create -f-\u0026lt;\u0026lt;END apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: etcd-operator rules: - apiGroups: - etcd.database.coreos.com resources: - etcdclusters - etcdbackups - etcdrestores verbs: - \u0026#34;*\u0026#34; - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - \u0026#34;*\u0026#34; - apiGroups: - \u0026#34;\u0026#34; resources: - pods - services - endpoints - persistentvolumeclaims - events verbs: - \u0026#34;*\u0026#34; - apiGroups: - apps resources: - deployments verbs: - \u0026#34;*\u0026#34; # The following permissions can be removed if not using S3 backup and TLS - apiGroups: - \u0026#34;\u0026#34; resources: - secrets verbs: - get END Deploy Etcd Operator\n$ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END Wait for the Etcd Operator Pod to start\nkubectl -n $NAMESPACE get pod -lapp=etcd-operator Create the EtcdCluster resource\n$kubectl-n$NAMESPACEcreate-f- \u0026lt;\u0026lt;ENDapiVersion:\u0026#34;etcd.database.coreos.com/v1beta2\u0026#34;kind:\u0026#34;EtcdCluster\u0026#34;metadata:name:\u0026#34;storageos-etcd\u0026#34;spec:size:3version:\u0026#34;3.4.7\u0026#34;pod:etcdEnv:- name:ETCD_QUOTA_BACKEND_BYTESvalue:\u0026#34;2147483648\u0026#34;# 2 GB- name:ETCD_AUTO_COMPACTION_RETENTIONvalue:\u0026#34;1000\u0026#34;# Keep 1000 revisions (default)- name:ETCD_AUTO_COMPACTION_MODEvalue:\u0026#34;revision\u0026#34;# Set the revision moderesources:requests:cpu:200mmemory:300MisecurityContext:runAsNonRoot:truerunAsUser:9000fsGroup:9000tolerations:- operator:\u0026#34;Exists\u0026#34;affinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight:100podAffinityTerm:labelSelector:matchExpressions:- key:etcd_clusteroperator:Invalues:- storageos-etcdtopologyKey:kubernetes.io/hostnameEND Installation Verification $ kubectl -n storageos-etcd get pod,svc NAME READY STATUS RESTARTS AGE pod/etcd-operator-55978c4587-8kx7b 1/1 Running 0 2h pod/storageos-etcd-qm9tmrpnlm 1/1 Running 0 2h pod/storageos-etcd-rzhjdz74hp 1/1 Running 0 2h pod/storageos-etcd-wvvv2d9g98 1/1 Running 0 2h NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/storageos-etcd ClusterIP None \u0026lt;none\u0026gt; 2379/TCP,2380/TCP 22h service/storageos-etcd-client ClusterIP 172.30.132.255 \u0026lt;none\u0026gt; 2379/TCP 22h The URL from the Service storageos-etcd-client.storageos-etcd.svc:2379 will be used later in the StorageOS Cluster CustomResource the kvBackend.address.\n Known etcd-operator issues Etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd using the CoreOS Operator, it is important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster causes the loss of all the etcd keystore and make StorageOS unable to perform metadata changes.\n Production - Etcd on External Virtual Machines For production installations, StorageOS strongly recommends running etcd outside of Kubernetes on a minimum of 3 dedicated virtual machines. This topology offers strong guarantees of resilience and uptime. We recommend this architecture in all environments, including those where Kubernetes is being deployed as a managed service.\nStorageOS doesn\u0026rsquo;t require a high performance etcd cluster as the throughput of metadata to the cluster is low.\nDepending on the level of redundancy you feel comfortable with you can install etcd on the Kubernetes Master nodes. Take extreme care to avoid collisions of the StorageOS etcd installation with the Kubernetes etcd when using the Kubernetes Master nodes. Precautions such as changing the default configuration for the client and peer ports, and ensuring the etcd data directory is modified. The ansible playbook below will default the etcd installation directory to /var/lib/storageos-etcd.\nYou can choose between two installation options.\n Manual Installation Ansible Installation Installation - Manual This section documents the steps required for manual installation of etcd using standard package management commands and systemd manifests.\n Repeat the following steps on all the nodes that will run etcd as a systemd service.\n Configure Etcd version and ports\nexport ETCD_VERSION=\u0026#34;3.4.9\u0026#34; export CLIENT_PORT=\u0026#34;2379\u0026#34; export PEERS_PORT=\u0026#34;2380\u0026#34; If targeting Kubernetes Master nodes, you must change CLIENT_PORT, PEERS_PORT\n Download Etcd from CoreOS official site\ncurl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz mkdir -p /tmp/etcd-v${ETCD_VERSION}-linux-amd64 tar -xzvf /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -C /tmp/etcd-v${ETCD_VERSION}-linux-amd64 --strip-components=1 rm /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz Install Etcd binaries\ncd /tmp/etcd-v${ETCD_VERSION}-linux-amd64 mv etcd /usr/local/sbin/etcd3 mv etcdctl /usr/local/sbin/etcdctl chmod 0755 /usr/local/sbin/etcd3 /usr/local/sbin/etcdctl Set up persistent Etcd data directory\nmkdir /var/lib/storageos-etcd Create the systemd environment file\nOn all nodes that will run etcd create a systemd environemnt file /etc/etcd.conf which has the IPs of all the nodes. The NODE_IP will need to change to correspond to the node IP where the environment file resides. NODE1_IP, NODE2_IP and NODE3_IP will remain the same across all three files.\n$ cat \u0026lt;\u0026lt;END \u0026gt; /etc/etcd.conf # NODE_IP is the IP of the node where this file resides. NODE_IP=10.64.10.228 # Node 1 IP NODE1_IP=10.64.10.228 # Node 2 IP NODE2_IP=10.64.14.233 # Node 3 IP NODE3_IP=10.64.12.111 CLIENT_PORT=${CLIENT_PORT} PEERS_PORT=${PEERS_PORT} END # Verify that variables are expaneded in the file $ cat /etc/etcd.conf Create the systemd unit file for etcd3 service\nCreate a systemd unit file /etc/systemd/system/etcd3.service with the following information:\n[Unit] Description=etcd3 Documentation=https://github.com/coreos/etcd Conflicts=etcd2.service [Service] Type=notify Restart=always RestartSec=5s LimitNOFILE=40000 TimeoutStartSec=0 EnvironmentFile=/etc/etcd.conf ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ --data-dir /var/lib/storageos-etcd \\ --quota-backend-bytes 8589934592 \\ --auto-compaction-retention 1000 \\ --auto-compaction-mode revision \\ --initial-cluster-state new \\ --initial-cluster-token etcd-token \\ --listen-client-urls http://${NODE_IP}:${CLIENT_PORT},http://127.0.0.1:${CLIENT_PORT} \\ --advertise-client-urls http://${NODE_IP}:${CLIENT_PORT} \\ --listen-peer-urls http://${NODE_IP}:${PEERS_PORT} \\ --initial-advertise-peer-urls http://${NODE_IP}:${PEERS_PORT} \\ --initial-cluster etcd-${NODE1_IP}=http://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=http://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=http://${NODE3_IP}:${PEERS_PORT} [Install] WantedBy=multi-user.target $NODE_IP is the IP address of the machine you are installing etcd on.`\n Note that setting the advertise-client-urls incorrectly will cause any client connection to fail. StorageOS will fail to communicate to Etcd.\n If enabling TLS, it is recomended to generate your own CA certificate and key. You will need to distribute the keys and certificates for the client auth on all etcd nodes. Moreover, the ExecStart value should look as below:\n ExecStart=/usr/local/sbin/etcd3 --name etcd-${NODE_IP} \\ --data-dir /var/lib/storageos-etcd \\ --quota-backend-bytes 8589934592 \\ --auto-compaction-retention 1000 \\ --auto-compaction-mode revision \\ --peer-auto-tls \\ --client-cert-auth --trusted-ca-file=/path/to/client-cert.pem \\ --cert-file=/path/to/ca.pem \\ --key-file=/path/to/client-key.pem \\ --initial-cluster-state new \\ --initial-cluster-token etcd-token \\ --listen-client-urls https://${NODE_IP}:${CLIENT_PORT} \\ --advertise-client-urls https://${NODE_IP}:${CLIENT_PORT} \\ --listen-peer-urls https://${NODE_IP}:${PEERS_PORT} \\ --initial-advertise-peer-urls https://${NODE_IP}:${PEERS_PORT} \\ --initial-cluster etcd-${NODE1_IP}=https://${NODE1_IP}:${PEERS_PORT},etcd-${NODE2_IP}=https://${NODE2_IP}:${PEERS_PORT},etcd-${NODE3_IP}=https://${NODE3_IP}:${PEERS_PORT} Reload and start the etc3 systemd service\n$ systemctl daemon-reload $ systemctl enable etcd3.service $ systemctl start etcd3.service Installation Verification\n The etcdctl binary is installed at /usr/local/bin on the nodes.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=http://127.0.0.1:${CLIENT_PORT} member list # $NODE_IP - the IP of the node 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Read the etcd operations page for our etcd recommendations.\n Installation - Ansible For a repeatable and automated installation, use of a configuration management tool such as ansible is recommended. StorageOS provides an ansible playbook to help you deploy etcd on standalone virtual machines.\n Clone StorageOS deployment repository\ngit clone https://github.com/storageos/deploy.git cd k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd Edit the inventory file\n The inventory file targets the nodes that will run etcd. The file hosts is an example of such an inventory file.\n $ cat hosts [nodes] centos-1 ip=\u0026#34;10.64.10.228\u0026#34; fqdn=\u0026#34;ip-10-64-10-228.eu-west-2.compute.internal\u0026#34; centos-2 ip=\u0026#34;10.64.14.233\u0026#34; fqdn=\u0026#34;ip-10-64-14-233.eu-west-2.compute.internal\u0026#34; centos-3 ip=\u0026#34;10.64.12.111\u0026#34; fqdn=\u0026#34;ip-10-64-12-111.eu-west-2.compute.internal\u0026#34; # Edit the inventory file $ vi hosts # Or your own inventory file The ip or fqdn are used to expose the advertise-client-urls of Etcd. Failing to provide valid ip/fqdn will cause any client connection to fail. StorageOS will fail to communicate to Etcd.\n Edit the etcd configuration\n If targeting Kubernetes Master nodes, you must change etcd_port_client, etcd_port_peers\n $ cat group_vars/all etcd_version: \u0026#34;3.4.9\u0026#34; etcd_port_client: \u0026#34;2379\u0026#34; etcd_port_peers: \u0026#34;2380\u0026#34; etcd_quota_bytes: 8589934592 # 8 GB etcd_auto_compaction_mode: \u0026#34;revision\u0026#34; etcd_auto_compaction_retention: \u0026#34;1000\u0026#34; members: \u0026#34;{{ groups[\u0026#39;nodes\u0026#39;] }}\u0026#34; installation_dir: \u0026#34;/var/lib/storageos-etcd\u0026#34; advertise_format: \u0026#39;fqdn\u0026#39; # fqdn || ip backup_file: \u0026#34;/tmp/backup.db\u0026#34; tls: enabled: false ca_common_name: \u0026#34;eu-west-2.compute.internal\u0026#34; etcd_common_name: \u0026#34;*.eu-west-2.compute.internal\u0026#34; cert_dir: \u0026#34;/etc/etcdtls\u0026#34; ca_cert_file: \u0026#34;etcd-ca.pem\u0026#34; etcd_server_cert_file: \u0026#34;server.pem\u0026#34; etcd_server_key_file: \u0026#34;server-key.pem\u0026#34; etcd_client_cert_file: \u0026#34;etcd-client.crt\u0026#34; etcd_client_key_file: \u0026#34;etcd-client.key\u0026#34; $ vi group_vars/all Choose between using IP addressing or FQDN in the advertise_format parameter. It allows you to decide how Etcd advertises its address to clients. This is particularly relevant when using TLS.\n If enabling TLS, it is recomended to generate your own CA certificate and key. You can do it by generating the CA from the machine running Ansible by: ansible-playbook create_ca.yaml.\n Install\nansible-playbook -i hosts install.yaml Installation Verification\n The playbook installs the etcdctl binary on the nodes, at /usr/local/bin.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 member list 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Benefits of Running External to Kubernetes Etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. In the case that quorum is lost, an etcd node stops and marks its contents as read-only. Another peer might have a newer version that has not been committed to the database. Quorum is fundamental for etcd operations.\nIn a Kubernetes environment, applications are scheduled across and in some scenarios such as \u0026ldquo;DiskPressure\u0026rdquo; they may need to be evicted from a node, and be scheduled onto a different node. With an application such as etcd, the scenario described can result in quorum being lost, making the cluster unable to recover automatically. Usually a 3 node etcd cluster can survive losing one node and recover. However, losing a second node at the same time or even having a network partition between them will result in quorum lost.\nBind Etcd IPs to Kubernetes Service Kubernetes external services use a DNS name to reference external endpoints, making them easy to reference from inside the cluster. You can use the example from the helper github repository to deploy the external Service. Using an external service can make monitoring of etcd from Prometheus easier.\n Using Etcd with StorageOS During installation of StorageOS the kvBackend.address parameter of the StorageOS operator is used to specify the address of the etcd cluster. See the StorageOS cluster operator configuration examples for more information.\n","excerpt":"StorageOS requires an etcd cluster in order to function. For more information on why etcd is …","ref":"/v2.2/docs/prerequisites/etcd/","title":"Etcd"},{"body":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. StorageOS implements dynamic provisioning, so the creation of a PVC will automatically provision a PV (PersistentVolume) that can be used to persist data written by a Pod.\nCreate the PersistentVolumeClaim You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic PVC definition\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;fast\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC will dynamically provision a 5GB volume using the fast StorageClass. This StorageClass was created during the StorageOS install and triggers creation of a PeristentVolume by StorageOS.\nFor installations with CSI, you can create multiple StorageClasses in order to specify default labels.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels. For simplicities sake this example will use unreplicated volumes.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# Reference to the StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1labels:storageos.com/replicas:\u0026#34;1\u0026#34;spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC has the storageos.com/replicas label set. This label tells StorageOS to add a replica for the volume that is created. For the sake of keeping this example simple an unreplicated volume will be used.\n Move into the examples folder and create a PVC using the PVC definition above.\n$ # from storageos-usecases/00-basic $ kubectl create -f ./pvc-basic.yaml You can view the PVC that you have created with the command below\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-1 Bound pvc-f8ffa027-e821-11e8-bc0b-0ac77ccc61fa 5Gi RWO fast 1m Create a pod that mounts the PVC created in step 2.\n$ kubectl create -f ./pod.yaml The command above creates a Pod that uses the PVC that was created in step 1.\napiVersion:v1kind:Podmetadata:name:d1spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mntname:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-1In the Pod definition above volume v1 references the PVC created in step 2, and is mounted in the pod at /mnt. In this example a debian image is used for the container but any container image with a shell would work for this example.\n Confirm that the pod is up and running\n$ kubectl get pods NAME READY STATUS RESTARTS AGE d1 1/1 Running 0 1m Execute a shell inside the container and write some contents to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo \u0026#34;Hello World!\u0026#34; \u0026gt; /mnt/helloworld root@d1:/# cat /mnt/helloworld Hello World! By writing to /mnt inside the container, the StorageOS volume created by the PVC is being written to. If you were to kill the pod and start it again on a new node, the helloworld file would still be avaliable.\nIf you wish to see more use cases with actual applications please see our Use Cases documentation.\n ","excerpt":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. …","ref":"/v2.2/docs/operations/firstpvc/","title":"StorageOS Volume Guide"},{"body":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available Commands: cluster Fetch cluster-wide configuration details diagnostics Fetch a cluster diagnostic bundle namespace Retrieve basic details of cluster namespaces node Retrieve basic details of nodes in the cluster user Fetch user details volume Retrieve basic details of volumes Flags: -h, --help help for get Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos get [command] --help\u0026#34; for more information about a command. get cluster $ storageos get cluster --help Fetch cluster-wide configuration details Usage: storageos get cluster [flags] Examples: $ storageos get cluster Flags: -h, --help help for cluster get diagnostics $ storageos get diagnostics --help Fetch a cluster diagnostic bundle Usage: storageos get diagnostics [flags] Examples: $ storageos get diagnostics $ storageos get diagnostics --output-file ~/my-diagnostics Flags: -h, --help help for diagnostics --output-file string writes the generated diagnostic bundle to a specified file path get namespace $ storageos get namespace -h Retrieve basic details of cluster namespaces Usage: storageos get namespace [namespace names...] [flags] Aliases: namespace, namespaces Examples: $ storageos get namespaces $ storageos get namespace my-namespace-name Flags: -h, --help help for namespace -l, --selector stringArray filter returned results by a set of comma-separated label selectors get node $ storageos get node -h Retrieve basic details of nodes in the cluster Usage: storageos get node [node names...] [flags] Aliases: node, nodes Examples: $ storageos get node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors get user $ storageos get user -h Fetch user details Usage: storageos get user [user names...] [flags] Aliases: user, users Examples: $ storageos get user my-username $ storageos get user my-username-1 my-username-2 $ storageos get user --use-ids my-userid $ storageos get user --use-ids my-userid-1 my-userid-2 Flags: -h, --help help for user get volume $ storageos get volume -h Retrieve basic details of volumes Usage: storageos get volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos get volumes --all-namespaces $ storageos get volume --namespace my-namespace-name my-volume-name Flags: --all-namespaces retrieves volumes from all accessible namespaces. This option overrides the namespace configuration -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors ","excerpt":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available …","ref":"/v2.2/docs/reference/cli/get/","title":"Get"},{"body":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any of the nodes in the cluster. Initially you can log in as the default administrator, with the username and password from the storageos-api Secret. By default storageos, storageos.\n You can access the GUI by either port-forwarding with kubectl or using an Ingress rule. i.e kubectl -n kube-system port-forward svc/storageos 5705.\n Nodes Volumes You can create volumes, including replicated volumes, and view volume details:\nLicensing Cluster info Namespaces Volumes can be namespaced across different projects or teams, and you can switch namespace using the left hand panel:\n","excerpt":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any …","ref":"/v2.2/docs/reference/gui/","title":"Graphical user interface (GUI)"},{"body":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the status of nodes in the cluster.\n$ storageos get nodes NAME HEALTH AGE LABELS node1 online 44 minutes ago node2 online 44 minutes ago node3 online 44 minutes ago node4 online 44 minutes ago node5 online 44 minutes ago ","excerpt":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the …","ref":"/v2.2/docs/operations/health/","title":"Cluster health"},{"body":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Use \u0026#34;storageos [command] --help\u0026#34; for more information about a command. ","excerpt":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to …","ref":"/v2.2/docs/reference/cli/help/","title":"Help"},{"body":"StorageOS uses the storage available on the nodes where it is installed to present as available for volumes.\nIn order to mitigate against problems caused by filling the host root disk, we recommend mounting a separate device into the /var/lib/storageos directory. StorageOS is agnostic to the type of filesystem mounted in /var/lib/storageos.\nExtending Available Storage StorageOS uses subdirectories of /var/lib/storageos/data to hold user data. By default, the directory /var/lib/storageos/data/dev1 will be created when a node is bootstrapped, and used for pool data. It is possible to shard the data by creating more directories into this structure. StorageOS will save data in any directory that conforms to the pattern /var/lib/storageos/data/dev[0-9]+, such as /var/lib/storageos/data/dev2 or /var/lib/storageos/data/dev5. This functionality enables operators to mount different devices into devX directories and StorageOS will recognise them as available storage automatically.\nThere are two possible options to expand the available disk space for StorageOS to allocate:\n Mount filesystem in /var/lib/storageos/data/devX Use LVM to expand the logical volume available to StorageOS Option 1: Mount Additional Devices This option enables operators to expand the cluster\u0026rsquo;s available space at any time without having to stop applications or forcing operational downtime. The expansion of disk is transparent for applications and StorageOS Volumes. StorageOS will use the new available space to create new data files.\n Context\nWe assume that there is a disk available in our Linux system without formatting in addition to the root filesystem. StorageOS data dir dev1 (/var/lib/storageos/data/dev1) is using /dev/xvda1. We will use the device /dev/xvdf to expand StorageOS available space.\nList available block devices in the host.\nroot@node0:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Format device\nroot@node0:/var/lib/storageos/data# mkfs -t ext4 /dev/xvdf mke2fs 1.42.12 (29-Aug-2014) Creating filesystem with 26214400 4k blocks and 6553600 inodes Filesystem UUID: 380712fa-6f82-477a-81a5-d7466d4c6b7f Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 4096000, 7962624, 11239424, 20480000, 23887872 Allocating group tables: done Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done Mount filesystem\nroot@node0:~# mkdir -p /var/lib/storageos/data/dev2 root@node0:~# mount /dev/xvdf /var/lib/storageos/data/dev2 Verify available storage\nIn less than 30 seconds, StorageOS will see the new available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 244,491,013,324 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Note that the node node0 has increased the TOTAL capacity in 100Gi.\n Persist the mount at boot by adding the mount endpoint to /etc/fstab\n Option 2: Expand Existing Devices Backed by LVM This option enables operators to take advantage of LVM to manage disks.\n Context\nWe assume that /var/lib/storageos is mounted onto an LVM volume. We are using a volumegroup named storageos and logical volume called data. There is a second physical disk /dev/xvdg unused.\nList available block devices in the host.\nroot@node2:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk `-storageos-data 254:0 0 99G 0 lvm /var/lib/storageos xvdg 202:96 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 107,696,304,947 # --\u0026gt; LVM storageos/data volume Add physical disk to LVM\nroot@node2:~# vgextend storageos /dev/xvdg Volume group \u0026quot;storageos\u0026quot; successfully extended The volume group storageos must have 2 physical volumes (#PV)\nroot@node2:~# vgs VG #PV #LV #SN Attr VSize VFree storageos 2 1 0 wz--n- 199.99g 104.99g Extend logical volume data\nroot@node2:~# lvextend -L+100G /dev/storageos/data Size of logical volume storageos/data changed from 95.00 GiB (24320 extents) to 195.00 GiB (49920 extents). Logical volume data successfully resized Resize the FileSystem\n Your filesystem must support the option to be expanded, and to do so while in use. Otherwise, you need to unmount first.\n root@node2:~# resize2fs /dev/storageos/data resize2fs 1.42.12 (29-Aug-2014) Filesystem at /dev/storageos/data is mounted on /var/lib/storageos; on-line resizing required old_desc_blocks = 6, new_desc_blocks = 13 The filesystem on /dev/storageos/data is now 51118080 (4k) blocks long. Check new available space\nThe mounted file system to /var/lib/storageos has increased its size.\nroot@node2:~# df -h /dev/mapper/storageos-data Filesystem Size Used Avail Use% Mounted on /dev/mapper/storageos-data 192G 60M 183G 1% /var/lib/storageos StorageOS available storage has increased too.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 206,158,430,208 # --\u0026gt; 100G more available Persist the mount at boot by adding the mount point to /etc/fstab\n ","excerpt":"StorageOS uses the storage available on the nodes where it is installed to present as available for …","ref":"/v2.2/docs/operations/managing-host-storage/","title":"Managing Host Storage"},{"body":"InfluxDB with StorageOS InfluxDB is a popular open source time series database application optimised for managing datasets consisting of many small measurements. Its advantages include the ability to handle very high write and query loads. Its uses include monitoring, analytics and the recording and analysis of data from sensors.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying InfluxDB on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition:\napiVersion:apps/v1kind:StatefulSetmetadata:name:influxdbspec:replicas:1selector:matchLabels:app:influxdbserviceName:influxdb...spec:serviceAccountName:influxdb...volumeMounts:- mountPath:/var/lib/influxdbname:data...volumeClaimTemplates:- metadata:name:dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:20GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaimTemplate that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Create the InfluxDB objects\nkubectl create -f ./influxdb Confirm InfluxDB is up and running.\n$ kubectl get pods NAME READY STATUS RESTARTS AGE influxdb-client 1/1 Running 0 1m influxdb-0 1/1 Running 0 1m Connect to the InfluxDB client pod, then to the InfluxDB server through the service (this reflects the common kubernetes pattern of maintaining a client pod to conveniently inspect a resource interactively). The default user (admin) and password (admin) are defined in the StatefulSet.\n$ kubectl exec -it influxdb-client -- bash root@influxdb-client:/# influx -host influxdb-0.influxdb Connected to http://influxdb-0.influxdb:8086 version 1.8.2 InfluxDB shell version: 1.8.2 \u0026gt; auth username: admin password: \u0026gt; show databases name: databases name _internal \u0026gt; CREATE DATABASE weather; \u0026gt; USE weather Using database weather \u0026gt; INSERT temperature,location=London value=26.4 \u0026gt; INSERT temperature,location=London value=24.9 \u0026gt; INSERT temperature,location=London value=22.2 \u0026gt; INSERT temperature,location=London value=14.7 \u0026gt; INSERT temperature,location=London value=19.5 \u0026gt; INSERT temperature,location=Paris value=27.1 \u0026gt; INSERT temperature,location=Paris value=27.5 \u0026gt; INSERT temperature,location=Paris value=21.3 \u0026gt; INSERT temperature,location=Paris value=26.7 \u0026gt; INSERT temperature,location=Paris value=30.0 \u0026gt; SELECT MEAN(*) FROM \u0026#34;temperature\u0026#34; GROUP BY \u0026#34;location\u0026#34; name: temperature tags: location=London time mean_value 0 25.65 name: temperature tags: location=Paris time mean_value 0 26.90 In the above steps we have inserted some time series data on the temperature at two locations, and calculated the mean for both. InfluxDB offers a variety of such aggregations (see the docs here), allowing convenient analysis of time series data.\nConfiguration If you need custom startup options, you can edit or add to the environment variables within the 20-statefulset.yaml file.\nBackups In this example of how to perform backups of an InfluxDB database on a Kubernetes cluster, we write the output backup file to an Amazon Web Services (AWS) S3 bucket. Other approaches, such as backing up to internal servers or other StorageOS volumes, are possible. For this example to run successfully, Base64-encoded AWS credentials and an S3 bucket name should be inserted into the data field of the backup/50-secret-config.yaml file.\n$ echo -n \u0026#39;\u0026lt;your-aws-access-key-id\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX $ echo -n \u0026#39;\u0026lt;your-aws-secret-access-key\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX $ echo -n \u0026#39;\u0026lt;your-aws-default-region\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX $ echo -n \u0026#39;\u0026lt;your-S3-bucket-name\u0026gt;\u0026#39; | base64 XXXXXXXXXXXX Secret definition:\napiVersion:v1kind:Secretmetadata:name:backup-pod-environmenttype:Opaquedata:AWS_ACCESS_KEY_ID:XXXXXXXXXXXXAWS_SECRET_ACCESS_KEY:XXXXXXXXXXXXAWS_DEFAULT_REGION:XXXXXXXXXXXXBUCKET_NAME:XXXXXXXXXXXXDB_NAME:d2VhdGhlcg==DB_HOST:aW5mbHV4ZGItMC5pbmZsdXhkYjo4MDg4To perform the backup, create the secret and job from the manifest files in the backup directory.\nkubectl create -f ./influxdb/backup/ Confirm that the backup pod has been created, and the backup performed successfully.\nNAME READY STATUS RESTARTS AGE backup-ks976 0/1 Completed 0 1m client 1/1 Running 0 10m influxdb-0 1/1 Running 0 10m mysql-0 1/1 Running 0 10m The files generated by the backup operation should now be available in you S3 bucket. Re-perform the operation at any time by deleting and re-creating the backup job.\n$ kubectl delete -f \u0026#34;influxdb/40-backup-job.yaml\u0026#34; job.batch \u0026#34;backup\u0026#34; deleted $ kubectl create -f \u0026#34;influxdb/40-backup-job.yaml\u0026#34; job.batch/backup created ","excerpt":"InfluxDB with StorageOS InfluxDB is a popular open source time series database application optimised …","ref":"/v2.2/docs/usecases/influxdb/","title":"InfluxDB"},{"body":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init container that sets the system configuration for StorageOS. The container also manages configuration changes required when upgrading StorageOS versions.\nThe container belongs to the DaemonSet that the StorageOS Cluster Operator starts when a StorageOSCluster resource is created. The storageos-init container is executed as an initContainer as part of a Kubernetes Pod. Therefore, only successful execution of the storageos-init container processes will result in the main container starting.\nScript Framework The code responsible for fulfilling the requirements is based on a Script Framework.\nThe script framework executes a set of scripts, performing checks, verifications and other procedures needed for StorageOS to be able to start. The scripts stdout and stderr are written to the stdout and stderr of the init app. The container logs contain all the logs of the individual scripts that run. The exit statuses of the scripts are used to determine initialization failure or success. Any non-zero exit status is logged as an event in the Kubernetes Pod events.\nIf any of the scripts fail, the storageos-init container will propagate the failure to Kubernetes, showing the status of the Pod as Init:Err.\nTo view the output of all storageos-init containers the following command can be used:\nkubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init For more details, check the StorageOS init container project.\nScripts executed The storageos-init container executes the following scripts.\n enable-lio dbupgrade ","excerpt":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init …","ref":"/v2.2/docs/reference/init-container/","title":"Init container"},{"body":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a StorageOS persistent volume being mounted on /var/jenkins_home. Deploying Jenkins using StorageOS offers multiple benefits. Firstly Jenkins can spin up multiple build pods at once to allow concurrent builds of different projects. Secondly Jenkins configuration is on a PersistentVolume so even if the Jenkins pod is rescheduled the configuration will persist.\nUsing StorageOS volume replicas allows for failure of nodes holding the PersistentVolume without interrupting Jenkins. Lastly by enabling StorageOS fencing Jenkins time to recover, in case of node failures, is greatly reduced.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information\nDeploying Jenkins on Kubernetes You can find the latest files in the StorageOS example deployment repository.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./jenkins Confirm that Jenkins is up and running\n$ kubectl get pods -w -l app=jenkins NAME READY STATUS RESTARTS AGE jenkins-0 1/1 Running 0 1m Connect to the Jenkins UI through the Jenkins service.\nYou can do this by port forwarding the Jenkins Kubernetes service to your localhost and accessing the UI via your browser. Alternatively if you have network access to your Kubernetes nodes then you can create a NodePort service and access Jenkins like that. A NodePort service has been left in 10-service.yaml commented out.\nTo port-foward the Jenkins service use the following command.\n$ kubectl port-foward svc/jenkins 8080 To login to the Jenkins UI use the credentials specified in 07-config.yaml, unless these have been changed from the defaults the username/password is admin/password.\n Create a Jenkins job.\nOnce you are logged into the UI you can create a job that will be farmed out to a Kubernetes plugin build agent. Click New Item, enter a name for the project and select Freestyle project. Next add an Execute shell build step. As a proof of concept you can use the bash below to have the pod execute a sleep.\n#!/bin/bash sleep 1000 Save the project and select Schedule a build of your project. You can watch for the appearance of a build pod using kubectl get pods -l jenkins=agent -w. Once the pod is created you should see the Build Executor status in the Jenkins UI display the pod.\nTo see multiple projects being built at once create another project and try scheduling a build of both projects at the same time.\n ","excerpt":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a …","ref":"/v2.2/docs/usecases/jenkins/","title":""},{"body":"Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub and traditional queues.\nUsing StorageOS persistent volumes with Apache Kafka means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the kafka pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nKafka has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Kafka to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Apache Zookeeper is required by Kafka to function; we assume it to already exist and be accessible within the Kubernetes cluster as zookeeper, see how to run Zookeeper with StorageOS here StorageOS is assumed to have been installed; please check for the latest available version here Kafka pods require 1536 MB of memory for successful scheduling Helm To simplify the deployment of kafka, we\u0026rsquo;ve used this Kafka helm chart (incubator) (version 0.13.8, app version 5.0.1) and rendered it into the example deployment files you can find in our GitHub repo.\nClone the use cases repo You can find the latest files in the StorageOS use cases repository in /kafka/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition\n---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:kafkalabels:app:kafka...spec:serviceName:kafka-headlesspodManagementPolicy:OrderedReadyupdateStrategy:type:OnDeletereplicas:3# \u0026lt;--- number of kafa pods to runtemplate:...spec:serviceAccountName:kafkacontainers:...- name:kafka-brokerimage:\u0026#34;confluentinc/cp-kafka:5.0.1\u0026#34;imagePullPolicy:\u0026#34;IfNotPresent\u0026#34;...volumeMounts:- name:datadirmountPath:\u0026#34;/var/data\u0026#34;volumes:- name:jmx-configconfigMap:name:kafka-metricsterminationGracePeriodSeconds:60volumeClaimTemplates:- metadata:name:datadirspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]resources:requests:storage:10Gi# \u0026lt;--- storage requested for each podstorageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Create the kubernetes objects\nkubectl apply -f ./kafka/ Confirm kafka is up and running\n$ kubectl get pods -l app=kafka NAME READY STATUS RESTARTS AGE kafka-0 2/2 Running 0 10m kafka-1 2/2 Running 0 9m26s kafka-2 2/2 Running 0 7m59s Connect to kafka\nConnect to the kafka test client pod and send some test data to kafka through its service endpoint\n Connect to the pod\nkubectl exec -it kafka-test-client /bin/bash Create a topic\n/usr/bin/kafka-topics --zookeeper zookeeper:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1 Send some test data\n/usr/bin/kafka-run-class org.apache.kafka.tools.ProducerPerformance --topic test-rep-one --num-records 5000 --record-size 100 --throughput -1 --print-metrics --producer-props acks=1 bootstrap.servers=kafka:9092 buffer.memory=67108864 batch.size=8196 ","excerpt":"Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub …","ref":"/v2.2/docs/usecases/kafka/","title":""},{"body":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual machines (VMs) in Kubernetes pods.\nDeploying Kubevirt using StorageOS offers multiple benefits. Kubevirt can spin up VMs as Kubernetes pods, using images on StorageOS persistent volumes. Doing this allows the VM data to persist through restarts and rescheduling. Using StorageOS volume replicas also allows for failure of nodes holding the PersistentVolume without interrupting the VM running off the PersistentVolume. Containerized Data Importer (CDI) can also be used to prepare StorageOS volumes with disk images in an automated fashion. Simply by declaring that a VirtualMachine will use a DataVolume and providing the disk image URL, a StorageOS volume can be dynamically provisioned and automatically prepared with the disk image.\nThis usecase will guide you through installing KubeVirt and CDI on your Kubernetes cluster, and create a VM. By the end of the guide you\u0026rsquo;ll be able to launch a shell inside the KubeVirt VM that\u0026rsquo;s running as a Kubernetes pod.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Please ensure you have met the Kubevirt prerequisites, please see the Kubevirt installation instructions for more information.\nAs part of this installation it is assumed that you are running a Kubernetes cluster on VMs. As such nested virtualization or hardware emulation need to be enabled.\nDeploying KubeVirt on Kubernetes For ease of installation we have enabled hardware emulation. If your VMs support nested virtualization then edit the Kubevirt ConfigMap ./kubevirt-install/10-cm.yaml, removing the line debug.useEmulation: \u0026quot;true\u0026quot;.\n In order to deploy Kubevirt you just need to clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/kubevirt $ kubectl create -f ./kubevirt-install Check that the Kubevirt pods are running.\n$ kubectl get pods -w -n kubevirt NAME READY STATUS RESTARTS AGE virt-api-57546d479b-p26d4 1/1 Running 0 1m virt-api-57546d479b-zs5dw 1/1 Running 0 1m virt-controller-56b5498854-7xsfz 1/1 Running 1 1m virt-controller-56b5498854-bz559 1/1 Running 1 1m virt-handler-6z4kq 1/1 Running 0 1m virt-handler-7szhl 1/1 Running 0 1m virt-handler-jmm6w 1/1 Running 0 1m virt-operator-79c9bdd859-8xq98 1/1 Running 0 1m virt-operator-79c9bdd859-kfjz6 1/1 Running 0 1m Once Kubevirt is running install CDI.\n$ kubectl create -f ./cdi Check that the CDI pods are running correctly.\n$ kubectl get pods -n cdi NAME READY STATUS RESTARTS AGE cdi-apiserver-8668f888df-s6pp4 1/1 Running 0 1m cdi-deployment-5cf794896b-whh4j 1/1 Running 0 1m cdi-operator-5887f96c-dz2hg 1/1 Running 0 1m cdi-uploadproxy-97fbbfcbf-6f9xs 1/1 Running 0 1m Now that CDI and Kubevirt are running, VMs can be created. In this example VMs running Cirros, a small and lightweight OS, will be created. The vm-cirros.yaml manifest creates a VirtualMachine that uses a DataVolume. This means that CDI will create a StorageOS backed PVC and download the image that the VirtualMachineInstance (VMI) will boot from onto the PVC.\n$ kubectl create -f ./vm-cirros.yaml Check that the VMI is running. Note that the VMI will only be created after CDI has downloaded the Cirros disk image onto a StorageOS persistent volume so depending on your connection speed this may take some time.\n$ kubectl get vmi NAME AGE PHASE IP NODENAME cirros 1m Running 10.244.2.12 ip-10-1-10-154.storageos.net $ kubectl get pods NAME READY STATUS RESTARTS AGE virt-launcher-cirros-drqhr 1/1 Running 0 1m Connect to the VM console.\nThis example uses the virtctl kubectl plugin in order to connect to the VMs console. The escape sequence ^] is ctrl + ]\n$ kubectl virt console cirros Successfully connected to cirros console. The escape sequence is ^] login as \u0026#39;cirros\u0026#39; user. default password: \u0026#39;gocubsgo\u0026#39;. use \u0026#39;sudo\u0026#39; for root. cirros login: cirros Password: $ Cloning Volumes CDI allows for images to be cloned using a DataVolume manifest. Verify that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n N.B. Ensure that the VMI is stopped before continuing!\n Verify that the VMI is stopped before continuing, and that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE cirros Bound pvc-f4833060-5a77-420c-927e-6bc518d9df3c 12Gi RWO fast 1m Once the PVC\u0026rsquo;s existence is confirmed then create a new DataVolume that uses the cirros PVC as its source.\n$ kubectl create -f ./cloned.yaml Watch as the CDI pods are created.\n$ kubectl get pods -w You\u0026rsquo;ll see that a cdi-upload-cloned-datavolume pod is created and then a cdi-clone-source pod is created. The cdi-source pod mounts the original cirros volume and sends the contents of the volume to the cdi-upload pod. The cdi-upload pod creates and mounts a new PVC and writes the contents of the original volume to it.\n ","excerpt":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual …","ref":"/v2.2/docs/usecases/kubevirt/","title":""},{"body":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to various StorageOS artefacts. Applying specific feature labels triggers compression, replication and other storage features. No feature labels are present by default.\nStorageOS Node labels Nodes do not have any feature labels present by default. When StorageOS is run within Kubernetes, the Cluster Operator syncs any node labels set on Kubernetes nodes within StorageOS. Node labels may also be set with the CLI or UI.\n Feature Label Values Description Compute only storageos.com/computeonly true / false Specifies whether a node should be computeonly where it only acts as a client and does not host volume data locally, otherwise the node is hyperconverged (the default), where the node can operate in both client and server modes. You can set the computeonly label on the Kubernetes node and the label will be sync\u0026rsquo;d to the StorageOS node (labels take an eventual consistency reconciliation time of ~1min).\nkubectl label node $NODE storageos.com/computeonly=true StorageOS Volume labels Volumes do not have any feature labels present by default. WARNING: The caching and compression labels can only apply during provision time, they can\u0026rsquo;t be changed during execution.\n Feature Label Values Description Caching storageos.com/nocache true / false Switches off caching. Compression storageos.com/nocompress true / false Switches off compression of data at rest and in transit. Replication storageos.com/replicas integers [0, 5] Replicates entire volume across nodes. Typically 1 replica is sufficient (2 copies of the data); more than 2 replicas is not recommended. To create a volume with a feature label:\n Option 1: PVC Label\nAdd the label in the PVC definition, for instance:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc-3labels:storageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;-----spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:1G Option 2: Set label in the StorageClass\nAny PVC using the StorageClass inherits the label. The PVC label takes precedence over the StorageClass parameters.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;--------provisioner:storageos# CSI driver (recommended) Option 3:\nOnce a PVC is created, you can update the Labels in StorageOS both in the UI or CLI. Those labels are going to be visible only for StorageOS and will not be synced to the Kubernetes resource.\n ","excerpt":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to …","ref":"/v2.2/docs/reference/labels/","title":"StorageOS Feature labels"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying a Free Developer licence.\nTo learn how to apply a licence to your cluster, check the operations licensing page.\nFree Developer licence The developer licence only requires the user to register and issue a licence through StorageOS.\nThe developer licence is free and grants a licence for a 5 node StorageOS cluster with 5TB of provisioned capacity. It is designed to enable basic cloud native workflows in Kubernetes that require the persistence of stateful application data. Dynamic provisioning, distributed access to data and high availability of volumes through synchronous replication and automatic failover are some of the features that are available under the developer licence.\nThe free developer licence includes community support through our public Slack channel..\nCommercial licences For information on our commercial offerings, including support, please contact sales@storageos.com.\nNote about capacity limits Some StorageOS licences have limits on capacity. StorageOS allows provisioning volumes until the limit of the licence is reached. Only the size of the volume requested by the Persistent Volume Claim counts for the licence limit, regardless whether of whether the volume has replication enabled.\nOnce the licence limit is reached, new volumes are not able to provision unless provisioned capacity is released, i.e deleting volumes. That behaviour is not tight to the capacity of the backend disks on your nodes.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2.2/docs/reference/licence/","title":"Licence"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying for a Free Developer licence.\nObtaining a Developer licence via the GUI You will need access to the StorageOS GUI on port 5705 of any of your nodes. For convenience, it is often easiest to port forward the service using the following kubectl incantation (this will block, so a second terminal window may be advisable):\n$ kubectl port-forward -n kube-system svc/storageos 5705 As an alternative, an Ingress controller may be preferred.\nOnce you have obtained access to the GUI, login using whatever credentials you used to create the cluster.\nYou can obtain and apply a free Developer licence in the StorageOS web GUI by creating or logging in with a StorageOS account on the StorageOS portal via the licence page of the GUI:\nWait a few seconds for the licence generation process to complete, at which point your licence will be visible.\nApplying a previously obtained licence via the GUI Commercial licences are delivered through contact with the StorageOS team. To apply such keys, via the web GUI, visit the licence section of the GUI and click on the tab \u0026ldquo;Upgrade\u0026rdquo;, for the specific licence level you purchased. Then paste the licence key and click on \u0026ldquo;UPLOAD KEY TO CLUSTER\u0026rdquo;. Note that you can also view your cluster ID on the same page.\nApplying a licence via the CLI Before getting a licence, you need to know the ID of your StorageOS cluster.\nThis CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence. Once, given the key, you can apply the licence by using the following command.\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos apply licence --from-stdin Read the licence CLI command reference for further information.\nObtaining an Enterprise licence Please contact sales@storageos.com to discuss pricing for commercial licences.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2.2/docs/operations/licensing/","title":"Licensing"},{"body":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on linux.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MS SQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1...spec:serviceAccountName:mssql...volumeMounts:- name:datamountPath:/var/opt/mssql...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MS SQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mssql Confirm MS SQL is up and running.\n$ kubectl get pods -w -l app=mssql NAME READY STATUS RESTARTS AGE mssql-0 1/1 Running 0 1m Connect to the MS SQL client pod and connect to the MS SQL server through the service\n$ kubectl exec -it mssql-0 -- /opt/mssql-tools/bin/sqlcmd -S mssql-0.mssql -U SA -P \u0026#39;Password15\u0026#39; 1\u0026gt; USE master; 2\u0026gt; GO Changed database context to \u0026#39;master\u0026#39;. 1\u0026gt; SELECT name, database_id, create_date FROM sys.databases ; 2\u0026gt; GO name database_id create_date --------------------------- ----------- ----------------------- master 1 2003-04-08 09:13:36.390 tempdb 2 2018-11-02 16:30:37.907 model 3 2003-04-08 09:13:36.390 msdb 4 2018-10-19 01:18:57.300 (4 rows affected) Configuration If you need custom startup options, you can edit the ConfigMap file 15-mssql-configmap.yaml with your desired MS SQL configuration settings.\n","excerpt":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on …","ref":"/v2.2/docs/usecases/mssql/","title":""},{"body":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With mTLS both StorageOS and etcd authenticate each other ensuring that communication only happens between mutually authenticated end points, and that all communication is encrypted.\nStorageOS uses the certificates and keys from a Secret to cypher and authenticate Etcd traffic.\nHow to create the certificates Secret The client auth certificates need the following file names, in the Secret.\n etcd-client-ca.crt - containing the etcd Certificate Authority certificate etcd-client.crt - containing the etcd Client certificate etcd-client.key - cotaining the etcd Client key kubectl create secret -n storageos-etcd generic \\ etcd-client-tls \\ --from-file=\u0026#34;etcd-client-ca.crt\u0026#34; \\ --from-file=\u0026#34;etcd-client.crt\u0026#34; \\ --from-file=\u0026#34;etcd-client.key\u0026#34; How to use the mTLS certificates Secret with StorageOS Below is an example StorageOSCluster resource that can be used to setup StorageOS with etcd using mTLS.\napiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:storageos-clusternamespace:\u0026#34;storageos-operator\u0026#34;spec:# StorageOS Pods are in kube-system by defaultsecretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;default\u0026#34;images:nodeContainer:\u0026#34;storageos/node:v2.2.0\u0026#34;namespace:\u0026#34;storageos\u0026#34;# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;etcd\u0026#34;# Namespace of the client certificates secretkvBackend:address:\u0026#34;https://storageos-etcd-cluster-client.storagos-etcd.svc:2379\u0026#34;# Etcd client service address.backend:\u0026#34;etcd\u0026#34;# Backend typetlsEtcdSecretRefName and tlsEtcdSecretRefNamespace are used to pass a reference to the Secret.\nThe StorageOS operator uses the etcd secret that contains the client certificates, to build a secret in the StorageOS installation namespace. This secret contains the certificate filenames and certificate file contents. The StorageOS daemonset that is created by the operator mounts the secret as a volume so that the certificate files are available inside the pod. Environment variables containing the file paths are passed to the StorageOS process in order to use the files from the mounted path.\nA worked example of setting up StorageOS with external etcd using mTLS is avaliable here. For ease of use the example uses the CoreOS etcd operator and the CoreOS guide The example uses the CoreOS etcd operator and follows the CoreOS guide for Cluster TLS.\n","excerpt":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With …","ref":"/v2.2/docs/operations/etcd/storageos-secret-info/","title":"Encrypting communication with Etcd"},{"body":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular web-based applications including WordPress.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MySQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiversion:apps/v1kind:statefulsetmetadata:name:mysqlspec:selector:matchlabels:app:mysqlenv:prodservicename:mysqlreplicas:1...spec:serviceaccountname:mysql...volumemounts:- name:datamountpath:/var/lib/mysqlsubpath:mysql- name:confmountpath:/etc/mysql/mysql.conf.d...volumeclaimtemplates:- metadata:name:datalabels:env:prodspec:accessmodes:[\u0026#34;readwriteonce\u0026#34;]storageclassname:\u0026#34;fast\u0026#34;# storageos storageclass resources:requests:storage:5giThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MySQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mysql Confirm MySQL is up and running.\n$ kubectl get pods -w -l app=mysql NAME READY STATUS RESTARTS AGE mysql-0 1/1 Running 0 1m Connect to the MySQL client pod and connect to the MySQL server through the service\n$ kubectl exec client -- mysql -h mysql-0.mysql -e \u0026#34;show databases;\u0026#34; Database information_schema mysql performance_schema Configuration If you need custom startup options, you can edit the ConfigMap file 15-mysqld-configmap.yaml with your desired MySQL configuration settings.\n","excerpt":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular …","ref":"/v2.2/docs/usecases/mysql/","title":""},{"body":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a StorageOS cluster to be used by multiple teams across multiple projects.\nIt is not necessary to create StorageOS namespaces manually, as StorageOS maps Kubernetes namespaces on a one-to-one basis when PersistentVolumeClaims using the StorageOS StorageClass are created.\nAccess to Namespaces is controlled through user or group level policies.\n","excerpt":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a …","ref":"/v2.2/docs/concepts/namespaces/","title":"Namespaces"},{"body":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is created by default.\nNamespaces apply to volumes.\nManaging Namespaces In order to create a new namespace navigate to \u0026ldquo;Namespaces\u0026rdquo; in the GUI, and select \u0026ldquo;Create Namespace\u0026rdquo;.\nWhen a Kubernetes PVC is created in a namespace, StorageOS automatically maps the Volume in the same namespace. Namespaces are created by StorageOS to fulfil the RBAC rules enforced by Kubernetes roles.\nIn order to delete a namespace, all volumes must be deleted from the namespace before the namespace can be deleted.\n","excerpt":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is …","ref":"/v2.2/docs/operations/namespaces/","title":"Namespaces"},{"body":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load balancer or even as a Kubernetes ingress controller.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Nginx on Kubernetes You can find the latest files in the StorageOS use cases repostiory\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet definition\napiVersion:apps/v1kind:StatefulSetmetadata:name:nginxspec:serviceName:nginxspec:serviceAccountName:nginxcontainers:- name:nginximage:nginxports:- containerPort:80volumeMounts:- name:nginx-datamountPath:/usr/share/nginx/htmlsubPath:html- name:nginx-configmountPath:/etc/nginx/conf.dvolumes:- name:nginx-configconfigMap:name:nginxvolumeClaimTemplates:- metadata:name:nginx-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Nginx examples folder and create the objects\ncd storageos-usecases kubectl create -f ./nginx Confirm Nginx is up and running.\n$ kubectl get pods -w -l app=nginx NAME READY STATUS RESTARTS AGE nginx-0 1/1 Running 0 1m Connect to the nginx pod and write a file to /usr/share/nginx/html that Nginx will serve.\n$ kubectl exec nginx-0 -it -- bash root@nginx-0:/# echo Hello world! \u0026gt; /usr/share/nginx/html/greetings.txt Connect to the Busybox pod and connect to the Nginx server through the service and retrieve the directory index from Nginx.\n$ kubectl exec -it busybox -- /bin/sh / # wget -q -O- nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;greetings.txt\u0026#34;\u0026gt;greetings.txt\u0026lt;/a\u0026gt; 27-Feb-2019 12:04 13 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; Retrieve and display the contents of the greetings.txt file\n/ # wget -q -O- nginx/greetings.txt Hello world! Configuration If you need custom startup options, you can edit the ConfigMap file 15-nginx-configmap.yaml with your desired Nginx configuration settings.\n","excerpt":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load …","ref":"/v2.2/docs/usecases/nginx/","title":""},{"body":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A node must be running a daemonset pod in order to consume and/or present storage.\nBy default StorageOS nodes run in hyperconverged mode. This means that the node hosts data from StorageOS volumes and can present volumes to applications.\nAlternatively, a node can run in computeonly mode, which means no storage is consumed on the node itself and the node only presents volumes hosted by other nodes. Volumes presented to applications running on compute only nodes are therefore all remote. Compute only nodes can be very useful for topologies where nodes are ephemeral and should not host data, but the ephemeral nodes host applications that require StorageOS volumes. The nodes that are not intended to hold data, but just to present StorageOS volumes, can be set as computeonly.\nA node can be marked as compute only at any point in time by adding the label storageos.com/computeonly=true, following the labels reference.\n","excerpt":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A …","ref":"/v2.2/docs/concepts/nodes/","title":"Nodes"},{"body":"The following document lists the open source software attributions in the StorageOS Control Plane, Data Plane and CLI.\nCNats The MIT License (MIT) Copyright (c) 2015 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CppUTest Copyright (c) 2007, Michael Feathers, James Grenning and Bas Vodde All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the \u0026lt;organization\u0026gt; nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE EARLIER MENTIONED AUTHORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL \u0026lt;copyright holder\u0026gt; BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Hayai Copyright (c) 2011 - Nick Bruun. This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. If you meet (any of) the author(s), you're encouraged to buy them a beer, a drink or whatever is suited to the situation, given that you like the software. This notice may not be removed or altered from any source distribution. LZ4 LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LibreSSL LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- ==================================================================== Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit. (http://www.openssl.org/)\u0026quot; 4. The names \u0026quot;OpenSSL Toolkit\u0026quot; and \u0026quot;OpenSSL Project\u0026quot; must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact openssl-core@openssl.org. 5. Products derived from this software may not be called \u0026quot;OpenSSL\u0026quot; nor may \u0026quot;OpenSSL\u0026quot; appear in their names without prior written permission of the OpenSSL Project. 6. Redistributions of any form whatsoever must retain the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)\u0026quot; THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==================================================================== This product includes cryptographic software written by Eric Young (eay@cryptsoft.com). This product includes software written by Tim Hudson (tjh@cryptsoft.com). Original SSLeay License ----------------------- Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) All rights reserved. This package is an SSL implementation written by Eric Young (eay@cryptsoft.com). The implementation was written so as to conform with Netscapes SSL. This library is free for commercial and non-commercial use as long as the following conditions are aheared to. The following conditions apply to all code found in this distribution, be it the RC4, RSA, lhash, DES, etc., code; not just the SSL code. The SSL documentation included with this distribution is covered by the same copyright terms except that the holder is Tim Hudson (tjh@cryptsoft.com). Copyright remains Eric Young's, and as such any Copyright notices in the code are not to be removed. If this package is used in a product, Eric Young should be given attribution as the author of the parts of the library used. This can be in the form of a textual message at program startup or in documentation (online or textual) provided with the package. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: \u0026quot;This product includes cryptographic software written by Eric Young (eay@cryptsoft.com)\u0026quot; The word 'cryptographic' can be left out if the rouines from the library being used are not cryptographic related :-). 4. If you include any Windows specific code (or a derivative thereof) from the apps directory (application code) you must include an acknowledgement: \u0026quot;This product includes software written by Tim Hudson (tjh@cryptsoft.com)\u0026quot; THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The licence and distribution terms for any publically available version or derivative of this code cannot be changed. i.e. this code cannot simply be copied and put under another distribution licence [including the GNU Public Licence.] protobuf This license applies to all parts of Protocol Buffers except the following: - Atomicops support for generic gcc, located in src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Code generated by the Protocol Buffer compiler is owned by the owner of the input file used when generating it. This code is not standalone and requires a support library to be linked with it. This support library is itself covered by the above license. rocksdb BSD License For rocksdb software Copyright (c) 2011-present, Facebook, Inc. All rights reserved. --------------------------------------------------------------------- Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/armon/go-metrics The MIT License (MIT) Copyright (c) 2013 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/armon/go-radix The MIT License (MIT) Copyright (c) 2014 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/bgentry/speakeasy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \u0026quot;[]\u0026quot; replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \u0026quot;printed page\u0026quot; as the copyright notice for easier identification within third-party archives. Copyright [2013] [the CloudFoundry Authors] Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/dgrijalva/jwt-go Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgrijalva/jwt-go/request Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgryski/go-metro MetroHash This package is a mechanical translation of the reference C++ code for MetroHash, available at https://github.com/jandrewrogers/MetroHash I claim no additional copyright over the original implementation. The MIT License (MIT) Copyright (c) 2015 J. Andrew Rogers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/docker/docker/pkg/mount Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv/store Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/golang/protobuf/proto Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/context Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/handlers Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/mux Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/websocket Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/consul/api Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/errwrap Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-cleanhttp Mozilla Public License, version 2.0 1. Definitions 1.1. \u0026quot;Contributor\u0026quot; means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. \u0026quot;Contributor Version\u0026quot; means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. \u0026quot;Contribution\u0026quot; means Covered Software of a particular Contributor. 1.4. \u0026quot;Covered Software\u0026quot; means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. \u0026quot;Incompatible With Secondary Licenses\u0026quot; means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. \u0026quot;Executable Form\u0026quot; means any form of the work other than Source Code Form. 1.7. \u0026quot;Larger Work\u0026quot; means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. \u0026quot;License\u0026quot; means this document. 1.9. \u0026quot;Licensable\u0026quot; means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. \u0026quot;Modifications\u0026quot; means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. \u0026quot;Patent Claims\u0026quot; of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. \u0026quot;Secondary License\u0026quot; means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. \u0026quot;Source Code Form\u0026quot; means the form of the work preferred for making modifications. 1.14. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) means an individual or a legal entity exercising rights under this License. For legal entities, \u0026quot;You\u0026quot; includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, \u0026quot;control\u0026quot; means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an \u0026quot;as is\u0026quot; basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - \u0026quot;Incompatible With Secondary Licenses\u0026quot; Notice This Source Code Form is \u0026quot;Incompatible With Secondary Licenses\u0026quot;, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-msgpack/codec Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/go-multierror Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/memberlist Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/coordinate Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/serf Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/kelseyhightower/envconfig Copyright (c) 2013 Kelsey Hightower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/mattn/go-isatty Copyright (c) Yasuhiro MATSUMOTO \u0026lt;mattn.jp@gmail.com\u0026gt; MIT License (Expat) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/miekg/dns Extensions of the original work are copyright (c) 2011 Miek Gieben As this is fork of the official Go code the same license applies: Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/mitchellh/cli Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/conf The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/logger The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server/pse The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/util The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats/encoders/builtin The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nuid The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/pborman/uuid Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/Sirupsen/logrus The MIT License (MIT) Copyright (c) 2014 Simon Eskildsen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/thejerf/suture Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/urfave/negroni The MIT License (MIT) Copyright (c) 2014 Jeremy Saenz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang.org/x/crypto/bcrypt Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/crypto/blowfish Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/net/context Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/unix Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/registry Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/svc/eventlog Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/time/rate Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. gopkg.in/yaml.v2 Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ","excerpt":"The following document lists the open source software attributions in the StorageOS Control Plane, …","ref":"/v2.2/docs/reference/open_source_attribution/","title":"StorageOS Open Source Software Attribution Notice"},{"body":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform actions such as creation, deletion and mounting of volumes through CSI. The CSI container running in the StorageOS Daemonset creates a Linux socket that allows the communication between OpenShift and StorageOS\nInstallation StorageOS v2 supports OpenShift 4.0, 4.1, 4.2, 4.3, 4.4 and 4.5.\nTo install StorageOS on OpenShift, please follow our installation instructions page.\nOpenShift Upgrades OpenShift provides an upgrade operator that automates the process of orchestrator version changes.\nThis procedure can cause StorageOS to malfunction due to sequential node restarts not taking the presence of stateful application data into consideration. To avoid this issue, make sure all stateful workloads using StorageOS Volumes are stopped - usually by scaling StatefulSets to 0. Please contact StorageOS support for further advice if required.\n OpenShift requires the internal registry to be available during the upgrade, however StorageOS volumes may not be available. Therefore using StorageOS for the internal registry is not recommended.\n CSI (Container Storage Interface) Note CSI is the standard that enables storage drivers to release on their own schedule. This allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nStorageOS v2 implements communication with the OpenShift controlplane with CSI only. The native driver has been deprecated\nStorageOS PersistentVolumeClaims The user can provide standard PVC definitions and StorageOS will dynamically provision them. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\n","excerpt":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform …","ref":"/v2.2/docs/platforms/openshift/","title":"OpenShift"},{"body":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a user or group access to a namespace, a policy needs to be created mapping the user or group to the namespace.\n Note: Users always have access to the default namespace\n For more information on how to use policies, see the Policies operations page.\n","excerpt":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a …","ref":"/v2.2/docs/concepts/policies/","title":"Policies"},{"body":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user level so access can be controlled granularly.\nUsers can belong to one or more groups to control their namespace permissions. Additionally user specific policies can be created to grant a user access to a namespace. Users can belong to any number of groups and have any number of user level policies configured.\n Note: Users are created with access to the default namespace. Policies cannot be applied to the default namespace.\n Managing Policies To start creating policies, at least one custom namespace and user are required. For more information on how to create namespaces see our Namespace guide, for users see our Users CLI reference.\nIn order to create a policy navigate to \u0026ldquo;Policies\u0026rdquo; in the GUI and select \u0026ldquo;Create Policy\u0026rdquo;. A policy controls access to a variety of StorageOS resources and is applied to a user, by placing the user in the policies group.\nIn order to delete a policy, all users must be removed from the policy group before deletion of the policy can be completed.\n","excerpt":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user …","ref":"/v2.2/docs/operations/policies/","title":"Policies"},{"body":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational database management system (ORDBMS).\nPostgres is deployed across a wide variety of platforms with a mix of workloads ranging from small, single-node use cases to large Internet-facing clusters with many concurrent users.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. If you need to setup StorageOS on Kubernetes then please see our guide to [installing StorageOS on Kubernetes]({% link \u0026ldquo;docs/install/kubernetes.md\u0026rdquo; \u0026gt;}}).\nDeploying PostgreSQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases PersistentVolumeClaim and Pod definition excerpts\nkind:PersistentVolumeClaimmetadata:name:pg-dataannotations:volume.beta.kubernetes.io/storage-class:fast...kind:Podmetadata:name:postgressspec:securityContext:fsGroup:26containers:- name:pgimage:crunchydata/crunchy-postgres:centos7-10.4-1.8.3volumeMounts:- mountPath:/pgdataname:data...volumes:- name:datapersistentVolumeClaim:claimName:pg-dataThis excerpt is from the PersistentVolumeClaim and Pod definition. The pod definition references the pg-data VolumeClaim so storage is dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the PostgreSQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./postgres Confirm PostgreSQL is up and running.\n$ kubectl get pod postgres-0 -w NAME READY STATUS RESTARTS AGE postgres-0 1/1 Running 0 1m Connect to the PostgreSQL client pod and connect to the PostgreSQL server through the service.\n$ kubectl exec -it postgres-0 -- psql -h postgres-0.postgres -U primaryuser postgres -c \u0026#34;\\l\u0026#34; Password for user primaryuser: password List of Databases Name | Owner | Encoding | Collate | Ctype | Access privileges +=========================================================================+ postgres | postgres | SQL_ASCII | C | C | template0 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres template1 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres userdb | postgres | SQL_ASCII | C | C | =Tc/postgres + | | | | | postgres=CTc/postgres+ | | | | | testuser=CTc/postgres (4 rows) The password for the primary user is password. You can see this is set in the ConfigMap file.\n Configuration If you need custom startup options, you can edit the ConfigMap file 15-postgresd-configmap.yaml with your desired PostgreSQL configuration settings.\n","excerpt":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational …","ref":"/v2.2/docs/usecases/postgres/","title":""},{"body":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting in Kubernetes.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Prometheus on Kubernetes This is the Prometheus use case for StorageOS. Following are the steps for creating a Prometheus StatefulSet and using StorageOS to provide persistent storage.\n You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases Prometheus Custom Resource definition\napiVersion:monitoring.coreos.com/v1kind:Prometheusmetadata:name:prometheus-storageoslabels:app:prometheus-operatorspec:...storage:volumeClaimTemplate:metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:storageos-replicatedresources:requests:storage:1GiThis excerpt is from the Prometheus Custom Resource definition. This file contains the VolumeClaimTemplate that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs due to the VolumeClaimTemplate in the Prometheus StatefulSet. The Prometheus StatefulSet is created by the Prometheus Operator, triggered by the creation of the Prometheus resource.\n Move into the Prometheus examples folder and install the Prometheus Operator and create a Prometheus resource.\n$ cd storageos-usecases/prometheus $ ./install-prometheus.sh Confirm Prometheus is up and running.\n$ kubectl get pods -w -l app=prometheus NAME READY STATUS RESTARTS AGE prometheus-prometheus-storageos-0 3/3 READY 0 1m You can see the created PVC using.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-prometheus-prometheus-storageos-0 Bound pvc-b6c17c0a-e76b-4a0b-8fc6-46c0e1629210 1Gi RWO storageos-replicated 65m In the Prometheus deployment script, a service monitor is created. A Service Monitor is a special object that the Prometheus operator uses to create configuration for Endpoints for Prometheus to scrape. Although the name implies that a Service Monitor defines Kubernetes services that will be scraped, Prometheus actually targets the Endpoints that the services point to. The new Prometheus instance will use the storageos-etcd service monitor to start scraping metrics from the storageos-etcd pods. Assuming the storageos cluster was setup using ETCD as pods. For more information about service monitors, have a look at the upstream documentation.\n$ kubectl get servicemonitor NAME AGE storageos-etcd 5d1h The Prometheus web ui can be accessed by port-forwarding the Prometheus pods port to localhost.\n$ kubectl port-forward prometheus-prometheus-storageos-0 9090 Then launch a web browser and go to localhost:9090 to access the Prometheus web ui. You can confirm that the storageos-etcd target is configured there.\n Configuration In the storageos-usecases/prometheus/manifests/prometheus directory there are other example Service Monitors. For more information about Prometheus, check out the prometheus documentation.\n","excerpt":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and …","ref":"/v2.2/docs/usecases/prometheus/","title":""},{"body":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS transparently supports Rancher deployments using CentOS, RHEL or Debian and can support other Linux distributions detailed in the systems supported page if the appropriate kernel modules are present.\n","excerpt":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS …","ref":"/v2.2/docs/platforms/rancher/","title":"Rancher"},{"body":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional durability to disk.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Redis on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:redisspec:selector:matchLabels:app:redisenv:prodserviceName:redisreplicas:1...spec:serviceAccountName:redis...volumeMounts:- name:datamountPath:/bitnami/redis/data...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the Redis examples folder and create the objects\ncd storageos-usecases kubectl create -f ./redis Confirm Redis is up and running.\n$ kubectl get pods -w -l app=redis NAME READY STATUS RESTARTS AGE redis-0 1/1 Running 0 1m Connect to the Redis client pod and connect to the Redis server through the service\n$ kubectl exec -it redis-0 -- redis-cli -a password Warning: Using a password with \u0026#39;-a\u0026#39; option on the command line interface may not be safe. 127.0.0.1:6379\u0026gt; CONFIG GET maxmemory 1) \u0026#34;maxmemory\u0026#34; 2) \u0026#34;0\u0026#34; Configuration If you need custom startup options, you can edit the ConfigMap file 15-redis-configmap.yaml with your desired Redis configuration settings.\n","excerpt":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional …","ref":"/v2.2/docs/usecases/redis/","title":""},{"body":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than \u0026ldquo;latest\u0026rdquo;, and to perform upgrades only after reading the release notes.\nThe latest tagged release is v2.2.0. For installation instructions please see our Install page.\nThe latest CLI release is v2.2.0, available from Github.\nUpgrading To upgrade from version 1.x to 2.x, please contact support for assistance.\nv2.2.0 - Released 2020-08-18 This release focuses on performance. We analysed existing performance characteristics across a variety of real-world use cases and ended up with improvements across the board. Of particular note:\n Sequential reads have improved by up to 130% Sequential writes have improved by up to 737% Random reads have improved by up to 45% Random writes have improved by up to 135% I/O for large block sizes (128K) has improved by up to 353% We are extremely proud of our performance and we love to talk about it. Have a look at the Benchmarking section of the self-evaluation guide and consider sharing your results. Our PRE engineers are available to discuss in our slack channel.\nNew Data engine revamp focused on provable consistency and performance. Key characteristics:\n Metadata is stored in an optimised index, lowering I/O latency and improving performance for all workloads. Large block reads/writes are now be handled in a single operation. Applications like Kafka will go much faster. On-disk compression is now disabled by default as in most scenarios this offers better performance. To enable on-disk compression for a specific workload, see compression.\n Improved dataplane: The number of I/O threads are now determined by the number of processing cores available. This improves scalability and performance on larger servers. ha: Improve partition tolerance behaviour when a volume master that has lost its connection to etcd rejoins. ha: Allow replicas in unhealthy states to be remediated and re-used while maintaining partition tolerance. ha: When a master fails and the new master is not yet available, introduce a back-off to the redirection logic to avoid spamming the logs with connection failure errors. ha: Ignore health advertisements for local node. Local nodes are handled directly. node delete: Only refuse to delete a node if the node health can be authoritatively verified to be in use. api: Increase HTTP server write timeout. cli/ui: Allow partial diagnostic bundle downloads. ui: Namespace dropdown can now be scrolled. ui: Add \u0026ldquo;Job title\u0026rdquo; to UI licence form. logging: Log version at startup at INFO level. logging: Lower verbosity of SCSI warnings that do not apply to StorageOS. diagnostics: Include logs that have been rotated. diagnostics: Bundle collection across providers is now done in parallel. build: Update base image to RHEL 8.2. operator: Removed DB migration utility required for v1.3 -\u0026gt; v1.4 upgrades. operator: Automatically refreshes StorageOS API token without failing requests when the token expires. operator: Updated CSI attacher and provisioner to latest upstream. operator: Remove cluster.local suffix on Pod scheduler service address. This allows the scheduler to work in clusters with custom DNS configuration. operator: Defaults are now set for most CSI configuration options in the StorageOSCluster custom resource. Fixed csi: When unmount request is received for a volume that has already been unmounted, return success. csi: Verify volume is attached on the node before mounting it. xfs: Support older RHEL kernels which have an XFS library that does not allow reflinks/dedupe. dataplane: Reserve 1GiB of capacity on the target disk to allow manual cleanup operations, rather than filling target disk to capacity. operator: In some cases /var/lib/storageos could fail to unmount cleanly after a restart. This resulted in multiple entries in /proc/mounts. v2.1.0 - Released 2020-06-26 New csi: Volume expansion now supported in offline mode. To expand a volume, stop any workloads accessing the volume, then edit the PVC to increase the capacity. For more information, see our Volume Resize operations page and the CSI Volume Expansion page. api: Volume configuration including replica count can now be updated while the volume is in use. Other updateable fields include labels and description. failover: Before determining that a node is offline and performing recovery operations, the I/O path is also verified. This provides more robust failure detection and ensures that nodes that are still responding to I/O do not get replaced. This I/O path verification is in addition to the gossip-based failure detection. operator: Default tolerations are now set for the StorageOS node container. This helps ensure that the StorageOS node container does not get evicted when the node is running low on resources. Improved api: Added checks to prevent deletion of a node with active volumes, or if it is the master of at least one volume. This helps prevent orphaned volumes. cli: Add an --offline-delete flag to allow removal of volumes whose master and replica nodes are offline. This allows cleanup of orphaned volumes. ui: Add an offline volume delete option. ui: Volumes can now be detached from the UI. cli: Labels are no longer truncated. api: When a new node is added to the cluster, its capacity is available to use immediately. Fixed ui: Favicon was missing. ui: Duplicate volumes could be shown on the node details page. operator: During uninstall a ClusterRoleBinding was not removed. v2.0.0 - Released 2020-05-05 New operator: StorageOS containers now run in the kube-system namespace by default to allow the system-node-critical priority class to be set. This instructs Kubernetes to start StorageOS before application Pods, and to evict StorageOS only after application Pods have finished. This setting was previously recommended in documentation; it is now the default. operator: StorageOS CSI helper containers now run as privileged. This ensures that the CSI endpoint can be seen on systems with SELinux enabled. ui: replication progress for new or re-joining replicas is now displayed. ui: show warning for unlicensed clusters. cli: new commands: licence management get policy create namespace create policy describe user describe namespace describe policy delete user delete namespace delete policy licence: removed the default licence expiry date added for v2.0.0-rc.1. Improved dataplane: improved retry behaviour for network I/O. cli: \u0026ldquo;get volumes\u0026rdquo; for all namespaces should be done in parallel. cli: help text document config file ui: link node name and get to node details on the volume details page. ui: node details add available capacity spinner. ui: node list remove capacity values / address port. ui: node list show master/replica counts. ui: node list remove edit action. ui: format entity labels. ui: node details link volumes. ui: align buttons for licences. ui: k8s warning in \u0026ldquo;create volume\u0026rdquo; modal. ui: node list remove \u0026ldquo;API\u0026rdquo; from \u0026ldquo;API Address\u0026rdquo; ui: add some details about the Licence on the licence page. api: include valid for duration in login response. licence: restrict nodes which are unregistered after 24 hours. scheduler: return error for namespace/volume not found dataplane: start gRPC threads separately from rest of the supervisor. Fixed ui: centre licence types. ui: capacity in ui is per namespace. cli: fail gracefully if missing some output details (i.e. no node exists for id). v2.0.0-rc.1 - Released 2020-03-31 Initial release of version 2.x. See StorageOS v2.0 Release Blog for details.\n","excerpt":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than …","ref":"/v2.2/docs/reference/release_notes/","title":"Release notes"},{"body":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous replication ensures strong consistency for applications such as databases and Elasticsearch, incurring one network round trip on writes.\nThe basic model for StorageOS replication is of a master volume with distributed replicas. Each volume can be replicated between 0 and 5 times, which are provisioned to 0 to 5 nodes, up to the number of remaining nodes in the cluster.\nIn this diagram, the master volume D was created on node 1, and two replicas, D2 and D3 on nodes 3 and 5.\nWrites that come into D (step 1) are written in parallel to D2 and D3 (step 2). When both replicas and the master acknowledge that the data has been written (step 3), the write operation return successfully to the application (step 4).\nFor most applications, one replica is sufficient (storageos.com/replicas=1).\nAll replication traffic on the wire is compressed using the lz4 algorithm, then streamed over tcp/ip to target port tcp/5703.\nIf the master volume is lost, a replica is promoted to master (D2 or D3 above) and a new replica is created and synced on an available node (Node 2 or 4). This is transparent to the application and does not cause downtime.\nIf a replica volume is lost and there are enough remaining nodes, a new replica is created and synced on an available node. While a new replica is created and being synced, the volume\u0026rsquo;s health will be marked as degraded.\nIf the lost replica comes back online before the new replica has finished synchronizing, then StorageOS will calculate which of the two synchronizing replicas has the smallest difference compared to the master volume and keep that replica. The same holds true if a master volume is lost and a replica is promoted to be the new master. If possible, a new replica will be created and begin to sync. Should the former master come back online it will be demoted to a replica and the replica will the smallest difference to the current master will be kept.\nWhile the replica count is controllable on a per-volume basis, some environments may prefer to set default labels on the StorageClass.\nNumber of nodes StorageOS replicas are distributed across available nodes. When a node fails, a new replica is provisioned and synced as described above. Failure to provision a new replica will eventually cause the volume to become read-only. StorageOS will re-attempt creation of the replica for 90 seconds. After that period, if the old replica is not available and a new replica cannot be provisioned, StorageOS cannot guarantee that the data is safe and stored on multiple nodes as requested by the user. It will therefore force the volume to be set to read-only.\nTo ensure that a new replica can always be created, an additional node should be available. To guarantee high availability, clusters using volumes with 1 replica must have at least 3 storage nodes. When using volumes with 2 replicas, at least 4 storage nodes, 3 replicas, 5 nodes, etc.\nMinimum number of storage nodes = 1 (primary) + N (replicas) + 1\nFor an application to be able to use the volume normally, the volume needs to recover from a degraded state. This is achieved by having enough nodes on which to place the data, and triggering a remount of the volumes - i.e. deleting the Pod mounting the volume once it is in a healthy state.\nDelta Sync StorageOS implements a delta sync between a volume master and its replicas. This means that if a replica for a volume goes offline, that when the replica comes back online only the regions with changed blocks need to be synchronized. This optimization reduces the time it takes for replicas to catch up, improving volume resilience. Additionally, it reduces network and IO bandwidth which can reduce costs when running in public clouds.\n","excerpt":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous …","ref":"/v2.2/docs/concepts/replication/","title":"Replication"},{"body":"Managing Resources for StorageOS containers Kubernetes resource requests and limits are two optional Pod properties that allow you to specify how much of a resource a container in a Pod needs or can use. They are two main resources that you can specify requests and limits for, CPU and Memory.\nAs StorageOS is an infrastructure component, the health of other applications depends on being able to write to the StorageOS volumes. As such it is of paramount importance to avoid restarts of the StorageOS DaemonSet Pods. Restarting a StorageOS Pod results in the volumes of the node the StorageOS Pod is running on being marked as Read Only, and causes the failover of primary volumes on that node to their replicas. After a StorageOS Pod restart, once the StorageOS DaemonSet Pod is \u0026ldquo;READY\u0026rdquo;, the application Pods running on the node need to be restarted in order to trigger a mount of the filesystem hosted on the StorageOS volume and resume normal operations. To avoid restarts of the StorageOS main container by Kubernetes due to resource limits being reached, it is recommended to not set resource limits on the StorageOS DaemonSet. In addition to avoiding resource limits, StorageOS uses a high priority class when the DaemonSet is installed in the \u0026lsquo;kube-system\u0026rsquo; namespace. That avoids the DaemonSet Pods of being evicted.\nFor more information about managing resources for containers please see the Kubernetes documentation\nDefining Pod resource requests and reservations To add resource requests and reservations to the StorageOS DaemonSet configure them in the StorageOSCluster resource.\n","excerpt":"Managing Resources for StorageOS containers Kubernetes resource requests and limits are two optional …","ref":"/v2.2/docs/reference/resource_requests_and_limits/","title":"Resource Requests and Limits"},{"body":"","excerpt":"","ref":"/v2.2/search/","title":"Search Results"},{"body":"Backing up files from StorageOS volumes In this example use case we provide three different strategies for accessing files that have been written to a StorageOS persistent volume.\nIn the following examples the \u0026ldquo;application\u0026rdquo; container is the container main, which has a rsync, Nginx or sftp sidecar container. The StorageOS volume that the application is writing to will be mounted into the sidecar container so files written by the application are available for export. Files can be exported using Nginx as a web file server, transferred using rsync or accessed via SFTP.\nThe files create a stateful set that can be used AFTER a StorageOS cluster has been created. See our guide on how to install StorageOS on Kubernetes for more information.\nClone Repository In order to deploy the examples, clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/backup Before deploying the backup-example stateful set we recommend looking through the examples to understand how the different containers are configured\n Exfiltrating files through HTTP Deploy the Nginx example\n$ kubectl create -f nginx/ service/backup-example created configmap/nginx-config created statefulset.apps/backup-example created pod/busybox created Check that a backup-example pod is running\n$ kubectl get pods -w -l app=backup-example-nginx NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-nginx-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-nginx ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 80/TCP 46s Use wget to access the files served by Nginx. Nginx is sharing files from the same volume that the main application container is writing to. The connection to the Nginx container is made via the backup-example service.\n$ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;lost%2Bfound/\u0026#34;\u0026gt;lost+found/\u0026lt;/a\u0026gt; 12-Feb-2019 12:32 - \u0026lt;a href=\u0026#34;date.txt\u0026#34;\u0026gt;date.txt\u0026lt;/a\u0026gt; 12-Feb-2019 12:49 29 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx/date.txt Tue Feb 12 12:49:15 UTC 2019 Depending on what files have been written to the StorageOS volume the output of the index file will be different. In the example the date.txt file we created in Step 2 is present on the volume.\nExfiltrating files through Rsync Deploy the rsync example\n$ kubectl create -f rsync/ service/backup-example created configmap/rsync-config created secret/rsync-credentials created statefulset.apps/backup-example created pod/rsync created Check that a backup-example pod is running\n$ kubectl get pods -w -l app=backup-example-rsync NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-rsync-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-rsync NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-rsync ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 873/TCP 46s Use rsync to access the files shared by the rsync daemon. rsync is sharing files from the same volume that the main container is writing to. A username and password that are set in the rsync-credentials secret. The secret supplied in the example has the username and password set to username and password.\n$ kubectl exec -it rsync sh / # rsync --list-only rsync://username@backup-example-rsync/share Password: drwxr-xr-x 4,096 2019/02/12 12:49:15 . -rw-r--r-- 29 2019/02/12 12:49:15 date.txt drwx------ 16,384 2019/02/12 12:32:40 lost+found / # rsync -chavzP rsync://username@backup-example-rsync/share/date.txt . Password: receiving incremental file list date.txt 29 100% 28.32kB/s 0:00:00 (xfr#1, to-chk=0/1) sent 43 bytes received 135 bytes 50.86 bytes/sec total size is 29 speedup is 0.16 / # cat date.txt Tue Feb 12 12:49:15 UTC 2019 In the example above the list of available files was shown and a file called date.txt was synchronized to the rsync container.\nExfiltrating files through SFTP Deploy the sftp example\n$ kubectl create -f sftp/ Exec into the main container and write some data to a file\n$ kubectl exec -it backup-example-sftp-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists\n$ kubectl get svc backup-example-sftp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-sftp ClusterIP 100.70.50.56 \u0026lt;none\u0026gt; 22/TCP 2h Use SFTP to access the files shared by the SFTP container. If you have made no changes to the sftp-config secret the password is password.\n$ kubectl exec -it sftp -- bash root@sftp:/# sftp alex@backup-example-sftp alex@backup-example-sftp\u0026#39;s password: Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found sftp\u0026gt; get date.txt Fetching /date.txt to date.txt /date.txt 100% 29 15.9KB/s 00:00 sftp\u0026gt; bye root@sftp:/# cat date.txt Tue Feb 12 17:51:32 UTC 2019 In order to do this a SFTP user needs to be configured. The details for the user are stored in the sftp-config secret (see sftp/17-secret.yaml). The secret consists of base64 encoded username:password:uid:guid and the user is chroot\u0026rsquo;ed inside their home directory so the mount point for the StorageOS volume in the SFTP container in sftp/20-backup-pod.yaml needs to be configured.\nUsing custom SSH Keys The ConfigMap ssh-key-pub (see sftp/15-configmap.yaml) needs to be populated with a public key. The corresponding private key needs to be base64 encoded and put into the ssh-key-private secret (see sftp/17-secret.yaml). The user to connect as is determined by the user that is configured in the sftp-config configMap. To restrict logins to the SSH key edit the sftp-config secret so it contains no password (user::uid:guid).\n Connect to the sftp pod and connect through the service to the SFTP container running inside the backup-example pod.\n$ kubectl exec -it sftp -- bash root@sftp:/# sftp -i /home/alex/.ssh/id_rsa alex@backup-example-sftp Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found ","excerpt":"Backing up files from StorageOS volumes In this example use case we provide three different …","ref":"/v2.2/docs/usecases/sidecar-backups/","title":""},{"body":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for instance, StorageOS. A StorageClass defines parameters to pass to the provisioner, which in the case of StorageOS can be translated into behaviour applied to the Volumes. Many StorageClasses can be provisioned to apply different feature labels to the StorageOS Volumes.\nBy default the StorageOS Cluster Operator installs the fast StorageClass at bootstrap of StorageOS. You can define its name in the StorageOS Cluster Resource.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:fastlabels:app:storageosprovisioner:csi.storageos.com# CSI DriverallowVolumeExpansion:trueparameters:pool:defaultcsi.storage.k8s.io/fstype:ext4# Change the NameSpace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/controller-expand-secret-name:csi-controller-expand-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-expand-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# NameSpace that runs StorageOS DaemonsetStorageClasses can be created to define default labels for StorageOS volumes, but also to map to any semantic aggregation of volumes that suits your use case, whether there are different roles (dev, staging, prod), or a StorageClass maps to a team or customer using the cluster.\nExamples You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic StorageClass definition in v2-storageclass-replicated.yaml.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# CSI DriverallowVolumeExpansion:trueparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the NameSpace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/controller-expand-secret-name:csi-controller-expand-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-expand-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# NameSpace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# NameSpace that runs StorageOS DaemonsetThat StorageClass can be used by a PVC:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels.\nYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\n","excerpt":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for …","ref":"/v2.2/docs/operations/storageclasses/","title":"Kubernetes StorageClasses"},{"body":"","excerpt":"","ref":"/v2.2/","title":"StorageOS"},{"body":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for two different purposes.\nTelemetry Telemetry is made up a DNS version check query and a once per day report. Error reporting is the sentry.io crash dump reporting. sentry.io StorageOS sends crash reports to sentry.io. This information helps our developers monitor and fix crashes. Information is sent to sentry.io when a process inside the StorageOS container crashes.\n The crash report contains the signal that triggered the shutdown (e.g. SIGSEGV), the exit code and whether or not the crash generated a core dump. All StorageOS clusters with a routable connection to the internet will send crash reports to sentry.io over tcp/443. StorageOS respects environment variables that ProxyFromEnvironment uses.\nAn exhaustive list of information included in the crash report is below:\n StorageOS version Crash description string Anonymized Cluster ID Anonymized Node ID DNS Query StorageOS will perform a \u0026ldquo;latest version check\u0026rdquo; using a DNS query in order to inform administrators that a new version is available. StorageOS will also send anonymized node ids, cluster id and StorageOS version information to StorageOS using a DNS query. The information that we send in the query is encoded as well as being anonymized. This query allows us to inform Cluster admins when StorageOS upgrades are available in the StorageOS GUI and in the logs.\nThe DNS query includes:\n Anonymized StorageOS Cluster ID Anonymized StorageOS node ID StorageOS version number Once Per Day Report The once per day report contains information about the StorageOS cluster and Kubernetes versions to help StorageOS focus our development efforts on the most popular platforms. The once per day data is encrypted and sent to a StorageOS telemetry server so it is never processed outside of StorageOS assets.\nAn exhaustive list of information included in the once per day report is below:\n api_call_metrics cluster_disable_crash_reporting cluster_disable_version_check cluster_log_format cluster_log_level cluster_tls_provided k8s_distribution k8s_in_k8s k8s_scheduler_extender_enabled k8s_version node_available_bytes node_capacity node_crash_files_on_disk node_created_at_time node_etcd_config node_etcd_namespacing_enabled node_etcd_tls_enabled node_free_bytes node_health node_http_tls_enabled node_id node_labels node_status node_storageos_version node_system_clock_time node_total_bytes node_version volume_fs_type volume_id volume_labels volume_master_attach volume_master_delete_deployment volume_master_detach volume_master_failover_deployment volume_master_promote volume_master_provision volume_master_recover_replica volume_master_trigger_rejoin volume_metrics volume_placement_strategy volume_provision_source_user_agent volume_replica_delete_deployment volume_replica_failover_deployment volume_replica_promote volume_replica_provision volume_replicas volume_replica_trigger_rejoin volume_size_bytes Disable Telemetry It is possible to disable telemetry using the GUI, CLI, API, environment variables or the StorageOS Cluster Spec.\nStorageOS Cluster Spec Disable telemetry explicitly through the configurable spec parameters of the StorageOSCluster custom resource.\nEnvironment Variables You can use the following environmental variables to disable or enable telemetry.\nDISABLE_VERSION_CHECK # Disable the DNS version check DISABLE_TELEMETRY # Disable the once per day reporting DISABLE_ERROR_REPORTING # Disable sentry.io crash reports ","excerpt":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for …","ref":"/v2.2/docs/reference/telemetry/","title":"StorageOS Telemetry"},{"body":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The bundle packages the information needed for the engineering team to understand the context of your cluster and begin troubleshooting any issues you may be experiencing.\nGenerate Bundle CLI The StorageOS CLI can generate the bundle.\n$ storageos get diagnostics The file generated is in the form of diagnostics-${TIME_STAMP}.gz\n GUI Or you can use the StoregeOS GUI.\n Go to section \u0026ldquo;Cluster\u0026rdquo; Press the button \u0026ldquo;DOWNLOAD DIAGNOSTICS\u0026rdquo;. Data collected in the bundle Most of the data collected in the bundle is regarding the state of the StorageOS cluster, however some other information regarding the infrastructure is also gathered. The information is used to have a clear view of the cluster where StorageOS is running.\nThe bundle incorporates for each node:\n StorageOS Daemonset Pod logs lshw dmesg (kernel logs) StorageOS metadata for the ControlPlane and DataPlane StorageOS metadata collected cluster metadata namespaces metadata nodes metadata volumes metadata capacity stats environment variables health Privacy StorageOS can only obtain the bundle if it is downloaded by the user and given to our engineering team, or uploaded for analysis. The data received by StorageOS is private and never leaves nor will leave StorageOS Inc.\nThe data contained in the cluster diagnostic bundle has the sole purpose of helping customers troubleshoot their issues.\n","excerpt":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The …","ref":"/v2.2/docs/reference/bundle/","title":"Support bundle"},{"body":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base directory for storing configuration and blob files. Recommended host filesystem types are ext4 or xfs. If you require a specific filesystem please contact StorageOS.\nPersistent Volume Filesystems StorageOS provides a block device on which a file system can be created. The creation of the filesystem is either handled by StorageOS or by Kubernetes which affects what filesystems can be created.\nCSI Driver When using StorageOS with the CSI driver, StorageOS is responsible for running mkfs against the block device that pods mount. StorageOS is able to create ext2, ext3, ext4 and xfs file systems.\n","excerpt":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base …","ref":"/v2.2/docs/reference/filesystems/","title":"Supported File systems"},{"body":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain node taints. Taints can be thought of as the opposite of Node affinity in that taints repel pods. Node taints are automatically applied by Kubernetes in response to node resources coming under contention. As StorageOS provides storage to pods it should not be evicted during periods of resource contention, as any pods using StorageOS volumes on the same node would need to be restarted.\nAs StorageOS runs as a daemonset some Kubernetes tolerations are added by Kubernetes while others are automatically added by the StorageOS operator.\nFor more information about tolerations please see the Kubernetes documentation.\ntolerations:# The unreachable and not-ready tolerations are added by Kubernetes to daemonsets automatically- key:\u0026#34;node.kubernetes.io/unreachable\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;- key:\u0026#34;node.kubernetes.io/not-ready\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;# The following tolerations are added to the StorageOS daemonset by the StorageOS operator- key:node.kubernetes.io/disk-pressureoperator:Exists- key:node.kubernetes.io/memory-pressureoperator:Exists- key:node.kubernetes.io/network-unavailableoperator:Exists- key:node.kubernetes.io/out-of-diskoperator:Exists- key:node.kubernetes.io/pid-pressureoperator:Exists- key:node.kubernetes.io/unschedulableoperator:ExistsAdding Custom Tolerations To add custom tolerations to the StorageOS daemonset configure them in the StorageOSCluster resource.\n","excerpt":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain …","ref":"/v2.2/docs/reference/tolerations/","title":"Tolerations"},{"body":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to the StorageOS installation, integration with orchestrators or common misconfigurations.\nTools To be able to troubleshoot issues the StorageOS cli is required.\nPod in pending because of mount error Issue: The output of kubectl describe pod $POD_ID contains no such file or directory and references the StorageOS volume device file.\nroot@node1:~# kubectl -n kube-system describe $POD_ID (...) Events: (...) Normal Scheduled 11s default-scheduler Successfully assigned default/d1 to node3 Warning FailedMount 4s (x4 over 9s) kubelet, node3 MountVolume.SetUp failed for volume \u0026#34;pvc-f2a49198-c00c-11e8-ba01-0800278dc04d\u0026#34; : stat /var/lib/storageos/volumes/d9df3549-26c0-4cfc-62b4-724b443069a1: no such file or directory Reason: There are two main reasons this issue may arise:\n The StorageOS DEVICE_DIR location is wrongly configured when using Kubelet as a container Mount Propagation is not enabled (Option 1) Misconfiguration of the DeviceDir/SharedDir\nSome Kubernetes distributions such as Rancher, DockerEE or some installations of OpenShift deploy the Kubelet as a container, because of this, the device files that StorageOS creates to mount into the containers need to be visible to the kubelet. StorageOS can be configured to share the device directory.\nModern installations use CSI, which handles the complexity internally.\nAssert: root@node1:~# kubectl -n default describe stos | grep \u0026#34;Shared Dir\u0026#34; Shared Dir: # \u0026amp;lt;-- Shouldn\u0026#39;t be blank Solution: The Cluster Operator Custom Definition should specify the SharedDir option as follows.\nspec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container ... See example on how to configure the StorageOS Custom Resource.\n (Option 2) Mount propagation is not enabled.\n Applies only if Option 1 is configured properly.\n Assert: If not using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/storageos/volumes is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/storageos/volumes/ root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/storageos/volumes bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nIf using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/kubelet/plugins/kubernetes.io~storageos/devices is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/kubelet/plugins/kubernetes.io~storageos/devices root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/kubelet/plugins/kubernetes.io~storageos/devices bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nSolution: Older versions of Kubernetes need to enable mount propagation as it is not enabled by default. Most Kubernetes distributions allow MountPropagation to be enabled using FeatureGates. Rancher specifically, needs to enable it in the \u0026ldquo;View in API\u0026rdquo; section of your cluster. You need to edit the section \u0026ldquo;rancherKubernetesEngineConfig\u0026rdquo; to enable the Kubelet feature gate.\nPVC pending state - Failed to dial StorageOS A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: root@node1:~/# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE vol-1 Pending fast 7s kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 7s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: Get http://storageos-cluster/version: failed to dial all known cluster members, (10.233.59.206:5705) Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount volume can\u0026rsquo;t be transmitted to StorageOS, hence the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nIn this case, the Event message indicates that StorageOS API is not responding, implying that StorageOS is not running. For Kubernetes to define StorageOS pods ready, the health check must pass.\nAssert: Check the status of StorageOS pods.\nroot@node1:~/# kubectl -n kube-system get pod --selector app=storageos # for CSI add --selector kind=daemonset NAME READY STATUS RESTARTS AGE storageos-qrqkj 0/1 Running 0 1m storageos-s4bfv 0/1 Running 0 1m storageos-vcpfx 0/1 Running 0 1m storageos-w98f5 0/1 Running 0 1m If the pods are not READY, the service will not forward traffic to the API they serve hence PVC will remain in pending state until StorageOS pods are available.\n Kubernetes keeps trying to execute the action until it succeeds. If a PVC is created before StorageOS finish starting, the PVC will be created eventually.\n Solution: StorageOS health check takes 60 seconds of grace before reporting as READY. If StorageOS is starting properly after that period, the volume will be created when StorageOS finishes its bootstrap. If StorageOS is not running or is not starting properly, the solution would be to troubleshoot the installation. PVC pending state - Secret Missing A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 13s (x2 over 28s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: failed to get secret from [\u0026#34;storageos\u0026#34;/\u0026#34;storageos-api\u0026#34;] Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount a volume can\u0026rsquo;t be transmitted to StorageOS, and the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nThe StorageClass provisioned for StorageOS references a Secret from where it retrieves the API endpoint and the authentication parameters. If that secret is incorrect or missing, the connections won\u0026rsquo;t be established. It is common to see that the Secret has been deployed in a different namespace where the StorageClass expects it or that is has been deployed with a different name.\nAssert: Check the StorageClass parameters to know where the Secret is expected to be found.\nkubectl get storageclass fast -o yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: 2018-09-25T08:44:57Z labels: app: storageos name: fast resourceVersion: \u0026#34;108853\u0026#34; selfLink: /apis/storage.k8s.io/v1/storageclasses/fast uid: 48490a9b-c09f-11e8-ba01-0800278dc04d parameters: adminSecretName: storageos-api adminSecretNamespace: storageos description: Kubernetes volume fsType: ext4 pool: default provisioner: kubernetes.io/storageos reclaimPolicy: Delete Note that the parameters specify adminSecretName and adminSecretNamespace.\n Check if the secret exists according to those parameters\nkubectl -n kube-system get secret storageos-api No resources found. Error from server (NotFound): secrets \u0026#34;storageos-api\u0026#34; not found If no resources are found, it is clear that the Secret doesn\u0026rsquo;t exist or it is not deployed in the right location.\n Solution: Deploy StorageOS following the installation procedures. If you are using the manifests provided for Kubernetes to deploy StorageOS rather than using automated provisioners, make sure that the StorageClass parameters and the Secret reference match.\nNode name different from Hostname Issue: StorageOS nodes can\u0026rsquo;t join the cluster showing the following log entries.\ntime=\u0026#34;2018-09-24T13:47:02Z\u0026#34; level=error msg=\u0026#34;failed to start api\u0026#34; error=\u0026#34;error verifying UUID: UUID aed3275f-846b-1f75-43a1-adbfec8bf974 has already been registered and has hostname \u0026#39;debian-4\u0026#39;, not \u0026#39;node4\u0026#39;\u0026#34; module=command Reason: The StorageOS registration process to start the cluster uses the hostname of the node where the StorageOS container is running, provided by the Kubelet. However, StorageOS verifies the network hostname of the OS as a prestart check to make sure it can communicate with other nodes. If those names don\u0026rsquo;t match, StorageOS will be unable to start.\nSolution: Make sure the hostnames match with the Kubernetes advertised names. If you have changed the hostname of your nodes, make sure that you restart the nodes to apply the change.\nPeer discovery - Networking Issue: StorageOS nodes can\u0026rsquo;t join the cluster showing the following logs after one minute of container uptime.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover nodes in the cluster. When StorageOS starts, one or more nodes can be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nAssert: It is likely that ports are block by a firewall.\nSSH into one of your nodes and check connectivity to the rest of the nodes.\n# Successfull execution: [root@node06 ~]# nc -zv node04 5705 Ncat: Version 7.50 ( https://nmap.org/ncat ) Ncat: Connected to 10.0.1.166:5705. Ncat: 0 bytes sent, 0 bytes received in 0.01 seconds. StorageOS exposes network diagnostics in its API, viewable from the CLI. To use this feature, the CLI must query the API of a running node. The diagnostics show information from all known cluster members. If all the ports are blocked during the first bootstrap of the cluster, the diagnostics won\u0026rsquo;t show any data as nodes couldn\u0026rsquo;t register.\n StorageOS networks diagnostics are available for storageos-rc5 and storageos-cli-rc3 and above.\n # Example: root@node1:~# storageos cluster connectivity SOURCE NAME ADDRESS LATENCY STATUS MESSAGE node4 node2.nats 172.28.128.4:5708 1.949275ms OK node4 node3.api 172.28.128.5:5705 3.070574ms OK node4 node3.nats 172.28.128.5:5708 2.989238ms OK node4 node2.directfs 172.28.128.4:5703 2.925707ms OK node4 node3.etcd 172.28.128.5:5707 2.854726ms OK node4 node3.directfs 172.28.128.5:5703 2.833371ms OK node4 node1.api 172.28.128.3:5705 2.714467ms OK node4 node1.nats 172.28.128.3:5708 2.613752ms OK node4 node1.etcd 172.28.128.3:5707 2.594159ms OK node4 node1.directfs 172.28.128.3:5703 2.601834ms OK node4 node2.api 172.28.128.4:5705 2.598236ms OK node4 node2.etcd 172.28.128.4:5707 16.650625ms OK node3 node4.nats 172.28.128.6:5708 1.304126ms OK node3 node4.api 172.28.128.6:5705 1.515218ms OK node3 node2.directfs 172.28.128.4:5703 1.359827ms OK node3 node1.api 172.28.128.3:5705 1.185535ms OK node3 node4.directfs 172.28.128.6:5703 1.379765ms OK node3 node1.etcd 172.28.128.3:5707 1.221176ms OK node3 node1.nats 172.28.128.3:5708 1.330122ms OK node3 node2.api 172.28.128.4:5705 1.238541ms OK node3 node1.directfs 172.28.128.3:5703 1.413574ms OK node3 node2.etcd 172.28.128.4:5707 1.214273ms OK node3 node2.nats 172.28.128.4:5708 1.321145ms OK node1 node4.directfs 172.28.128.6:5703 1.140797ms OK node1 node3.api 172.28.128.5:5705 1.089252ms OK node1 node4.api 172.28.128.6:5705 1.178439ms OK node1 node4.nats 172.28.128.6:5708 1.176648ms OK node1 node2.directfs 172.28.128.4:5703 1.529612ms OK node1 node2.etcd 172.28.128.4:5707 1.165681ms OK node1 node2.api 172.28.128.4:5705 1.29602ms OK node1 node2.nats 172.28.128.4:5708 1.267454ms OK node1 node3.nats 172.28.128.5:5708 1.485657ms OK node1 node3.etcd 172.28.128.5:5707 1.469429ms OK node1 node3.directfs 172.28.128.5:5703 1.503015ms OK node2 node4.directfs 172.28.128.6:5703 1.484ms OK node2 node1.directfs 172.28.128.3:5703 1.275304ms OK node2 node4.nats 172.28.128.6:5708 1.261422ms OK node2 node4.api 172.28.128.6:5705 1.465532ms OK node2 node3.api 172.28.128.5:5705 1.252768ms OK node2 node3.nats 172.28.128.5:5708 1.212332ms OK node2 node3.directfs 172.28.128.5:5703 1.192792ms OK node2 node3.etcd 172.28.128.5:5707 1.270076ms OK node2 node1.etcd 172.28.128.3:5707 1.218522ms OK node2 node1.api 172.28.128.3:5705 1.363071ms OK node2 node1.nats 172.28.128.3:5708 1.349383ms OK Solution: Open ports following the prerequisites page.\nPeer discovery - Pod allocation Issue: StorageOS nodes can\u0026rsquo;t join the cluster and show the following log entries.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover the nodes in the cluster. When StorageOS starts, one or more active nodes must be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nIf there are no active StorageOS nodes, the bootstrap process will elect the first node in the JOIN variable as master, and the rest will try to discover from it. In case of that node not starting, the whole cluster will remain unable to bootstrap.\nInstallations of StorageOS use a DaemonSet, and by default do not schedule StorageOS pods to master nodes, due to the presence of the node-role.kubernetes.io/master:NoSchedule taint that is typically present. In such cases the JOIN variable must not contain master nodes or the StorageOS cluster will remain unable to start.\nAssert: Check that the first node of the JOIN variable started properly.\nroot@node1:~/# kubectl -n kube-system describe ds/storageos | grep JOIN JOIN: 172.28.128.3,172.28.128.4,172.28.128.5 root@node1:~/# kubectl -n kube-system get pod -o wide | grep 172.28.128.3 storageos-8zqxl 1/1 Running 0 2m 172.28.128.3 node1 Solution: Make sure that the JOIN variable doesn\u0026rsquo;t specify the master nodes. In case you are using the discovery service, it is necessary to ensure that the DaemonSet won\u0026rsquo;t allocate Pods on the masters. This can be achieved with taints, node selectors or labels.\nFor installations with the StorageOS operator you can specify which nodes to deploy StorageOS on using nodeSelectors. See examples in the Cluster Operator Examples page.\nFor more advanced installations using compute-only and storage nodes, check the storageos.com/deployment=computeonly label that can be added to the nodes through Kubernetes node labels, or StorageOS in the Labels page.\nLIO Init:Error Issue: StorageOS pods not starting with Init:Error\nkubectl -n kube-system get pod NAME READY STATUS RESTARTS AGE storageos-2kwqx 0/3 Init:Err 0 6s storageos-cffcr 0/3 Init:Err 0 6s storageos-d4f69 0/3 Init:Err 0 6s storageos-nhq7m 0/3 Init:Err 0 6s Reason: This indicates that since the Linux open source SCSI drivers are not enabled, StorageOS cannot start. The StorageOS DaemonSet enables the required kernel modules on the host system. If you are seeing these errors it is because that container couldn\u0026rsquo;t load the modules.\nAssert Check the logs of the init container.\nkubectl -n kube-system logs $ANY_STORAGEOS_POD -c storageos-init In case of failure, it will show the following output, indicating which kernel modules couldn\u0026rsquo;t be loaded or that they are not properly configured:\nChecking configfs configfs mounted on sys/kernel/config Module target_core_mod is not running executing modprobe -b target_core_mod Module tcm_loop is not running executing modprobe -b tcm_loop modprobe: FATAL: Module tcm_loop not found. Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and delete StorageOS pods, allowing the DaemonSet to create the pods again.\nLIO not enabled Issue: StorageOS node can\u0026rsquo;t start and shows the following log entries.\ntime=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;liocheck returned error\u0026#34; category=liocheck error=\u0026#34;exit status 1\u0026#34; module=dataplane stderr=\u0026#34;Sysfs root \u0026#39;/sys/kernel/config/target\u0026#39; is missing, is kernel configfs present and target_core_mod loaded? category=fslio level=warn\\nRuntime error checking stage \u0026#39;target_core_mod\u0026#39;: SysFs root missing category=fslio level=warn\\nliocheck: FAIL (lio_capable_system() returns failure) category=fslio level=fatal\\n\u0026#34; stdout= time=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;failed to start dataplane services\u0026#34; error=\u0026#34;system dependency check failed: exit status 1\u0026#34; module=command Reason: This indicates that one or more kernel modules required for StorageOS are not loaded.\nAssert The following kernel modules must be enabled in the host.\nlsmod | egrep \u0026#34;^tcm_loop|^target_core_mod|^target_core_file|^configfs\u0026#34; Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and restart the container.\n(OpenShift) StorageOS pods missing \u0026ndash; DaemonSet error StorageOS DaemonSet doesn\u0026rsquo;t have any pod replicas. The DaemonSet couldn\u0026rsquo;t allocate any Pod due to security issues.\nIssue: [root@master02 standard]# oc get pod No resources found. [root@master02 standard]# oc describe daemonset storageos (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedCreate 0s (x12 over 10s) daemonset-controller Error creating: pods \u0026#34;storageos-\u0026#34; is forbidden: unable to validate against any security context constraint: [provider restricted: .spec.securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used provider restricted: .spec.securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.volumes[0]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[1]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[2]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[3]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.initContainers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.initContainers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.initContainers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.initContainers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.containers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.containers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.containers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used] Reason: The OpenShift cluster has security context constraint policies enabled that forbid any pod, without an explicitly set policy for the service account, to be allocated.\nAssert: Check if the StorageOS ServiceAccount can create pods with enough permissions\noc get scc privileged -o yaml # Or custom scc with enough privileges (...) users: - system:admin - system:serviceaccount:openshift-infra:build-controller - system:serviceaccount:management-infra:management-admin - system:serviceaccount:management-infra:inspector-admin - system:serviceaccount:storageos:storageos \u0026amp;lt;-- - system:serviceaccount:tiller:tiller If the StorageOS sa system:serviceaccount:storageos:storageos is in the privileged scc it will be able to create pods.\nSolution: Add the ServiceAccount system:serviceaccount:storageos:storageos to a scc with enough privileges.\noc adm policy add-scc-to-user privileged system:serviceaccount:storageos:storageos Getting Help If our troubleshooting guides do not help resolve your issue, please see our support section for details on how to get in touch with us.\n","excerpt":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to …","ref":"/v2.2/docs/operations/troubleshooting/","title":"Troubleshooting"},{"body":"This document details a step-by-step procedure to remove StorageOS from a Kubernetes cluster.\nRemember that StorageOS enables the stateful applications within your cluster. It is very important to remove any applications that rely on StorageOS before you remove StorageOS itself, or those applications will suffer unrecoverable errors.\nRemove Stateful Workloads and Data Delete any resources using StorageOS volumes\nDelete any statefulsets, deployments or pods that are using StorageOS Volumes.\n Delete PVCs using StorageOS\nDelete any Persistent Volume Claims that are using StorageOS.\n$ kubectl -n $NS delete pvc $PVC This will delete data held by StorageOS and won\u0026rsquo;t be recoverable.\n Remove StorageOS Cluster Delete StorageOS Cluster\n$ kubectl get storageoscluster --all-namespaces # Find the namespace where the Custom Resource runs $ kubectl -n $NS delete storageoscluster --all # Usually to be found in storageos-operator Wait until the StorageOS resources are gone\n$ kubectl -n kube-system get pod # NS: Namespace where StorageOS Daemonset is running, usually \u0026#39;kube-system\u0026#39; Uninstall the StorageOS Operator Delete the Cluster Operator once the StorageOS Pods are terminated\n Delete the StorageOS Operator deployment\n$ kubectl delete -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Procedure is finished. StorageOS is now uninstalled.\nRemove StorageOS contents and metadata (unrecoverable) The steps up until now have been recoverable - as long as the etcd backing StorageOS and the contents of /var/lib/storageos on your nodes are safe then StorageOS can be reinstalled. For complete removal and recovery of disk space, you can use the following procedure.\n The following steps will delete all data held by StorageOS and won\u0026rsquo;t be recoverable.\n Remove the StorageOS data directory\nThere are two ways to remove the StorageOS data directory:\n (Option 1) Login in to the hosts and execute the following commands\n$ sudo rm -rf /var/lib/storageos $ sudo umount /var/lib/kubelet/plugins_registry/storageos (Option 2) Execute the following command to deploy a DaemonSet that removes the StorageOS data directory.\n N.B This step is irreversible and once the data is removed it cannot be recovered.\n Run the following command where kubectl is installed and with the context set for your Kubernetes cluster.\n $ curl -s https://docs.storageos.com/v2.2/sh/permanently-delete-storageos-data.sh | bash Flush Etcd Data\nThis will remove any keys written by StorageOS.\n$ export ETCDCTL_API=3 $ etcdctl --endpoints=http://$ETCD_IP:2379 del --prefix \u0026#34;storageos\u0026#34; If running Etcd with mTLS, you can set the certificates location with the following command.\n$ export ETCDCTL_API=3 $ etcdctl --endpoints=https://$ETCD_IP:2379 \\ --cacert=/path/to/ca.pem \\ --cert=/path/to/client-cert.pem \\ --key=/path/to/client-key.pem \\ del --prefix \u0026#34;storageos\u0026#34; ","excerpt":"This document details a step-by-step procedure to remove StorageOS from a Kubernetes cluster. …","ref":"/v2.2/docs/operations/uninstall/","title":"Uninstall StorageOS"},{"body":"This document details a step-by-step procedure to upgrade a StorageOS v2 cluster.\nKeep in mind that upgrading a cluster will require minor downtime of applications using StorageOS volumes. However we will take steps to minimize the required downtime as much as possible.\nUpgrade StorageOS Warning: To reduce downtime, it is recommended to docker pull the new StorageOS container image storageos/node:v2.2.0 on the nodes beforehand so that the cluster spins up faster!\n First make sure you keep a backup of all the StorageOS yaml files. You can reuse the StorageOSCluster configuration file to easily upgrade your cluster. You can also backup the Statefulset yaml files to keep track of the replicas.\nkubectl get pod -n storageos-operator -o yaml \u0026gt; storageos_operator.yaml kubectl get storageoscluster -n storageos-operator -o yaml \u0026gt; storageos_cr.yaml kubectl get statefulset --all-namespaces \u0026gt; statefulset-sizes.yaml Scale all stateful applications that use StorageOS volumes to 0.\n Delete the StorageOSCluster CR.\nkubectl delete storageoscluster cluster-storageos -n storageos-operator Deploy the new operator.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.2.0/storageos-operator.yaml Warning: If you have made changes to the CRDs, Service Account or Cluster Role, make sure you migrate those changes in the StorageOS operator yaml.\n Edit the StorageOSCluster Custom Resource (storageos_cr.yaml) with the new node image version.\nimages:nodeContainer:\u0026#34;storageos/node:v2.2.0\u0026#34;If you\u0026rsquo;re not using any of the hardcoded container image versions make sure to delete them as well and only leave the nodeContainer image as shown above.\nMake sure to also delete all metadata that you\u0026rsquo;re not using. The final metadata stanza should be something similar to that:\nmetadata:name:\u0026#34;cluster-storageos\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34; Re-create the StorageOSCluster\nkubectlcreate-fstorageos_cr.yaml Wait for all the StorageOS pods to enter the RUNNING state\nkubectl get pods -l app=storageos -A -w Scale your stateful applications back up.\n Congratulations, you now have the latest version StorageOS!\n","excerpt":"This document details a step-by-step procedure to upgrade a StorageOS v2 cluster.\nKeep in mind that …","ref":"/v2.2/docs/operations/upgrade/","title":"Upgrade StorageOS"},{"body":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces using policies.\n Note: Users are created with access to the default namespace. This access is only revoked when a policy is created for the user or their group.\n Managing users Users can be created and deleted by navigating to the \u0026ldquo;Users\u0026rdquo; section of the GUI.\nAlternatively users can also be managed using the CLI.\nTo create a user with the CLI, run:\n$ storageos user create jim The above command will create a user named jim. The command will also prompt you to enter a password for the newly created user.\n","excerpt":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces …","ref":"/v2.2/docs/operations/users/","title":"User Management"},{"body":"Velero Backups Using Velero to backup your StorageOS volumes Velero is an open source tool to safely backup and restore, perform disaster recovery, and migrate Kubernetes cluster resources and persistent volumes.\nVelero consists of the following components:\n A Velero Server in the cluster A CLI client A restic daemonset in the cluster Here\u0026rsquo;s a diagram on how Velero backups work:\nAnd here\u0026rsquo;s how Velero restores work:\nIn our case, we will use MinIO in the Kubernetes cluster as the \u0026ldquo;cloud provider\u0026rdquo; pictured in the backup diagram. MinIO is an object store that uses an S3 compatible API and as such can be used to store our backed up resources.\nWe will set up MinIO through a StatefulSet with a 50GiB StorageOS volume and a ClusterIP service. The UI can be accessed through the browser by port-forwarding the MinIO service.\nVelero uses Restic to backup Kubernetes volumes. Restic is a fast and secure backup program for filesystems whose documentation can be found here. The way it works is that it scans the volume directory for its files and then splits those files into blobs which are then sent to MinIO. Here\u0026rsquo;s how it integrates with Velero.\nPrerequisites Here are the prerequisites for running Velero in your Kubernetes cluster:\n Kubernetes cluster version 1.13+ with DNS Kubectl installed Velero cli installed https://Velero.io/docs/main/basic-install/ Velero can also be installed from a helm chart\n Install MinIO with a StorageOS volume First, make sure to clone the StorageOS use cases repository and navigate to the Velero directory:\ngit clone https://github.com/storageos/use-cases.git cd use-cases/Velero Installing MinIO is really simple, just deploy it using the minio-deploy.yaml manifest file:\nkubectl apply -f ./minio Confirm that MinIO was deployed successfully:\n$ kubectl get pods -n velero NAME READY STATUS RESTARTS AGE minio-0 1/1 Running 0 3m48s minio-setup-zvcdg 0/1 Completed 1 3m47s You can access the web UI of MinIO by port-forwarding the MinIO service with this command:\nkubectl port-forward service/minio -n velero 9000 Install Velero Use the following command to install Velero via the Velero CLI or alternatively use the helm chart. To install it using the Velero cli, just run this command:\n The AWS plugin is being used because MinIO implements the S3 API. This is required even if you\u0026rsquo;re not using AWS.\n velero install \\ --provider aws \\ --plugins velero/velero-plugin-for-aws:v1.0.0 \\ --bucket velero \\ --secret-file ./credentials-Velero \\ --use-volume-snapshots=false \\ --backup-location-config region=minio,s3ForcePathStyle=\u0026#34;true\u0026#34;,s3Url=http://minio.velero.svc:9000 \\ --use-restic Make sure that Velero is installed correctly:\n$ kubectl logs deployment/velero -n velero ... time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Server started successfully\u0026#34; logSource=\u0026#34;pkg/cmd/server/server.go:881\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=restic-repository logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=restore logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=backup-sync logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=backup logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=backup-deletion logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Checking for expired DeleteBackupRequests\u0026#34; controller=backup-deletion logSource=\u0026#34;pkg/controller/backup_deletion_controller.go:551\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Done checking for expired DeleteBackupRequests\u0026#34; controller=backup-deletion logSource=\u0026#34;pkg/controller/backup_deletion_controller.go:579\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=schedule logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=downloadrequest logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=gc-controller logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; time=\u0026#34;2020-08-25T15:33:09Z\u0026#34; level=info msg=\u0026#34;Starting controller\u0026#34; controller=serverstatusrequest logSource=\u0026#34;pkg/controller/generic_controller.go:76\u0026#34; Quiesce Before moving on to the MySQL use case, it is important to talk about quiescence. To quiesce is to pause or alter a device or application to achieve a consistent state, usually in preparation for a backup or other maintenance. Quiescence is very important when backing up real world applications because in flight or cached write operations of the filesystem during the backup process can cause corruption of the backed up data. It is a common mistake for engineers to back up their data without quiescing the filesystem and noticing corruption when they need to restore from that backup. A simple form of quiescence is the unix command fsfreeze, but in most cases, application specific quiescence should be used instead.\nMySQL use case For the our use case, we\u0026rsquo;ll use Velero with a real world application, MySQL.\n First deploy MySQL, based on the MySQL use case:\nkubectl apply -f ./mysql Notice that the Statefulset also includes 5 annotations:\nannotations:backup.velero.io/backup-volumes:datapre.hook.backup.velero.io/command:\u0026#39;[\u0026#34;/sbin/fsfreeze\u0026#34;, \u0026#34;--freeze\u0026#34;, \u0026#34;/var/lib/mysql\u0026#34;]\u0026#39;pre.hook.backup.velero.io/container:fsfreezepost.hook.backup.velero.io/command:\u0026#39;[\u0026#34;/sbin/fsfreeze\u0026#34;, \u0026#34;--unfreeze\u0026#34;, \u0026#34;/var/lib/mysql\u0026#34;]\u0026#39;post.hook.backup.velero.io/container:fsfreezeThe first annotation specifies which volume to backup using restic. The other annotations are used to perform an fsfreeze on the volume mount point using pre and post backup hooks, for more details about Velero pre/post backup hooks please see their documentation here.\nWe have to specify to use the fsfreeze ubuntu container since the MySQL container doesn\u0026rsquo;t support fsfreeze\n- name:fsfreezeimage:ubuntu:bionicsecurityContext:privileged:truevolumeMounts:- name:datamountPath:/var/lib/mysqlcommand:- \u0026#34;/bin/bash\u0026#34;- \u0026#34;-c\u0026#34;- \u0026#34;sleep infinity\u0026#34; Wait for the pod to spin up:\n$ kubectl get pods -n mysql NAME READY STATUS RESTARTS AGE client 1/1 Running 0 24m mysql-0 2/2 Running 0 24m Exec into the MySQL pod and populate it with data using the commands below.\n$ kubectl exec mysql-0 -n mysql -ti -c mysql -- mysql mysql\u0026gt; create database shop; mysql\u0026gt; use shop; mysql\u0026gt; create table books (title VARCHAR(256), price decimal(4,2)); mysql\u0026gt; insert into books value (\u0026#39;Gates of Fire\u0026#39;, 13.99); mysql\u0026gt; select * from books; +---------------+-------+ | title | price | +---------------+-------+ | Gates of Fire | 13.99 | +---------------+-------+ 1 row in set (0.00 sec) mysql\u0026gt; exit Create the Velero backup:\nvelero backup create mysql-backup --include-namespaces mysql --wait Confirm that all the Kubernetes objects are there and the restic backups completed successfully:\nvelero backup describe mysql-backup --details After the backup is finished, delete the StatefulSet and PVC.\n N.B. It\u0026rsquo;s important to make sure that the StatefulSet is deleted because the restore would be unable to complete if a StatefulSet pod is recreated during the restore process.\n kubectl delete statefulset mysql -n mysql kubectl delete pvc data-mysql-0 -n mysql Make sure that the pod is fully terminated:\n$ kubectl get pods -n mysql NAME READY STATUS RESTARTS AGE client 1/1 Running 0 25m Restore MySQL using Velero:\nvelero restore create --from-backup mysql-backup --wait Wait for the MySQL pod to spin up and see if the data is backed up:\n$ kubectl exec mysql-0 -n mysql -ti -c mysql -- mysql mysql\u0026gt; use shop; mysql\u0026gt; select * from books; +---------------+-------+ | title | price | +---------------+-------+ | Gates of Fire | 13.99 | +---------------+-------+ 1 row in set (0.00 sec) mysql\u0026gt; exit The data should be there and the backup was restored successfully!\n","excerpt":"Velero Backups Using Velero to backup your StorageOS volumes Velero is an open source tool to safely …","ref":"/v2.2/docs/usecases/velero-backups/","title":""},{"body":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version [flags] Flags: -h, --help help for version Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version …","ref":"/v2.2/docs/reference/cli/version/","title":"Version"},{"body":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating the volume config via the CLI or UI. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure the volume is not in use. Please note that StorageOS only supports increasing volume size. For more information about how the resize works please see our Resize concepts page.\nResizing a Volume In order to resize a PVC the storage request field must be updated.\napiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: fast accessModes: - ReadWriteOnce resources: requests: storage: 5Gi In order to edit a PVC you can use kubectl edit or kubectl apply to make changes.\nStorageOS supports offline resize of volumes through editing a PVC storage request. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This requires that any pods using a volume must be scaled down for the resize to take place.\n N.B. Resizing a volume without updating the PVC directly will NOT result in the PVC being updated. The methods below are included for completeness, in Kubernetes environments editing the PVC is the preferred method for resizing a volume.\n To resize a volume using the StorageOS CLI use the volume update command\n$ storageos update volume size pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa 10GiB Name: pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa ID: 925e667f-91d3-465a-9391-8fdb56d0c9ff Size: 11 GB Description: AttachedOn: Replicas: 1x ready Labels: - csi.storage.k8s.io/pv/name pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa - csi.storage.k8s.io/pvc/name pvc-1 - csi.storage.k8s.io/pvc/namespace default - foo bar - pool default - storageos.com/replicas 1 Volume pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa (925e667f-91d3-465a-9391-8fdb56d0c9ff) updated. Size changed. To resize a volume using the StorageOS UI, navigate to the volumes section and click the edit pencil in order to update the volume config.\n","excerpt":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating …","ref":"/v2.2/docs/operations/resize/","title":"Volume Resize"},{"body":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard POSIX semantics. StorageOS presents volumes as mounts into containers via the Linux LIO subsystem.\nConceptually, StorageOS volumes have a frontend presentation, which is the what the application sees, and a backend presentation, which is the actual on-disk format. Depending on the configuration, frontend and backend components may be on the same or different hosts.\nVolumes are formatted using the linux standard ext4 filesystem by default. Kubernetes users may change the default filesystem type to ext2, ext3, ext4, or xfs by setting the fsType parameter in their StorageClass (See Supported Filesystems for more information). Different filesystems may be supported in the future.\nStorageOS volumes are represented on disk in two parts. Actual volume data is written to blob files in /var/lib/storageos/data/dev[\\d+]. Inside these directories, each StorageOS block device gets two blob files of the form vol.xxxxxx.y.blob, where x is the inode number for the device, and y is an index between 0 and 1. We provide two blob files in order to ensure that certain operations which require locking do not impede in-flight writes to the volume.\nIn systems which have multiple /var/lib/storageos/data/dev[\\d+] directories, two blob files are created per block device. This allows us to load-balance writes across multiple devices. In cases where dev directories are added after a period of run time, later directories are favoured for writes until the data is distributed evenly across the blob files.\nMetadata is kept in directories named /var/lib/storageos/data/db[\\d+]. We maintain an index of all blocks written to the blob file inside the metadata store, including checksums. These checksums allow us to detect bitrot, and return errors on reads, rather than serve bad data. In future versions we may implement recovery from replicas for volumes with one or more replicas defined.\nStorageOS metadata requires approximately 2.7GB of storage per 1TB of allocated blocks in the associated volume. This size is consistent irrespective of data compression defined on the volume.\nTo ensure deterministic performance, individual StorageOS volumes must fit on a single node.\nVolume Resize StorageOS v2.1 supports offline resize of volumes. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure that the volume is not in use.\nThis means that if a Kubernetes pod is currently consuming a volume that a resize request has been issued for, the resize will not be actioned until the pod is terminated and the volume is detached from the node. The StorageOS controlplane will then attach the volume to the node that holds the master deployment and resize the underlying block device and then run resize2fs to expand the filesystem.\nFor a walk through of how to resize a volume please see the Volume Resize operations page.\n","excerpt":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard …","ref":"/v2.2/docs/concepts/volumes/","title":"Volumes"},{"body":"Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.\nUsing StorageOS persistent volumes with Apache Zookeeper means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Zookeeper pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nAs Zookeeper has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Zookeeper to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Zookeeper on Kubernetes Pre-requisites StorageOS is assumed to have been installed; please check for the latest available version here. Helm To simplify the deployment of Zookeeper, we\u0026rsquo;ve used this Zookeeper helm chart (incubator) (version 1.2.2, app version 3.4.10) and rendered it into the example deployment files you can find in our GitHub repo.\nDeployment Clone the use cases repo You can find the latest files in the StorageOS use cases repository in /zookeeper/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion ---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:zookeeper...spec:replicas:3# \u0026lt;--- number of zookeeper pods...containers:- name:zookeeperimage:\u0026#34;gcr.io/google_samples/k8szk:v3\u0026#34;imagePullPolicy:IfNotPresent...volumeMounts:- name:datamountPath:/var/lib/zookeepervolumeClaimTemplates:- metadata:name:dataspec:accessModes:- \u0026#34;ReadWriteOnce\u0026#34;storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useresources:requests:storage:\u0026#34;5Gi\u0026#34;# \u0026lt;--- storage requested per podThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nCreate the kubernetes objects cd storageos-usecases kubectl apply -f ./zookeeper/ Confirm Zookeeper is up and running $ kubectl get pods NAME READY STATUS RESTARTS AGE zookeeper-0 1/1 Running 0 2m30s zookeeper-1 1/1 Running 0 112s zookeeper-2 1/1 Running 0 56s zookeeper-test-client 1/1 Running 0 2m30s Connect to Zookeeper Connect to the zookeeper client pod and list existing topics using the service endpoint\nkubectl exec -it zookeeper-test-client /bin/bash and issue a command to the zookeeper service\nzkCli.sh -server zookeeper ls /zookeeper ","excerpt":"Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration …","ref":"/v2.2/docs/usecases/zookeeper/","title":""}]
\ No newline at end of file
diff --git a/v2.2/search/index.html b/v2.2/search/index.html
index 88c0f21c8..715bf8a1a 100644
--- a/v2.2/search/index.html
+++ b/v2.2/search/index.html
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2.2/offline-search-index.61f55f7973f492ad99241908e74e577d311ae5e5629c59c75ff755bebc2d012281513ced6efef8e1df08879881e37adf4f57b6d2096132b9d57e275d01ef9bef.json"
+ data-offline-search-index-json-src="/v2.2/offline-search-index.da7c89a408dd99e39c5b17781ce8cda56984e4e50a798e14b6192906b82b149f95faa6513f6e8c746f7952831ea2494a4d0fd5ef62acd7b20792b1600010d0ff.json"
data-offline-search-base-href="/"
>
diff --git a/v2.2/yaml/bundle-configuration.yaml b/v2.2/yaml/bundle-configuration.yaml
index f1c1004a0..5b21e3ed3 100644
--- a/v2.2/yaml/bundle-configuration.yaml
+++ b/v2.2/yaml/bundle-configuration.yaml
@@ -40,11 +40,18 @@ spec:
# InternalAPIPort = 5710 # Grpc API
# GossipPort = 5711 # Gossip+Healthcheck
- echo \"Source node for the test:\";
+ date;
+ echo \"Source node:\";
hostname -f -I; echo;
+ echo \"TCP:\";
parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
- ::: 5703 5704 5705 5710 5711
+ ::: 5703 5704 5705 5710 5711;
+ echo;
+
+ echo \"UDP:\";
+ parallel -j2 nc -vunz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
+ ::: 5711;
"
timeout: 90s
- exec:
diff --git a/v2/404.html b/v2/404.html
index 5800afa20..b296dc797 100644
--- a/v2/404.html
+++ b/v2/404.html
@@ -181,7 +181,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/best-practices/index.html b/v2/docs/best-practices/index.html
index 1c391fff7..f5da75498 100644
--- a/v2/docs/best-practices/index.html
+++ b/v2/docs/best-practices/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/architecture/index.html b/v2/docs/concepts/architecture/index.html
index 17167d18d..69fa40a24 100644
--- a/v2/docs/concepts/architecture/index.html
+++ b/v2/docs/concepts/architecture/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/clusters/index.html b/v2/docs/concepts/clusters/index.html
index 33c4a40cb..af0c76531 100644
--- a/v2/docs/concepts/clusters/index.html
+++ b/v2/docs/concepts/clusters/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/compression/index.html b/v2/docs/concepts/compression/index.html
index ce541336c..2177b4c45 100644
--- a/v2/docs/concepts/compression/index.html
+++ b/v2/docs/concepts/compression/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/etcd/index.html b/v2/docs/concepts/etcd/index.html
index 2fabf64a4..0e917f628 100644
--- a/v2/docs/concepts/etcd/index.html
+++ b/v2/docs/concepts/etcd/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/index.html b/v2/docs/concepts/index.html
index a3695c4e8..8e294b905 100644
--- a/v2/docs/concepts/index.html
+++ b/v2/docs/concepts/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/namespaces/index.html b/v2/docs/concepts/namespaces/index.html
index e7ab98d26..0372426ca 100644
--- a/v2/docs/concepts/namespaces/index.html
+++ b/v2/docs/concepts/namespaces/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/nodes/index.html b/v2/docs/concepts/nodes/index.html
index 7b69e96c9..139d0e79d 100644
--- a/v2/docs/concepts/nodes/index.html
+++ b/v2/docs/concepts/nodes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/policies/index.html b/v2/docs/concepts/policies/index.html
index 9284f79e8..6c8d5e3f6 100644
--- a/v2/docs/concepts/policies/index.html
+++ b/v2/docs/concepts/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/replication/index.html b/v2/docs/concepts/replication/index.html
index 2022db97e..f84252a0d 100644
--- a/v2/docs/concepts/replication/index.html
+++ b/v2/docs/concepts/replication/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/concepts/volumes/index.html b/v2/docs/concepts/volumes/index.html
index c90f2b7b8..1359a9acd 100644
--- a/v2/docs/concepts/volumes/index.html
+++ b/v2/docs/concepts/volumes/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/index.html b/v2/docs/index.html
index 58c5761ef..23a3d7c85 100644
--- a/v2/docs/index.html
+++ b/v2/docs/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/index.xml b/v2/docs/index.xml
index 11ebb9884..b4b0ea1a5 100644
--- a/v2/docs/index.xml
+++ b/v2/docs/index.xml
@@ -1620,6 +1620,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -1652,7 +1657,7 @@ the init container, perform the following steps:</p>
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -1765,6 +1770,11 @@ the init container, perform the following steps:</p>
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2/docs/install/index.html b/v2/docs/install/index.html
index c9d5c3c8b..6eebf3685 100644
--- a/v2/docs/install/index.html
+++ b/v2/docs/install/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/install/kubernetes/index.html b/v2/docs/install/kubernetes/index.html
index cb08e618d..240ce55f6 100644
--- a/v2/docs/install/kubernetes/index.html
+++ b/v2/docs/install/kubernetes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/install/openshift/index.html b/v2/docs/install/openshift/index.html
index 42f2276e1..d5b950faa 100644
--- a/v2/docs/install/openshift/index.html
+++ b/v2/docs/install/openshift/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/install/rancher/index.html b/v2/docs/install/rancher/index.html
index 075fc939e..13957bda7 100644
--- a/v2/docs/install/rancher/index.html
+++ b/v2/docs/install/rancher/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/introduction/index.html b/v2/docs/introduction/index.html
index 87bcfafa2..d5d332995 100644
--- a/v2/docs/introduction/index.html
+++ b/v2/docs/introduction/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/introduction/overview/index.html b/v2/docs/introduction/overview/index.html
index 49001dae6..2fb040867 100644
--- a/v2/docs/introduction/overview/index.html
+++ b/v2/docs/introduction/overview/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/introduction/platforms/index.html b/v2/docs/introduction/platforms/index.html
index 580bd1946..226b9daaf 100644
--- a/v2/docs/introduction/platforms/index.html
+++ b/v2/docs/introduction/platforms/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/introduction/quickstart/index.html b/v2/docs/introduction/quickstart/index.html
index 5e422c0c4..54a42067c 100644
--- a/v2/docs/introduction/quickstart/index.html
+++ b/v2/docs/introduction/quickstart/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/external-etcd/etcd-tls/index.html b/v2/docs/operations/external-etcd/etcd-tls/index.html
index 42b1c3971..3a9371ffb 100644
--- a/v2/docs/operations/external-etcd/etcd-tls/index.html
+++ b/v2/docs/operations/external-etcd/etcd-tls/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/external-etcd/index.html b/v2/docs/operations/external-etcd/index.html
index 2d5d86bf7..147f6ae8a 100644
--- a/v2/docs/operations/external-etcd/index.html
+++ b/v2/docs/operations/external-etcd/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/firstpvc/index.html b/v2/docs/operations/firstpvc/index.html
index ca3b13a74..e7e924a1e 100644
--- a/v2/docs/operations/firstpvc/index.html
+++ b/v2/docs/operations/firstpvc/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/health/index.html b/v2/docs/operations/health/index.html
index c7a0613a0..a84a9616f 100644
--- a/v2/docs/operations/health/index.html
+++ b/v2/docs/operations/health/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/index.html b/v2/docs/operations/index.html
index 3326a6687..7fa11c286 100644
--- a/v2/docs/operations/index.html
+++ b/v2/docs/operations/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/licensing/index.html b/v2/docs/operations/licensing/index.html
index 19143cf7f..41e1db21e 100644
--- a/v2/docs/operations/licensing/index.html
+++ b/v2/docs/operations/licensing/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/managing-host-storage/index.html b/v2/docs/operations/managing-host-storage/index.html
index d6b0e65b0..70ba1eb62 100644
--- a/v2/docs/operations/managing-host-storage/index.html
+++ b/v2/docs/operations/managing-host-storage/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/namespaces/index.html b/v2/docs/operations/namespaces/index.html
index 1167666b1..142e34465 100644
--- a/v2/docs/operations/namespaces/index.html
+++ b/v2/docs/operations/namespaces/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/policies/index.html b/v2/docs/operations/policies/index.html
index 8d5fb2e06..bbcde71b8 100644
--- a/v2/docs/operations/policies/index.html
+++ b/v2/docs/operations/policies/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/resize/index.html b/v2/docs/operations/resize/index.html
index d2d38af33..e950c4589 100644
--- a/v2/docs/operations/resize/index.html
+++ b/v2/docs/operations/resize/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/storageclasses/index.html b/v2/docs/operations/storageclasses/index.html
index b177d87b7..ffe4c7c9f 100644
--- a/v2/docs/operations/storageclasses/index.html
+++ b/v2/docs/operations/storageclasses/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/troubleshooting/index.html b/v2/docs/operations/troubleshooting/index.html
index 9f1986f6a..5eeffba61 100644
--- a/v2/docs/operations/troubleshooting/index.html
+++ b/v2/docs/operations/troubleshooting/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/operations/users/index.html b/v2/docs/operations/users/index.html
index 3ba0eeae1..5359f9fe8 100644
--- a/v2/docs/operations/users/index.html
+++ b/v2/docs/operations/users/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/platforms/aws-eks/index.html b/v2/docs/platforms/aws-eks/index.html
index 60673b0f8..7a37633a8 100644
--- a/v2/docs/platforms/aws-eks/index.html
+++ b/v2/docs/platforms/aws-eks/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/platforms/azure-aks/index.html b/v2/docs/platforms/azure-aks/index.html
index 167948dd7..ed4bde0d2 100644
--- a/v2/docs/platforms/azure-aks/index.html
+++ b/v2/docs/platforms/azure-aks/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/platforms/dockeree/index.html b/v2/docs/platforms/dockeree/index.html
index e52670957..d628df7b8 100644
--- a/v2/docs/platforms/dockeree/index.html
+++ b/v2/docs/platforms/dockeree/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/platforms/index.html b/v2/docs/platforms/index.html
index ca6e54447..4a8d78bf5 100644
--- a/v2/docs/platforms/index.html
+++ b/v2/docs/platforms/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/platforms/openshift/index.html b/v2/docs/platforms/openshift/index.html
index 5b77b4317..c9332f483 100644
--- a/v2/docs/platforms/openshift/index.html
+++ b/v2/docs/platforms/openshift/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/platforms/rancher/index.html b/v2/docs/platforms/rancher/index.html
index d6a429540..ef0593797 100644
--- a/v2/docs/platforms/rancher/index.html
+++ b/v2/docs/platforms/rancher/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/prerequisites/etcd/index.html b/v2/docs/prerequisites/etcd/index.html
index 5af930cbe..eb5220102 100644
--- a/v2/docs/prerequisites/etcd/index.html
+++ b/v2/docs/prerequisites/etcd/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/prerequisites/firewalls/index.html b/v2/docs/prerequisites/firewalls/index.html
index 8b2c1d342..9099d0808 100644
--- a/v2/docs/prerequisites/firewalls/index.html
+++ b/v2/docs/prerequisites/firewalls/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/prerequisites/index.html b/v2/docs/prerequisites/index.html
index d8f93bf9a..1f0597f3e 100644
--- a/v2/docs/prerequisites/index.html
+++ b/v2/docs/prerequisites/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/prerequisites/mountpropagation/index.html b/v2/docs/prerequisites/mountpropagation/index.html
index 66ab5590e..1f8ddc016 100644
--- a/v2/docs/prerequisites/mountpropagation/index.html
+++ b/v2/docs/prerequisites/mountpropagation/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/prerequisites/systemconfiguration/index.html b/v2/docs/prerequisites/systemconfiguration/index.html
index 125928c93..9a20a3467 100644
--- a/v2/docs/prerequisites/systemconfiguration/index.html
+++ b/v2/docs/prerequisites/systemconfiguration/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/api/index.html b/v2/docs/reference/api/index.html
index 6168e659c..be490435b 100644
--- a/v2/docs/reference/api/index.html
+++ b/v2/docs/reference/api/index.html
@@ -189,7 +189,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/bundle/index.html b/v2/docs/reference/bundle/index.html
index b81d5cc3f..1bd65b852 100644
--- a/v2/docs/reference/bundle/index.html
+++ b/v2/docs/reference/bundle/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/apply/index.html b/v2/docs/reference/cli/apply/index.html
index 356f3ae50..e6b0b7975 100644
--- a/v2/docs/reference/cli/apply/index.html
+++ b/v2/docs/reference/cli/apply/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/attach/index.html b/v2/docs/reference/cli/attach/index.html
index 96e0613dd..1354be9b4 100644
--- a/v2/docs/reference/cli/attach/index.html
+++ b/v2/docs/reference/cli/attach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/create/index.html b/v2/docs/reference/cli/create/index.html
index c013a511a..ddc85d257 100644
--- a/v2/docs/reference/cli/create/index.html
+++ b/v2/docs/reference/cli/create/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/delete/index.html b/v2/docs/reference/cli/delete/index.html
index ce4affe4c..43a1f949e 100644
--- a/v2/docs/reference/cli/delete/index.html
+++ b/v2/docs/reference/cli/delete/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/describe/index.html b/v2/docs/reference/cli/describe/index.html
index 1046a2038..356fdf5a6 100644
--- a/v2/docs/reference/cli/describe/index.html
+++ b/v2/docs/reference/cli/describe/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/detach/index.html b/v2/docs/reference/cli/detach/index.html
index 3bfb61522..f30d45f7f 100644
--- a/v2/docs/reference/cli/detach/index.html
+++ b/v2/docs/reference/cli/detach/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/get/index.html b/v2/docs/reference/cli/get/index.html
index deb7d0db6..fe3c0621d 100644
--- a/v2/docs/reference/cli/get/index.html
+++ b/v2/docs/reference/cli/get/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/help/index.html b/v2/docs/reference/cli/help/index.html
index 7c934784b..d59b8088e 100644
--- a/v2/docs/reference/cli/help/index.html
+++ b/v2/docs/reference/cli/help/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/index.html b/v2/docs/reference/cli/index.html
index 7c5c10d46..18ae7ceb4 100644
--- a/v2/docs/reference/cli/index.html
+++ b/v2/docs/reference/cli/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cli/version/index.html b/v2/docs/reference/cli/version/index.html
index 1d35b7dac..dd91b99ec 100644
--- a/v2/docs/reference/cli/version/index.html
+++ b/v2/docs/reference/cli/version/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cluster-operator/configuration/index.html b/v2/docs/reference/cluster-operator/configuration/index.html
index 64aa10373..5129d4361 100644
--- a/v2/docs/reference/cluster-operator/configuration/index.html
+++ b/v2/docs/reference/cluster-operator/configuration/index.html
@@ -30,7 +30,7 @@
-
+
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -1115,6 +1115,11 @@ StorageOSCluster Resource Confi
false |
+images.apiManagerContainer |
+StorageOS API Manager container image |
+storageos/api-manager:v1.0.0 |
+
+
images.csiClusterDriverRegistrarContainer |
CSI Cluster Driver Registrar Container image |
quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 |
@@ -1147,7 +1152,7 @@ StorageOSCluster Resource Confi
images.initContainer |
StorageOS init container image |
-storageos/init:1.0.0 |
+storageos/init:2.1.0 |
images.kubeSchedulerContainer |
@@ -1260,6 +1265,11 @@ StorageOSCluster Resource Confi
/var/lib/kubelet/plugins/kubernetes.io~storageos |
+storageClassName |
+The name of the default StorageClass created for StorageOS volumes |
+fast |
+
+
tlsEtcdSecretRefName |
Secret containing etcd client certificates |
|
diff --git a/v2/docs/reference/cluster-operator/examples/index.html b/v2/docs/reference/cluster-operator/examples/index.html
index 59a4e9fba..9d458baf4 100644
--- a/v2/docs/reference/cluster-operator/examples/index.html
+++ b/v2/docs/reference/cluster-operator/examples/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cluster-operator/index.html b/v2/docs/reference/cluster-operator/index.html
index 9095a74ca..0581d7466 100644
--- a/v2/docs/reference/cluster-operator/index.html
+++ b/v2/docs/reference/cluster-operator/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cluster-operator/index.xml b/v2/docs/reference/cluster-operator/index.xml
index 4b9eb8359..fbf7c4185 100644
--- a/v2/docs/reference/cluster-operator/index.xml
+++ b/v2/docs/reference/cluster-operator/index.xml
@@ -108,6 +108,11 @@
<td align="left"><code>false</code></td>
</tr>
<tr>
+<td align="left"><code>images.apiManagerContainer</code></td>
+<td align="left">StorageOS API Manager container image</td>
+<td align="left"><code>storageos/api-manager:v1.0.0</code></td>
+</tr>
+<tr>
<td align="left"><code>images.csiClusterDriverRegistrarContainer</code></td>
<td align="left">CSI Cluster Driver Registrar Container image</td>
<td align="left"><code>quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1</code></td>
@@ -140,7 +145,7 @@
<tr>
<td align="left"><code>images.initContainer</code></td>
<td align="left">StorageOS init container image</td>
-<td align="left"><code>storageos/init:1.0.0</code></td>
+<td align="left"><code>storageos/init:2.1.0</code></td>
</tr>
<tr>
<td align="left"><code>images.kubeSchedulerContainer</code></td>
@@ -253,6 +258,11 @@
<td align="left"><code>/var/lib/kubelet/plugins/kubernetes.io~storageos</code></td>
</tr>
<tr>
+<td align="left"><code>storageClassName</code></td>
+<td align="left">The name of the default StorageClass created for StorageOS volumes</td>
+<td align="left"><code>fast</code></td>
+</tr>
+<tr>
<td align="left"><code>tlsEtcdSecretRefName</code></td>
<td align="left">Secret containing etcd client certificates</td>
<td></td>
diff --git a/v2/docs/reference/cluster-operator/install/index.html b/v2/docs/reference/cluster-operator/install/index.html
index be8ac51f0..964d349ed 100644
--- a/v2/docs/reference/cluster-operator/install/index.html
+++ b/v2/docs/reference/cluster-operator/install/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/cluster-operator/upgrade/index.html b/v2/docs/reference/cluster-operator/upgrade/index.html
index 3e400fe78..d14347ea3 100644
--- a/v2/docs/reference/cluster-operator/upgrade/index.html
+++ b/v2/docs/reference/cluster-operator/upgrade/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/contributing/index.html b/v2/docs/reference/contributing/index.html
index a0be002ba..e0e440c58 100644
--- a/v2/docs/reference/contributing/index.html
+++ b/v2/docs/reference/contributing/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/filesystems/index.html b/v2/docs/reference/filesystems/index.html
index 53895a5c6..bfdce2c9b 100644
--- a/v2/docs/reference/filesystems/index.html
+++ b/v2/docs/reference/filesystems/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/gui/index.html b/v2/docs/reference/gui/index.html
index 4a3ab2f9a..546b77ae8 100644
--- a/v2/docs/reference/gui/index.html
+++ b/v2/docs/reference/gui/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/index.html b/v2/docs/reference/index.html
index cb618f974..eab0bda10 100644
--- a/v2/docs/reference/index.html
+++ b/v2/docs/reference/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/init-container/index.html b/v2/docs/reference/init-container/index.html
index 67f0c064c..4bc0dbf99 100644
--- a/v2/docs/reference/init-container/index.html
+++ b/v2/docs/reference/init-container/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/labels/index.html b/v2/docs/reference/labels/index.html
index 15f7b9c8a..a4e633c8b 100644
--- a/v2/docs/reference/labels/index.html
+++ b/v2/docs/reference/labels/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/licence/index.html b/v2/docs/reference/licence/index.html
index 2667a3b05..be40fdafa 100644
--- a/v2/docs/reference/licence/index.html
+++ b/v2/docs/reference/licence/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/open_source_attribution/index.html b/v2/docs/reference/open_source_attribution/index.html
index f38c2af54..c3951eb11 100644
--- a/v2/docs/reference/open_source_attribution/index.html
+++ b/v2/docs/reference/open_source_attribution/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/release_notes/index.html b/v2/docs/reference/release_notes/index.html
index 361dce249..c0a6e8506 100644
--- a/v2/docs/reference/release_notes/index.html
+++ b/v2/docs/reference/release_notes/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/telemetry/index.html b/v2/docs/reference/telemetry/index.html
index aa7f53fc4..ee6ebc628 100644
--- a/v2/docs/reference/telemetry/index.html
+++ b/v2/docs/reference/telemetry/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/reference/tolerations/index.html b/v2/docs/reference/tolerations/index.html
index 1c425586b..85f44213a 100644
--- a/v2/docs/reference/tolerations/index.html
+++ b/v2/docs/reference/tolerations/index.html
@@ -188,7 +188,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -222,7 +222,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/self-eval/index.html b/v2/docs/self-eval/index.html
index 5672f310c..9493fb705 100644
--- a/v2/docs/self-eval/index.html
+++ b/v2/docs/self-eval/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/support/index.html b/v2/docs/support/index.html
index 4b980c71b..5ff923ae8 100644
--- a/v2/docs/support/index.html
+++ b/v2/docs/support/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/backups/index.html b/v2/docs/usecases/backups/index.html
index 7630658bc..4ba25e625 100644
--- a/v2/docs/usecases/backups/index.html
+++ b/v2/docs/usecases/backups/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/cassandra/index.html b/v2/docs/usecases/cassandra/index.html
index 6c97026a4..5325b4020 100644
--- a/v2/docs/usecases/cassandra/index.html
+++ b/v2/docs/usecases/cassandra/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/elasticsearch/index.html b/v2/docs/usecases/elasticsearch/index.html
index 462af9d2a..887d7765b 100644
--- a/v2/docs/usecases/elasticsearch/index.html
+++ b/v2/docs/usecases/elasticsearch/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/index.html b/v2/docs/usecases/index.html
index ae194d26f..b455275ff 100644
--- a/v2/docs/usecases/index.html
+++ b/v2/docs/usecases/index.html
@@ -186,7 +186,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -220,7 +220,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/jenkins/index.html b/v2/docs/usecases/jenkins/index.html
index 24e9e5086..3eea5b374 100644
--- a/v2/docs/usecases/jenkins/index.html
+++ b/v2/docs/usecases/jenkins/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/kafka/index.html b/v2/docs/usecases/kafka/index.html
index f1e1bd0d4..085757c45 100644
--- a/v2/docs/usecases/kafka/index.html
+++ b/v2/docs/usecases/kafka/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/kubevirt/index.html b/v2/docs/usecases/kubevirt/index.html
index 1bdb9e428..2743f25cf 100644
--- a/v2/docs/usecases/kubevirt/index.html
+++ b/v2/docs/usecases/kubevirt/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/mssql/index.html b/v2/docs/usecases/mssql/index.html
index 6fe3bc0e7..8287808e5 100644
--- a/v2/docs/usecases/mssql/index.html
+++ b/v2/docs/usecases/mssql/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/mysql/index.html b/v2/docs/usecases/mysql/index.html
index c8fae2198..e96760084 100644
--- a/v2/docs/usecases/mysql/index.html
+++ b/v2/docs/usecases/mysql/index.html
@@ -197,7 +197,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -231,7 +231,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/nginx/index.html b/v2/docs/usecases/nginx/index.html
index 1d2872e5b..648e56800 100644
--- a/v2/docs/usecases/nginx/index.html
+++ b/v2/docs/usecases/nginx/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/postgres/index.html b/v2/docs/usecases/postgres/index.html
index 99cdd933c..b2a05c96d 100644
--- a/v2/docs/usecases/postgres/index.html
+++ b/v2/docs/usecases/postgres/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/prometheus/index.html b/v2/docs/usecases/prometheus/index.html
index 89e9fa243..049fe04f4 100644
--- a/v2/docs/usecases/prometheus/index.html
+++ b/v2/docs/usecases/prometheus/index.html
@@ -194,7 +194,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -228,7 +228,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/redis/index.html b/v2/docs/usecases/redis/index.html
index 365d5921c..51d74d4ed 100644
--- a/v2/docs/usecases/redis/index.html
+++ b/v2/docs/usecases/redis/index.html
@@ -200,7 +200,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -234,7 +234,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/docs/usecases/zookeeper/index.html b/v2/docs/usecases/zookeeper/index.html
index 3f69d48ee..27f9200b0 100644
--- a/v2/docs/usecases/zookeeper/index.html
+++ b/v2/docs/usecases/zookeeper/index.html
@@ -191,7 +191,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -225,7 +225,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/index.html b/v2/index.html
index 0226358f5..1a0f81c64 100644
--- a/v2/index.html
+++ b/v2/index.html
@@ -183,7 +183,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
@@ -217,7 +217,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/index.json b/v2/index.json
index 609fd9831..354b06c08 100644
--- a/v2/index.json
+++ b/v2/index.json
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json b/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json
new file mode 100644
index 000000000..541add355
--- /dev/null
+++ b/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json
@@ -0,0 +1 @@
+[{"body":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher OpenShift ","excerpt":"To install the operator follow the installation page for your orchestrator.\n Kubernetes Rancher …","ref":"/v2/docs/reference/cluster-operator/install/","title":"Install"},{"body":"Current Version The current StorageOS version is v2.0.0\nSections Our documentation is arranged into sections, accessible from the navigation bar on the left.\nIntroduction - Quickstart and support information.\nConcepts - Architectural and deep technical information.\nPrerequisites - We require certain prerequisites to be met for the product to function correctly. Please read these carefully and ensure they are implemented.\nPlatforms - Due to differences in the various orchestrators that StorageOS can run under, we list install guides and other platform specific operations here.\nOperations - Platform agnostic operations.\nUse Cases - A set of examples to get up and running with StorageOS quickly.\nReference - Information on our GUI, CLI, and other important information.\n","excerpt":"Current Version The current StorageOS version is v2.0.0\nSections Our documentation is arranged into …","ref":"/v2/docs/introduction/","title":"Introduction"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed service such as EKS, AKS, GKE, DO or DockerEE platform can use the following Kubernetes guide to install StorageOS.\n StorageOS v2 uses CSI only. The native driver has been deprecated.\n Make sure to add a StorageOS licence after installing.\n 1.17 1.16 1.15 1.14 1.13 Install StorageOS on Kubernetes 1.17 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.16 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.15 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.14 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n Install StorageOS on Kubernetes 1.13 The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation upstream eks aks gke This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;upstream\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;eks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;aks\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;gke\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Any Kubernetes managed …","ref":"/v2/docs/install/kubernetes/","title":"Kubernetes"},{"body":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some significant enhancements over our v1 product. We\u0026rsquo;ve built V2 based on our observations of trends in the industry, as well as our own experience.\nMany of our customers want to run big clusters - in the tens or hundreds of nodes. In these sorts of big environments, the challenges multiply. Not only do we need to scale well, but we also need to be more failure tolerant. Bigger environments typically suffer higher failure rates (more nodes = greater chance of something failing), but are also subject to all sorts of transient conditions such as network partitions.\nThe second trend we\u0026rsquo;ve seen become increasingly common is the desire to run multiple clusters, and consume storage between them in some way - sometimes to implement novel topologies such as a centralised storage cluster with satellites consuming the storage, and sometimes to replicate data between those clusters for HA or DR purposes.\nWe\u0026rsquo;ve built V2 with these architectures and design patterns in mind. Not only does it scale well, but it contains the foundations we need to implement a rich set of multi-cluster functionality.\nUpgraded Control Plane At the heart of the V2 release is an upgraded control plane. We\u0026rsquo;ve changed a lot here. Firstly, our usage of etcd is vastly improved. We\u0026rsquo;ve learnt a lot about the subtleties of distributed consensus in the last year, particularly in noisy or unpredictable environments. Not only is StorageOS V2 much lighter on your etcd cluster, but it\u0026rsquo;s a lot more tolerant of transient failure conditions that are often found in cloud environments, or clusters under heavy load.\nWe spent some time describing and testing our internal state machine using the TLA+ formal verification language. This allows us to have a much higher degree of confidence that our algorithms will behave correctly, particularly under hard-to-test edge cases and failure conditions.\nAdditionally, we\u0026rsquo;ve changed the way volumes behave with respect to centralised scheduling. Each volume group (consisting of a master and 0 or more replicas) now behaves as a strongly consistent unit allowing it to take action independent of the activities of the rest of the cluster. Other state can be distributed around the cluster using eventually consistent mechanisms. This approach inherently scales better and allows StorageOS V2 to effectively manage many more nodes and volumes than before.\nWe\u0026rsquo;ve implemented TLS on all endpoints. Not only does this give you encrypted traffic between nodes in your storage cluster, it also protects all endpoints with strong, public key based authentication. Today\u0026rsquo;s IT environments can\u0026rsquo;t rely on firewalls to keep bad actors out - they must implement security at all layers within the stack - defense in depth. While we recognise that this brings a welcome relief to many security conscious administrators, we also know that managing certificate authorities (CAs) can be an unwelcome source of complexity. For this reason, StorageOS V2 implements an internal CA by default, to manage this complexity for you. If you\u0026rsquo;d prefer to integrate your own CA, we support that too - it\u0026rsquo;s up to you.\nFinally - our logging has undergone a complete transformation in this edition. We know that systems engineers and operators don\u0026rsquo;t just value headline features, but that observability and diagnostics are equally important. All logs are now decorated with rich context to help you understand what is happening within your cluster, and we\u0026rsquo;ll output in json by default, for easy ingestion into log aggregators such as Elasticsearch.\nUpgraded Data Plane Not to be outdone, our data plane contains some significant improvements.\nFirstly, we\u0026rsquo;ve completely re-written our sync algorithm (see Delta Sync, used when seeding or catching up replicas that have been offline or partitioned. Our new algorithm uses a Hash List to sync only changed sections of a volume (similar in some ways to what rsync does). StorageOS maintains these hashes during normal operation, meaning that when resyncing a failed replica, for example after a node reboot, we can very quickly and efficiently catch this replica up, rather than needing to promote and build a new one from scratch. This improves resiliency within your cluster, and prevents using excessive network bandwidth during failover conditions - at a time when it might be needed the most.\nSecondly, a new threading model, with dynamic pool sizing, means that StorageOS is faster, a lot faster. In our tests we observed improvements across the board, with improvements in throughput of up to 135% for some scenarios.\n","excerpt":"Over the past several months, we\u0026rsquo;ve been hard at work on StorageOS V2, which contains some …","ref":"/v2/docs/introduction/overview/","title":"What is StorageOS V2"},{"body":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source implementation of the SCSI target.\nWe require the following modules to be loaded:\n target_core_mod tcm_loop target_core_file configfs target_core_user uio N.B. Other applications utilising TCMU cannot be run concurrently with StorageOS. Doing so may result in corruption of data. On startup, StorageOS will detect if other applications are using TCMU and fall back to FUSE. However if StorageOS is started first there is no mechanism for StorageOS to fallback to FUSE if another application begins to use TCMU. TCMU can be disabled using the DISABLE_TCMU StorageOSCluster spec parameter.\n Depending on the distribution, the modules are shipped as part of the base kernel package or as part of a kernel extras package which needs to be installed.\nDistribution Specifics The following distributions are supported by default:\n RHEL 7.5 CentOS 7 Debian 9 Ubuntu Azure RancherOS - Note CSI is not supported on RancherOS Ubuntu 16.04/18.04 requires the installation of additional packages.\n N.B. Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not provide the necessary linux-image-extra package - see below for more information\n Ubuntu Package Installation Ubuntu 16.04/18.04 Generic and Ubuntu 16.04 GCE require extra packages:\nUbuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Ubuntu With AWS Or GCE Kernels Ubuntu 16.04/18.04 AWS and Ubuntu 18.04 GCE do not yet provide the linux-image-extra package. As such you should either use Debian, CentOS or RHEL, or install the non-cloud-provider optimised Ubuntu kernel.\nInstalling the non-cloud-provider optimised Ubuntu kernel is something that should only be done with full understanding of potential ramifications.\nsudo apt -y update sudo apt install -y linux-virtual linux-image-extra-virtual sudo apt purge -y linux*aws # Reboot the machine sudo shutdown -r now Automatic Configuration Once required kernel modules are installed on the system, for convenience we provide a container which will ensure the appropriate modules are loaded and ready for use at runtime. On Docker installations, you will need to run the init container prior to starting StorageOS. Our installation guides for Kubernetes and OpenShift include this step.\n# Load the required kernel modules. The Kubernetes and OpenShift installations include this step. docker run --name enable_lio \\ --privileged \\ --rm \\ --cap-add=SYS_ADMIN \\ -v /lib/modules:/lib/modules \\ -v /sys:/sys:rshared \\ storageos/init:0.2 Manual Configuration For those wishing to manage their own kernel configuration, rather than using the init container, perform the following steps:\n Ensure kernel modules are all loaded per list above Ensure configfs is loaded and mounted at /sys/kernel/config ","excerpt":"StorageOS requires certain kernel modules to function, in particular Linux-IO , an open-source …","ref":"/v2/docs/prerequisites/systemconfiguration/","title":"System Configuration"},{"body":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters of the StorageOSCluster custom resource and their default values.\n Parameter Description Default csi.deploymentStrategy CSI helper deployment strategy (statefulset or deployment) statefulset csi.enable Enable CSI setup false csi.enableControllerPublishCreds Enable CSI controller publish credentials false csi.enableNodePublishCreds Enable CSI node publish credentials false csi.enableProvisionCreds Enable CSI provision credentials false debug Enable debug mode for all the cluster nodes false disableFencing Disable Pod fencing false disableScheduler Disable StorageOS scheduler false disableTCMU Disable TCMU to allow co-existence with other TCMU users. Disabling TCMU degrades performance false disableTelemetry Disable telemetry reports false forceTCMU Forces TCMU to be enabled or causes StorageOS to abort startup false images.apiManagerContainer StorageOS API Manager container image storageos/api-manager:v1.0.0 images.csiClusterDriverRegistrarContainer CSI Cluster Driver Registrar Container image quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 images.csiExternalAttacherContainer CSI External Attacher Container image quay.io/k8scsi/csi-attacher:v1.0.1 images.csiExternalProvisionerContainer CSI External Provisioner Container image storageos/csi-provisioner:v1.0.1 ìmages.csiLivenessProbeContainer CSI Liveness Probe Container Image quay.io/k8scsi/livenessprobe:v1.0.1 images.csiNodeDriverRegistrarContainer CSI Node Driver Registrar Container image quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 images.hyperkubeContainer Deprecated field - HyperKube Container image Default dependent on Scheduler version images.initContainer StorageOS init container image storageos/init:2.1.0 images.kubeSchedulerContainer Kube scheduler container image Default dependent on Scheduler version images.nfsContainer StorageOS nfs container image storageos/nfs:1.0.0 images.nodeContainer StorageOS node container image storageos/node:v2.0.0 ingress.annotations Annotations of the ingress used by the cluster ingress.enable Enable ingress for the cluster false ingress.hostname Hostname to be used in cluster ingress storageos.local ingress.tls Enable TLS for the ingress false k8sDistro The name of the Kubernetes distribution is use, e.g. rancher or eks kvBackend.address Comma-separated list of addresses of external key-value store. (1.2.3.4:2379,2.3.4.5:2379) kvBackend.backend (v2 deprecated) Name of the key-value store to use. Set to etcd for external key-value store. embedded namespace Namespace where storageos cluster resources are created kube-system nodeSelectorTerms Set node selector for storageos pod placement pause Pause the operator for cluster maintenance false resources Set resource requirements for the containers secretRefName Reference name of storageos secret secretRefNamespace Namespace of storageos secret service.annotations Annotations of the Service used by the cluster service.externalPort External port of the Service used by the cluster 5705 service.internalPort Internal port of the Service used by the cluster 5705 service.name Name of the Service used by the cluster storageos service.type Type of the Service used by the cluster ClusterIP sharedDir Path to be shared with kubelet container when deployed as a pod /var/lib/kubelet/plugins/kubernetes.io~storageos storageClassName The name of the default StorageClass created for StorageOS volumes fast tlsEtcdSecretRefName Secret containing etcd client certificates tlsEtcdSecretRefNamespace Namespace of the tlsEtcdSecretRefName tolerations Set pod tolerations for storageos pod placement ","excerpt":"StorageOSCluster Resource Configuration The following table lists the configurable spec parameters …","ref":"/v2/docs/reference/cluster-operator/configuration/","title":"Configuration"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment of an etcd cluster.\n If you have installed OpenShift in AWS ensure that the requisite ports are opened for the worker nodes\u0026rsquo; security group.\n Make sure to add a StorageOS licence after installing.\n StorageOS v2 supports OpenShift v4. For more information check the OpenShift platform page.\nThe recommended way to run StorageOS on OpenShift is to deploy the StorageOS Cluster Operator using the Operator Lifecycle Manager (OLM) from the Red Hat OperatorHub, and bootstrap StorageOS using a Custom Resource. If that option is not suited for your use case, you can choose the \u0026ldquo;Manual\u0026rdquo; installation.\n Manual install The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\noc create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# oc -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;openshift\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# oc -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\nFirst StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding. Including the deployment …","ref":"/v2/docs/install/openshift/","title":"OpenShift"},{"body":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all nodes in the cluster:\n Port Number TCP/UDP Use 5701 TCP gRPC 5703 TCP DirectFS 5704 TCP Dataplane Supervisor 5705 TCP REST API 5711 TCP \u0026amp; UDP Gossip service StorageOS also uses ephemeral ports to dial-out to these ports on other StorageOS nodes. For this reason, outgoing traffic should be enabled.\nFirewalls and VPS providers Some VPS providers (such as Digital Ocean) ship default firewall rulesets which must be updated to allow StorageOS to run. Some example rules are shown below - modify to taste.\nUFW For distributions using UFW, such as RHEL and derivatives:\nufw default allow outgoing ufw allow 5701:5711/tcp ufw allow 5711/udp Firewalld For distributions that enable firewalld to control iptables such as some installations of OpenShift.\nfirewall-cmd --permanent --new-service=storageos firewall-cmd --permanent --service=storageos --add-port=5700-5800/tcp firewall-cmd --add-service=storageos --zone=public --permanent firewall-cmd --reload Iptables For those using plain iptables:\n# Inbound traffic iptables -I INPUT -i lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I INPUT -m state --state ESTABLISHED,RELATED -m comment --comment \u0026#39;Permit established traffic\u0026#39; -j ACCEPT iptables -I INPUT -p tcp --dport 5701:5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT iptables -I INPUT -p udp --dport 5711 -m comment --comment \u0026#39;StorageOS\u0026#39; -j ACCEPT # Outbound traffic iptables -I OUTPUT -o lo -m comment --comment \u0026#39;Permit loopback traffic\u0026#39; -j ACCEPT iptables -I OUTPUT -d 0.0.0.0/0 -m comment --comment \u0026#39;Permit outbound traffic\u0026#39; -j ACCEPT Please ensure that the iptables rules you have added above come before any default DROP or REJECT rules.\n","excerpt":"Port list StorageOS daemons listen on specific ports, which we require to be accessible between all …","ref":"/v2/docs/prerequisites/firewalls/","title":"Firewalls"},{"body":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, OpenShift or Rancher.\n Dive into our Use Case documentation for Workloads deployed in Kubernetes.\n ","excerpt":" Start by understanding the architecture.\n Follow our installation guides for Kubernetes, …","ref":"/v2/docs/introduction/quickstart/","title":"Quickstart"},{"body":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and Password in base64 encoding.\nkubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;default\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z END This example contains a default password, for production installations, use a unique, strong password.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Create a cluster-config.yaml according to your needs from the examples below.\nkubectl create -f cluster-config.yaml Note that StorageOS will be deployed in spec.namespace (storageos by default), irrespective of what NameSpace the CR is defined in.\n Examples You can checkout all the parameters configurable in the configuration page.\n All examples must reference the storageos-api Secret.\nspec:secretRefName:\u0026#34;storageos-api\u0026#34;# Reference to the Secret created in the previous stepsecretRefNamespace:\u0026#34;storageos-operator\u0026#34;# Namespace of the SecretCheck out Cluster Definition examples for full CR files.\nInstalling with an external etcd spec:kvBackend:address:\u0026#39;10.43.93.95:2379\u0026#39;# IP of the SVC that exposes ETCD# address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can specify individual IPs of the etcd serversInstalling to a subset of nodes In this case we select nodes that are workers. To make sure that StorageOS doesn\u0026rsquo;t start in Master nodes.\nYou can see the labels in the nodes by kubectl get node --show-labels.\nspec:nodeSelectorTerms:- matchExpressions:- key:\u0026#34;node-role.kubernetes.io/worker\u0026#34;operator:Invalues:- \u0026#34;true\u0026#34;# OpenShift uses \u0026#34;node-role.kubernetes.io/compute=true\u0026#34;# Rancher uses \u0026#34;node-role.kubernetes.io/worker=true\u0026#34;# Kops uses \u0026#34;node-role.kubernetes.io/node=\u0026#34; Different provisioners and Kubernetes distributions use node labels differently to specify master vs workers. Node Taints are not enough to make sure StorageOS doesn\u0026rsquo;t start in a node. The JOIN variable is defined by the operator by selecting all the nodes that match the nodeSelectorTerms.\n Enabling CSI spec:csi:enable:truedeploymentStrategy:deploymentenableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:trueThe credentials must be defined in the storageos-api Secret\napiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# Add base64 encoded creds below for CSI credentials.csiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zSpecifying a shared directory for use with kubelet as a container spec:sharedDir:\u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39;Defining pod resource requests and reservations spec:resources:requests:memory:\u0026#34;512Mi\u0026#34;# cpu: \u0026#34;1\u0026#34;# limits:# memory: \u0026#34;4Gi\u0026#34;Limiting StorageOS can cause malfunction for IO to StorageOS volumes, therefore we do not currently recommend applying upper limits to resources for StorageOS pods.\nSpecifying custom Tolerations spec:tolerations:- key:\u0026#34;key1\u0026#34;operator:\u0026#34;Equal\u0026#34;value:\u0026#34;value1\u0026#34;effect:\u0026#34;EffectToTolerate\u0026#34;- key:\u0026#34;key2\u0026#34;operator:\u0026#34;Exists\u0026#34;Custom tolerations specified in the StorageOSCluster definition are added to all StorageOS components; the StorageOS daemonset, CSI helper and scheduler.\nIn the above example a toleration key1=value1:EffectToTolerate would be tolerated and key2 would be tolerated regardless of the value and effect. For more information about tolerations please see the Kubernetes documentation.\n","excerpt":"Before deploying a StorageOS cluster, create a Secret to define the StorageOS API Username and …","ref":"/v2/docs/reference/cluster-operator/examples/","title":"Cluster Operator examples"},{"body":" Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by default.\n StorageOS requires mount propagation enabled to present devices as volumes for containers (see linux kernel documentation here).\nCertain versions of docker ship with a systemd manifest with MountFlags set to \u0026lsquo;slave\u0026rsquo;, thus preventing StorageOS from working. This can be removed or set to \u0026lsquo;shared\u0026rsquo; with a systemd drop in:\nmkdir -p /etc/systemd/system/docker.service.d/ cat \u0026lt;\u0026lt;EOF \u0026gt; /etc/systemd/system/docker.service.d/mount_propagation_flags.conf [Service] MountFlags=shared EOF # systemctl daemon-reload # systemctl restart docker.service To confirm behaviour, the following command should run without error.\ndocker run -it --rm -v /mnt:/mnt:shared busybox sh -c /bin/date Orchestrators such as Kubernetes or OpenShift have their own ways of exposing this setting. Kubernetes 1.10 and OpenShift 3.10 have mount propagation enabled by default. Previous versions require that feature gates are enabled on the Kubernetes master\u0026rsquo;s controller-manager and apiserver services and in the kubelet service on each node.\nInstallations of orchestrators using Docker require that mount propagation is enabled for both.\nRefer to our installation pages for the orchestrators to see details on how to check and enable mount propagation where appropriate.\n","excerpt":"Modern versions of Kubernetes, Docker or other Container Runtimes enable mount propagation by …","ref":"/v2/docs/prerequisites/mountpropagation/","title":"Mount Propagation"},{"body":" Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Make sure to add a StorageOS licence after installing.\n StorageOS transparently supports Rancher deployments on CentOS, RHEL, Debian, Ubuntu or RancherOS (CSI is not supported on RancherOS) and can support other Linux distributions as detailed on the System Configuration page if the appropriate kernel modules are available.\n The StorageOS v2 installation using the Rancher Catalog will be available soon.\n Manual Manual Installation The StorageOS Cluster Operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator is a Kubernetes controller that watches the StorageOSCluster CRD. Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\n Helm Note: If you want to use Helm to install StorageOS, follow the StorageOS Operator Helm Chart documentation.\nSteps to install StorageOS: Install StorageOS Operator Create a Secret for default username and password Trigger bootstrap using a CustomResource Apply StorageOS licence 1. Install StorageOS operator Install the StorageOS Cluster Operator using the following yaml manifest.\nkubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml Verify the Cluster Operator Pod Status [root@master03]# kubectl -n storageos-operator get pod NAME READY STATUS RESTARTS AGE storageoscluster-operator-68678798ff-f28zw 1/1 Running 0 3m The READY 1/1 indicates that storageoscluster resources can be created.\n 2. Create a Secret Before deploying a StorageOS cluster, create a Secret defining the StorageOS API Username and Password in base64 encoding. The API username and password are used to create the default StorageOS admin account which can be used with the StorageOS CLI and to login to the StorageOS GUI. The CSI credentials are used to register the CSI accounts, so Kubernetes and StorageOS communicate over an authenticated API.\napiVersion: v1 kind: Secret metadata: name: \u0026#34;storageos-api\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; labels: app: \u0026#34;storageos\u0026#34; type: \u0026#34;kubernetes.io/storageos\u0026#34; data: # echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64 apiUsername: c3RvcmFnZW9z apiPassword: c3RvcmFnZW9z # CSI Credentials csiProvisionUsername: c3RvcmFnZW9z csiProvisionPassword: c3RvcmFnZW9z csiControllerPublishUsername: c3RvcmFnZW9z csiControllerPublishPassword: c3RvcmFnZW9z csiNodePublishUsername: c3RvcmFnZW9z csiNodePublishPassword: c3RvcmFnZW9z csiControllerExpandUsername: c3RvcmFnZW9z csiControllerExpandPassword: c3RvcmFnZW9z This example contains a default password, for production installations, use a unique, strong password.\n You can define a base64 value by echo -n \u0026quot;mystring\u0026quot; | base64.\n Make sure that the encoding of the credentials doesn\u0026rsquo;t have special characters such as \u0026lsquo;\\n\u0026rsquo;. The echo -n ensures that a trailing new line is not appended to the string.\n If you wish to change the default accounts details post-install please see Managing Users\n 3 Trigger a StorageOS installation This is a Cluster Definition example.\napiVersion: \u0026#34;storageos.com/v1\u0026#34; kind: StorageOSCluster metadata: name: \u0026#34;example-storageos\u0026#34; namespace: \u0026#34;storageos-operator\u0026#34; spec: # StorageOS Pods are in kube-system by default secretRefName: \u0026#34;storageos-api\u0026#34; # Reference from the Secret created in the previous step secretRefNamespace: \u0026#34;storageos-operator\u0026#34; # Namespace of the Secret k8sDistro: \u0026#34;rancher\u0026#34; images: nodeContainer: \u0026#34;storageos/node:v2.0.0\u0026#34; # StorageOS version kvBackend: address: \u0026#39;storageos-etcd-client.storageos-etcd:2379\u0026#39; # Example address, change for your etcd endpoint # address: \u0026#39;10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379\u0026#39; # You can set ETCD server ips resources: requests: memory: \u0026#34;512Mi\u0026#34; cpu: 1 # nodeSelectorTerms: # - matchExpressions: # - key: \u0026#34;node-role.kubernetes.io/worker\u0026#34; # Compute node label will vary according to your installation # operator: In # values: # - \u0026#34;true\u0026#34; Additional spec parameters are available on the Cluster Operator configuration page.\n You can find more examples such as deployments referencing a external etcd kv store for StorageOS in the Cluster Operator examples page.\n Verify StorageOS Installation [root@master03]# kubectl -n kube-system get pods -w NAME READY STATUS RESTARTS AGE storageos-csi-helper-5cf59b5b4-f5nwr 2/2 Running 0 3m storageos-daemonset-75f6c 3/3 Running 0 3m storageos-daemonset-czbqx 3/3 Running 0 3m storageos-daemonset-zv4tq 3/3 Running 0 3m storageos-scheduler-6d67b46f67-5c46j 1/1 Running 0 3m The above command watches the Pods created by the Cluster Definition example. Note that pods typically take approximately 65 seconds to enter the Running Phase.\n 4. License cluster Newly installed StorageOS clusters must be licensed within 24 hours. Our developer license is free, and supports up to 5TiB of provisioned storage.\nTo obtain a license, follow the instructions on our licensing operations page.\n First StorageOS volume If this is your first installation you may wish to follow the StorageOS Volume guide for an example of how to mount a StorageOS volume in a Pod.\n","excerpt":"Make sure the prerequisites for StorageOS are satisfied before proceeding.\n Make sure to add a …","ref":"/v2/docs/install/rancher/","title":"Rancher"},{"body":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following yaml manifest.\nkubectl apply -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n Upgrade StorageOS Operator using Helm If you have installed the StorageOS Operator using the Helm Chart, then you can upgrade the operator using the following commands.\n$ helm list NAME REVISION STATUS CHART APP VERSION NAMESPACE storageos-v1 4 DEPLOYED storageos-operator-0.2.11 1.3.0 storageos-operator $ helm repo update $ helm upgrade $NAME storageos/storageos-operator When you run the above command StorageOS Operator resources will be updated. Since, the Update Strategy of the StorageOS Operator Deployment is set to rolling update, a new StorageOS Operator Pod will be created. Only when the new Pod enters the Running Phase will the old Pod be deleted. Your StorageOS Cluster will not be affected while the StorageOS Operator is upgrading.\n ","excerpt":"Upgrade StorageOS operator from yaml manifest Upgrade the StorageOS operator using the following …","ref":"/v2/docs/reference/cluster-operator/upgrade/","title":"Cluster Operator Upgrade"},{"body":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 active volumes per node 4.x kernels have a limitation of 4096 active volumes per node We are distribution agnostic as long as our prerequisites are met Orchestrators Kubernetes 1.13+ OpenShift 4.0+ ","excerpt":"OS Linux X86_64 Kernels satisfying our module prerequisites 3.x kernels have a limitation of 256 …","ref":"/v2/docs/introduction/platforms/","title":"Supported Platforms and Orchestrators"},{"body":"","excerpt":"","ref":"/v2/docs/concepts/","title":"Concepts"},{"body":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a 64-bit architecture. Kubernetes 1.9+ with CSI. Container Runtime Engine: CRI-O, Containerd or Docker 1.10+ with mount propagation enabled. The necessary ports should be open. See the ports and firewall settings. Etcd cluster for StorageOS A mechanism for device presentation. Recommended: At least three nodes for replication and high availability. Kubernetes 1.13+. Install the storageos CLI. If using Helm2, make sure the tiller ServiceAccount has enough privileges to create resources such as Namespaces, ClusterRoles, etc. For instance, following this installation procedure. System clocks synchronized using NTP or similar methods. While our distributed consensus algorithm does not require synchronised clocks, it does help to more easily correlate logs across multiple nodes. ","excerpt":"Minimum requirements: One machine with the following:\n Minimum two core with 4GB RAM. Linux with a …","ref":"/v2/docs/prerequisites/","title":"Prerequisites"},{"body":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different distributions such as OpenShift, Rancher, EKS, AKS, GKE, etc. The user can provide standard PVC definitions and StorageOS will dynamically provision matching volumes. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nKubernetes and StorageOS communicate with each other to perform actions such as creation, deletion or mounting of volumes. The CSI (Container Storage Interface) driver is the standard method of communication. Using CSI, Kubernetes and StorageOS communicate over a Unix domain socket.\nIt is recommended to use the CSI StorageOS installation from Kubernetes 1.13 onwards. Although the former communication procedure, the native driver, is still in use and StorageOS maintains support for it, its use is discouraged as it is a deprecated method, to be removed in newer versions of Kubernetes.\nCSI (Container Storage Interface) Note CSI is the standard method of communication that enables storage drivers for Kubernetes to release on their own schedule. The CSI standard allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nCSI was released GA from Kubernetes 1.13. StorageOS v2 only supports the use of CSI as a storage driver. In addition, the StorageOS Cluster Operator handles the configuration of the CSI driver and its complexity by detecting the version of the Kubernetes installed.\nCheck out the status of the CSI release cycle in relation with Kubernetes in the CSI project page.\nCSI communication is fully supported by StorageOS if the cluster is deployed with any supported Linux Distribution.\nKubernetes Upgrades on Managed Services Managed services that support in place upgrades are fully supported. However, upgrading Kubernetes using green/blue deployments is not supported. This is because nodes are replaced rather than being upgraded. Any data stored on the nodes is lost when new nodes replace the previous ones.\nDocker Some managed Kubernetes platforms such as Azure AKS, enable the \u0026lsquo;Live-Restore\u0026rsquo; Docker feature, enabling containers to continue running while Docker is stopped or upgraded. This feature can cause nodes to hang while shutting-down or rebooting, as rather than going through an orderly shutdown, StorageOS (and other processes) are killed before the disks are synced and unmounted. Devices in this inaccessible state will log a warning similar to:\nTransport endpoint not connected To prevent this behaviour, we advise disabling this feature by setting\n{ ... \u0026quot;live-restore\u0026quot;: false } in /etc/docker/daemon.json.\nHere\u0026rsquo;s an example Ansible snippet that might be used to achieve this\n... - name: configure /etc/docker/daemon.json lineinfile: path: /etc/docker/daemon.json regexp: '^.*\u0026quot;live-restore\u0026quot;: true,$' line: ' \u0026quot;live-restore\u0026quot;: false,' backrefs: yes notify: restart docker ... Note: Use at own risk; you may need to adapt the example to work in your environment\n ","excerpt":"Kubernetes with StorageOS StorageOS integrates transparently with Kubernetes and different …","ref":"/v2/docs/install/","title":"Install"},{"body":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to a specific platform.\nStorageOS can be used in any Kubernetes distribution when the prerequisites are met. That includes Kubernetes manged services.\n","excerpt":"While StorageOS is platform agnostic, there are certain configurations or operations that relate to …","ref":"/v2/docs/platforms/","title":"Platforms"},{"body":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential cluster metadata and manage distributed configuration state. An embedded etcd instance is included in the StorageOS container, but for production environments and testing of production workloads, we recommend deploying an external etcd cluster. For more details about, and an example of, how to run etcd, see the External etcd Operations page.\nIt is highly recommended to use external etcd for cloud environments and place the etcd cluster on stable nodes. Placing the etcd on nodes that are recycled often might affect the normal operations of StorageOS.\nSetup of storage on the hosts We recommend creating a separate filesystem for StorageOS to mitigate the risk of filling the root filesystem on nodes. This has to be done for each node in the cluster.\nFollow the managing host storage best practices page for more details.\nResource reservations StorageOS resource consumption depends on the workloads and the StorageOS features in use.\nThe recommended minimum memory reservation for the StorageOS Pods is 512MB for non-production environments. However it is recommended to prepare nodes so StorageOS can operate with at least with 1-2GB of memory. StorageOS frees memory when possible.\nFor production environments, we recommend 4GB of Memory and 1 CPU as a minimum and to test StorageOS using realistic workloads and tune resources accordingly.\nStorageOS Pods resource allocation will impact directly on the availability of volumes in case of eviction or resource limit triggered restart. It is recommended to not limit StorageOS Pods.\nStorageOS implements a storage engine, therefore limiting CPU consumption might affect the I/O throughput of your volumes.\nStorageOS API username/password The API grants full access to StorageOS functionality, therefore we recommend that the default administrative password of \u0026lsquo;storageos\u0026rsquo; is reset to something unique and strong.\nYou can change the default parameters by encoding the apiUsername and apiPassword values (in base64) into the storageos-api secret.\nTo generate a unique password, a technique such as the following, which generates a pseudo-random 24 character string, may be used:\n# Generate strong password PASSWORD=$(cat -e /dev/urandom | tr -dc \u0026#39;a-zA-Z0-9-!@#$%^\u0026amp;*()_+~\u0026#39; | fold -w 24 | head -n 1) # Convert password to base64 representation for embedding in a K8S secret BASE64PASSWORD=$(echo -n $PASSWORD | base64) Note that the Kubernetes secret containing a strong password must be created before bootstrapping the cluster. Multiple installation procedures use this Secret to create a StorageOS account when the cluster first starts.\nStorageOS Pod placement StorageOS must run on all nodes that will contribute storage capacity to the cluster or that will host Pods which use StorageOS volumes. For production environments, it is recommended to avoid placing StorageOS Pods on Master nodes.\nStorageOS is deployed with a DaemonSet controller, and therefore tolerates the standard unschedulable (:NoSchedule) action. If that is the only taint placed on master or cordoned nodes StorageOS pods might start on them (see the Kubernetes docs for more details). To avoid scheduling StorageOS pods on master nodes, you can add an arbitrary taint to them for which the StorageOS DaemonSet won\u0026rsquo;t have a toleration.\nDedicated instance groups Cloud environments give users the ability to quickly scale the number of nodes in a cluster in response to their needs. Because of the ephemeral nature of the cloud, StorageOS recommends setting conservative downscaling policies.\nFor production clusters, it recommended to use dedicated instance groups for Stateful applications that allow the user to set different scaling policies and define StorageOS pools based on node selectors to collocate volumes.\nLosing a few nodes at the same time could cause the loss of data even when volume replicas are being used.\nPort blocking StorageOS exposes ports to operate. It is recommended that the ports are not accessible from outside the scope of your cluster.\nStorageOS in Docker EE StorageOS does not support running on Swarm nodes nor on mixed (Kubernetes and Swarm) nodes. StorageOS volumes have to be provisioned and used from Kubernetes nodes.\n","excerpt":"Use an external Etcd cluster StorageOS uses the etcd distributed key-value store to store essential …","ref":"/v2/docs/best-practices/","title":"StorageOS Best Practices"},{"body":"","excerpt":"","ref":"/v2/docs/operations/","title":"Operations"},{"body":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or other Orchestrators that are derived from Kubernetes such as OpenShift or Rancher. This is useful for running stateful applications, such as databases or CI/CD applications, under the control of Kubernetes as Kubernetes can make scheduling decisions without the application data being lost.\nWhat we have outlined in the cookbooks below are some quick deployments of stateful applications into a Kubernetes cluster. These examples are not production ready but have been provided to give you some insight into how to use StorageOS with stateful applications.\nStatefulSets The examples we have provided use StatefulSets as a way to deploy applications into Kubernetes. The reason for this is that the StatefulSet controller is designed to manage stateful applications and it \u0026ldquo;provides guarantees about the ordering and uniqueness\u0026rdquo; of sets of pods.\nPractically this means that when a StatefulSet scales, pods are created in order from 0-(N-1), if a StatefulSet scales down then pods are deleted in reverse order from (N-1)-0.\nSecondly, it means that the behaviour of the StatefulSet controller differs from that of Deployment and ReplicaSet controllers. For Deployment and ReplicaSet controllers \u0026ldquo;\u0026hellip; at many points in the lifetime of a replica set there will be 2 copies of a pod\u0026rsquo;s processes running concurrently\u0026rdquo;. Having two different pods mount a volume at the same time can cause corruption of data. Currently Kubernetes accessModes only apply restrictions to nodes mounting volumes rather than pods, so it is important that StatefulSets are used with StorageOS volumes so the necessary pod uniqueness guarantees are maintained.\nStatefulSet Manifests The StorageOS specific part of the Kubernetes manifests for these examples lies in the VolumeClaimTemplate that\u0026rsquo;s part of the statefulset definition.\nVolumeClaimTemplate\napiVersion:apps/v1kind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1template:metadata:labels:app:mssqlenv:prodspec:serviceAccountName:mssqlcontainers:- name:fooimage:barvolumeMounts:- name:bazmountPath:/var/opt/barenvFrom:- configMapRef:name:mssqlvolumeClaimTemplates:- metadata:name:bazlabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiIn the StatefulSet definition above the container has a volume mount defined called baz. The definition for this volume is found in the VolumeClaimTemplate where the fast storageClass will be used to dynamically provision storage if the persistent volume does not already exist.\n","excerpt":"StorageOS can be used to provide permanent storage for other applications running in Kubernetes or …","ref":"/v2/docs/usecases/","title":"Use Cases: Install applications on Kubernetes with StorageOS"},{"body":"","excerpt":"","ref":"/v2/docs/reference/","title":"Reference"},{"body":"There are several ways to reach us if you require support. The fastest way to get in touch is to join our public Slack channel. \nYou can file a support ticket via email to support@storageos.com, or use our Help Desk portal\nTo help us provide effective support, we request that you provide as much information as possible when contacting us. The list below is a suggested starting point. Additionally, please include anything specific, such as log entries, that may help us debug your issue.\nInformation about the cluster can be automatically sent to StorageOS engineers as mentioned in the section StorageOS Diagnostic Bundle.\nPlatform Cloud provider/Bare metal OS distribution and version Kernel version docker version and installation procedure (distro packages or docker install) StorageOS Version of StorageOS storageos get nodes storageos get volumes storageos describe volume VOL_ID # in case of issues with a specific volume Orchestrator related (Kubernetes, OpenShift, etc) Version and installation method Managed or self managed? kubectl -n kube-system get pod kubectl -n kube-system logs -lapp=storageos -c storageos kubectl -n kube-system get storageclass Specific for your namespaces: kubectl describe pvc PVC_NAME Specific for your namespaces: kubectl describe pod POD_NAME StorageOS Cluster Diagnostic Bundle StorageOS has a cluster diagnostic function that aggregates cluster information.\nThe cluster diagnostic bundle is created only when a user chooses to do so. For convenience the cluster diagnostic bundle can either be uploaded to StorageOS or downloaded to the machine running the browser or CLI. The cluster diagnostic bundle is uploaded from a StorageOS node to a StorageOS GCP encrypted bucket using a TLS encrypted connection. The upload takes place only after user confirmation.\nAs a general outline the following information is sent to us:\n StorageOS logs StorageOS cluster state - node config, volumes and namespaces output of storageos describe node output of lshw output of dmesg See Support Bundle for an exhaustive list of what is included in the cluster diagnostic bundle.\nStorageOS engineers might ask for a cluster diagnostic bundle to be generated during support cases.\nThe information given in the cluster cluster diagnostic bundle is only used for support purposes and it will be removed once the data is no longer needed for such purposes. In case the information is sensitive and can\u0026rsquo;t be given to StorageOS, please make sure that support engineers have as much information about your environment as possible.\nYou can generate a cluster diagnostic bundle through the StorageOS GUI by navigating to the Cluster menu and downloading the StorageOS support Bundle. The cluster cluster diagnostic bundle can also be generated using the CLI command below.\n$ storageos get diagnostics -o storageos-diagnostic-bundle ","excerpt":"There are several ways to reach us if you require support. The fastest way to get in touch is to …","ref":"/v2/docs/support/","title":"Support"},{"body":"If you have any specific or more complex requirements please contact StorageOS as we\u0026rsquo;d be happy to organise a POC in conjunction with our Engineering team. You can join our slack channel or email us at: info@storageos.com\nSupport for Self Evaluations Should you have questions or require support, there are several ways to get in touch with us. The fastest way to get in touch is to join our public Slack channel. You can also get in touch via email to info@storageos.com.\nFurthermore you can fill out the form below and we will get in touch.\n hbspt.forms.create({ portalId: \"3402546\", formId: \"a07fecd3-ce5b-4835-b136-51a94a35632b\", sfdcCampaignId: \"70158000000BAZzAAO\" }); Installation The first phase of the self-evaluation is to install StorageOS. This section of the document aims to layout what options are exposed to you during installation and why some options may be preferable to you over others.\nA standard StorageOS installations uses the StorageOS operator, so as much of the necessary configuration is handled for you. The StorageOS operator has been certified by Red Hat and is open source.\nStorageOS Operator The StorageOS operator is a Kubernetes native application that manages the StorageOS cluster lifecycle. It simplifies cluster installation, cluster removal and other operations.\nThe StorageOS operator watches for the creation of StorageOSCluster Custom Resources. A StorageOSCluster is a declarative representation of a StorageOS cluster. For example if CSI is enabled in the StorageOSCluster resource, a StorageOS cluster will be created that uses the CSI driver.\nStorageOS Operator features Specific configuration options for the StorageOS Operator that we believe to be important during a self-evaluation will be laid out in this guide.\nA set of example StorageOSCluster are listed here. For an exhaustive list of configuration settings for the StorageOS operator please see our documentation.\nExternal Etcd StorageOS highly recommends that an etcd cluster on standalone nodes outside the Kubernetes cluster is used for production deployments. In this configuration the etcd cluster would run on separate boxes from the rest of the Kubernetes and StorageOS cluster ensuring stability and resilience of the etcd cluster. However for the purposes of a self-evaluation it is acceptable to run etcd as a container inside Kubernetes.\nWe do not recommend running etcd on the same nodes as StorageOS when node failure will be tested, as if the majority of etcd nodes fail then the etcd cluster cannot be recovered automatically. Therefore it is better to run etcd on separate nodes.\nPrerequisites StorageOS has some prerequisites that must be met to complete a successful installation\n Machines intended to run StorageOS have at least 1 CPU core, 2GB RAM Docker 1.10 or later with mount propagation enabled TCP ports 5701-5710 and TCP \u0026amp; UDP 5711 open between all nodes in the cluster A 64bit supported operating system - By default StorageOS supports Debian 9, RancherOS, RHEL7.5 and CentOS7. Note Ubuntu 16.04 and 18.04 are supported but additional packages are required. Ubuntu 16.04/18.04 with the AWS kernel and Ubuntu 18.04 with the GCE kernel do not provide the required packages and are therefore NOT supported.\n To install the required kernel modules on Ubuntu 16.04:\nsudo apt -y update sudo apt -y install linux-image-extra-$(uname -r) To install the required kernel modules on Ubuntu 18.04+:\nsudo apt -y update sudo apt -y install linux-modules-extra-$(uname -r) Installing StorageOS Installation steps are as follows:\n Install etcd Install StorageOS Operator Create a Kubernetes secret detailing the default StorageOS administrator account Install StorageOS using a StorageOSCluster Custom Resource Install etcd In order to get a StorageOS cluster stood up quickly, a single node etcd cluster can be installed in Kubernetes, on a Kubernetes master. The reason for installing on a master is that master nodes generally have predictable lifetimes and low Pod scheduling churn. As such there is a lesser risk of the etcd pod being evicted ensuring a stable etcd cluster.\nNote that if the etcd pod is stopped for any reason the etcd cluster will cease to function pending manual intervention. Please take this into account during testing of failure scenarios.\n Download repo\n$ git clone https://github.com/coreos/etcd-operator.git Configure NS, Role and RoleBinding\n$ export ROLE_NAME=etcd-operator $ export ROLE_BINDING_NAME=etcd-operator $ export NAMESPACE=etcd Create Namespace\n$ kubectl create namespace $NAMESPACE Deploy Operator\n$ ./etcd-operator/example/rbac/create_role.sh $ kubectl -n $NAMESPACE create -f - \u0026lt;\u0026lt;END apiVersion: apps/v1 kind: Deployment metadata: name: etcd-operator spec: selector: matchLabels: app: etcd-operator replicas: 1 template: metadata: labels: app: etcd-operator spec: containers: - name: etcd-operator image: quay.io/coreos/etcd-operator:v0.9.4 command: - etcd-operator env: - name: MY_POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name END The Kubernetes masters should then be labelled so a nodeSelector can be used in the EtcdCluster manifest\n$ kubectl label nodes \u0026lt;NODES\u0026gt; etcd-cluster=storageos-etcd Once the master node taints are known and the nodes have been labelled you can deploy an EtcdCluster manifest that contains tolerations for all taints on the master nodes and selects for the node label applied in the previous step. A sample manifest is below. Edit the size to match the number of masters you will deploy on and edit the tolerations to match all taints on the master nodes where etcd will be deployed.\n Create the EtcdCluster resource\n$ kubectl -n etcd create -f - \u0026lt;\u0026lt;END apiVersion: \u0026#34;etcd.database.coreos.com/v1beta2\u0026#34; kind: \u0026#34;EtcdCluster\u0026#34; metadata: name: \u0026#34;storageos-etcd\u0026#34; spec: size: 1 version: \u0026#34;3.4.7\u0026#34; pod: etcdEnv: - name: ETCD_QUOTA_BACKEND_BYTES value: \u0026#34;2147483648\u0026#34; # 2 GB - name: ETCD_AUTO_COMPACTION_RETENTION value: \u0026#34;100\u0026#34; # Keep 100 revisions - name: ETCD_AUTO_COMPACTION_MODE value: \u0026#34;revision\u0026#34; # Set the revision mode resources: requests: cpu: 200m memory: 300Mi securityContext: runAsNonRoot: true runAsUser: 9000 fsGroup: 9000 tolerations: - operator: \u0026#34;Exists\u0026#34; nodeSelector: etcd-cluster: storageos-etcd affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster operator: In values: - storageos-etcd topologyKey: kubernetes.io/hostname END Install StorageOS Operator In order to install the StorageOS operator download the requisite yaml manifests or apply them with kubectl.\n$ kubectl create -f https://github.com/storageos/cluster-operator/releases/download/v2.0.0/storageos-operator.yaml You can verify the operator is running using the following command\n$ kubectl get pods -n storageos-operator Install StorageOS Once the StorageOS operator has been installed a StorageOS cluster can be generated by creating a StorageOSCluster resource.\nA StorageOSCluster resource describes the state of the StorageOS cluster that is desired and the StorageOS operator will create the desired StorageOS cluster. For examples of StorageOSCluster resources please see our examples page here. For a full list of the configurable spec parameters of the StorageOSCluster resource please see here.\n Create a secret defining the API username and password\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:v1kind:Secretmetadata:name:\u0026#34;storageos-api\u0026#34;namespace:\u0026#34;storageos-operator\u0026#34;labels:app:\u0026#34;storageos\u0026#34;type:\u0026#34;kubernetes.io/storageos\u0026#34;data:# echo -n \u0026#39;\u0026lt;secret\u0026gt;\u0026#39; | base64apiUsername:c3RvcmFnZW9zapiPassword:c3RvcmFnZW9z# CSI CredentialscsiProvisionUsername:c3RvcmFnZW9zcsiProvisionPassword:c3RvcmFnZW9zcsiControllerPublishUsername:c3RvcmFnZW9zcsiControllerPublishPassword:c3RvcmFnZW9zcsiNodePublishUsername:c3RvcmFnZW9zcsiNodePublishPassword:c3RvcmFnZW9zEND Create a StorageOSCluster resource\n$kubectlcreate-f- \u0026lt;\u0026lt;ENDapiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:example-storageosclusternamespace:\u0026#34;storageos-operator\u0026#34;spec:secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;storageos-operator\u0026#34;k8sDistro:\u0026#34;upstream\u0026#34;# Set the Kubernetes distribution for your cluster (upstream, eks, aks, gke, rancher, dockeree)# storageClassName: fast # The storage class creates by the StorageOS operator is configurablecsi:enable:truedeploymentStrategy:\u0026#34;deployment\u0026#34;enableProvisionCreds:trueenableControllerPublishCreds:trueenableNodePublishCreds:truekvBackend:address:\u0026#34;storageos-etcd-client.etcd.svc.cluster.local:2379\u0026#34;END Confirm that the cluster has been created and that StorageOS pods are running\n$ kubectl -n kube-system get pods StorageOS pods enter a ready state after a minimum of 65s has passed.\n Deploy the StorageOS CLI as a container\n$ kubectl -n kube-system run \\ --image storageos/cli:v2.0.0 \\ --restart=Never \\ --env STORAGEOS_ENDPOINTS=storageos:5705 \\ --env STORAGEOS_USERNAME=storageos \\ --env STORAGEOS_PASSWORD=storageos \\ --command cli \\ -- /bin/sleep 999999 License cluster\n A newly installed StorageOS cluster does not include a licence. A cluster must be licensed within 24 hours of the installation. For more information, check the reference licence page.\n You can apply a Free Developer licence by following the operations licensing page, or purchase a licence by contacting sales@storageos.com.\n Confirm that StorageOS is working by creating a PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Verify that the CLI is working. pvc-1 should be listed in the CLI output\n$ kubectl -n kube-system exec -it cli -- storageos get volumes The StorageOS web UI can also be used to display information about the state of the cluster. The StorageOS UI can be accessed on any node that is running a StorageOS pod on port 5705. The username/password for the UI is defined by the storageos-api secret. For this self-evaluation the username/password is storageos:storageos\nhttp://\u0026lt;NODE_IP\u0026gt;:5705 Create a pod that consumes the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Check that the pod starts successfully. If the pod starts successfully then the StorageOS cluster is working correctly\n$ kubectl get pod d1 -w The pod mounts a StorageOS volume under /mnt so any files written there will persist the lifetime of the pod. This can be demonstrated using the following commands.\n Execute a shell inside the pod and write some data to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo Hello World! \u0026gt; /mnt/hello root@d1:/# cat /mnt/hello Hello World! Delete the pod\n$ kubectl delete pod d1 Recreate the pod\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: d1 spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-1 END Open a shell inside the pod and check the contents of /mnt/hello\n$ kubectl exec -it d1 -- cat /mnt/hello Hello World! Now that StorageOS has been successfully installed, the cluster has a standard license by default which allows for the creation of 50GB of persistent volumes. If you register the cluster then a developer license will be applied and 500GB of persistent volumes can be created. Replicas do not count towards the license total so a 500GB license could be used to created a 500GB volume with 5 replicas. For the purposes of this self-evaluation the standard license is sufficient.\n StorageOS Features Now that you have a correctly functioning StorageOS cluster we will explain some of our features that may be of use to you as you complete application and synthetic benchmarks.\nStorageOS features are all enabled/disabled by applying labels to volumes. These labels can be passed to StorageOS via persistent volume claims (PVCs) or can be applied to volumes using the StorageOS CLI or GUI.\nThe following is not an exhaustive feature list but outlines features which are commonly of use during a self-evaluation.\nVolume Replication StorageOS enables synchronous replication of volumes using the storageos.com/replicas label.\nThe volume that is active is referred to as the master volume. The master volume and its replicas are always placed on separate nodes. In fact if a replica cannot be placed on a node without a replica of the same volume, the volume will fail to be created. For example, in a three node StorageOS cluster a volume with 3 replicas cannot be created as the third replica cannot be placed on a node that doesn\u0026rsquo;t already contain a replica of the same volume.\nThe failure mode for a volume affects how many failed replicas can be tolerated before the volume is marked as offline. Replicas are also segregated according to the iaas/failure-domains node label. StorageOS will automatically place a master volume and its replicas in separate failure domains where possible.\nSee our replication documentation for more information on volume replication.\n To test volume replication create the following PersistentVolumeClaim\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-replicated labels: storageos.com/replicas: \u0026#34;1\u0026#34; spec: storageClassName: \u0026#34;fast\u0026#34; accessModes: - ReadWriteOnce resources: requests: storage: 5Gi END Note that volume replication is enabled by setting the storageos.com/replicas label on the volume.\n Confirm that a replicated volume has been created by using the StorageOS CLI or UI\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Create a pod that uses the PVC\n$ kubectl create -f - \u0026lt;\u0026lt;END apiVersion: v1 kind: Pod metadata: name: replicated-pod spec: containers: - name: debian image: debian:9-slim command: [\u0026#34;/bin/sleep\u0026#34;] args: [ \u0026#34;3600\u0026#34; ] volumeMounts: - mountPath: /mnt name: v1 volumes: - name: v1 persistentVolumeClaim: claimName: pvc-replicated END Write data to the volume\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# echo Hello World! \u0026gt; /mnt/hello root@replicated-pod:/# cat /mnt/hello Hello World! Find the location of the master volume and shutdown the. Shutting down a node causes all volumes, with online replicas, on the node to be evicted. For replicated volumes this immediately promotes a replica to become the new master.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-replicated Bound pvc-29e2ad6e-8c4e-11e9-8356-027bfbbece86 5Gi RWO fast 1m $ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker (online) kind-worker2 1/1 26 seconds ago Check the location of the master volume and notice that it is on a new node. If the pod that mounted the volume was located on the same node that was shutdown then the pod will need to be recreated.\n$ kubectl exec -it -n kube-system cli -- storageos get volumes NAMESPACE NAME SIZE LOCATION ATTACHED ON REPLICAS AGE default pvc-4e796a62-0271-45f9-9908-21d58789a3fe 5.0 GiB kind-worker2 (online) kind-worker2 1/1 46 seconds ago Check that the data is still accessible to the pod\n$ kubectl exec -it replicated-pod -- bash root@replicated-pod:/# cat /mnt/hello Hello World! Benchmarking As a rule the best performance is obtained by using unreplicated volumes that are co-located with the application or benchmarking tool writing to the volume. See Volume Placement below for more information.\nWhen running benchmarks in the cloud, benchmarks need to be run multiple times and nodes should be destroyed and recreated so that the underlying machine changes. This should be done to reduce the impact that noisy neighbours might have on benchmark results.\nConsiderations Application vs StorageOS replication Certain applications are able to natively replicate or shard data between application instances. When using these applications it is worth considering whether application replication, StorageOS replication or a mixture of both should be used.\nWhen StorageOS replication is enabled the time to recover in cases of node failure can be lessened. This is because StorageOS will promote a replica, Kubernetes will reschedule the application instance and the amount of data the application needs to catch up on is limited to whatever data was modified while the application was being rescheduled. Without StorageOS replication the application would have to rebuild an entire copy of data. Some applications have their performance greatly impacted by having to rebuild shards/replicas so this is also avoided.\nStatefulSets StatefulSets are the de facto controller for stateful applications. As such, when deploying applications that will use StorageOS volumes, StatefulSets should be used. You can find more information about StatefulSets here.\nVolume Placement StorageOS volumes give the best performance when the application pod and the master volume are co-located on the same node. When benchmarking applications, it is useful to take into account that using remote volumes and replicas impact the overall performance of a volume.\nGoing from 0 to 1 replica has the greatest performance impact for writes as now the latency of the operation is equal to the round trip time to the node with the replica over the network. Adding additional replicas poses less of a performance impact as writes to replicas are done in parallel, and the round trip time to each node is unlikely to greatly increase unless replicas land on nodes that are geographically distant to the master volumes\u0026rsquo; node.\nEven when volumes are replicated co-location of pod and master volume is still desirable because application writes are first sent to the master and then sent from the master volume to the replicas. Writing to a local master therefore saves network latency between the application and the master volume. As reads are always served from the master volume a remote master volume will add latency to reads as well as writes.\nWhen testing applications, such as databases, it is also necessary to run benchmarks for a sufficiently long time to account for caching, and cache flushing that databases do. We recommend running application benchmarks over a 20-30min period for this reason.\nHow to land a volume and a pod on the same node StorageOS has an automatic co-location feature on our development roadmap that we are calling pod locality. Until the feature is GA co-location of a master volume and a pod can be achieved by leveraging existing StorageOS and Kubernetes features.\nstorageos.com/hint.master is a volume label that influences the placement of a StorageOS master volume. By setting this label to the StorageOS nodeID that corresponds to the nodeSelector on a StatefulSet or Pod the master volume and the pod should co-locate on the same node. You can reference our FIO local volumes job for an example of how to do this. Synthetic Benchmarks Synthetic benchmarks using tools such as FIO are a useful way to begin measuring StorageOS performance. While not fully representative of application performance, they allow us to reason about the performance of storage devices without the added complexity of simulating real world workloads, and provide results easily comparable across platforms.\nAs with application benchmarks, when testing in public clouds multiple runs on newly created nodes should be considered to account for the impact of noisy neighbours.\nStorageOS has created a test suite for running FIO tests against StorageOS volumes that can be found here. The test suite can be deployed into a Kubernetes cluster using the instructions below.\n Clone the StorageOS use cases repo. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/FIO/local-volumes.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the FIO local-volumes folder\n$ cd storageos-usecases/FIO/local-volumes Get the name of the node that you wish the FIO pods and volumes to be created on. Make sure that the node name and the label kubernetes.io/hostname match and that the node has enough storage capacity to create 8Gi worth of volumes\n$ kubectl get node --show-labels Generate the FIO jobs by passing in the node name that the job should run on, and the StorageOS node id for that node. The number is the number of volumes that FIO will test concurrently. The StorageOS node ID can be found using the cli to describe node\n$ ./job-generator-per-volumecount.sh 4 $NODE_NAME $STORAGEOS_NODEID Upload the FIO profiles as ConfigMaps\n$ ./upload-fio-profiles.sh Run the FIO tests\n$ kubectl create -f ./jobs Check the PVCs have been provisioned\n$ kubectl get pvc Use the StorageOS CLI to check the location of the volumes\n$ kubectl -n kube-system exec cli -- storageos get volumes Verify that the Pod is running on the same node\n$ kubectl get pod -owide FIO has a number of parameters that can be adjusted to simulate a variety of workloads and configurations. Particularly the queue depth, block size and the number of volumes used affect the FIO results. To tune the FIO parameters the profiles file can be edited or the ConfigMap that is created from the profiles file can be edited directly.\nStorageOS configuration also affects the overall volume performance. For example adding a replica to a volume will increase the latency for writes and affect IOPS and bandwidth for the volume.\nTo see the effect a StorageOS replica has on performance rerun an FIO test but add the storageos.com/replicas: \u0026quot;1\u0026quot; label to the PersistentVolumeClaims in the jobs spec. The greatest performance impact from adding replicas comes when moving from 0 to 1 replica. Adding additional replicas does not incur a significant performance penalty.\nThe remote volumes folder contains a guide for performing the same FIO tests against remote volumes. Application Benchmarks While synthetic benchmarks are useful for examining the behaviour of StorageOS with very specific workloads, in order to get a realistic picture of StorageOS performance actual applications should be tested.\nMany applications come with test suites which provide standard workloads. For best results, test using your application of choice with a representative configuration and real world data.\nAs an example of benchmarking an application the following steps lay out how to benchmark a Postgres database backed by a StorageOS volume.\n Start by cloning the StorageOS use cases repository. Note this is the same repository that we cloned earlier so if you already have a copy just cd storageos-usecases/pgbench.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases Move into the Postgres examples folder\n$ cd storageos-usecases/pgbench Decide which node you want the pgbench pod and volume to be located on. The node needs to be labelled app=postgres\n$ kubectl label node \u0026lt;NODE\u0026gt; app=postgres Then set the storageos.com/hint.master label in 20-postgres-statefulset.yaml file to match the StorageOS nodeID for the node you have chosen before creating all the files. The StorageOS nodeID can be obtained using the cli and doing a describe node\n$ kubectl create -f . Confirm that Postgres is up and running\n$ kubectl get pods -w -l app=postgres Use the StorageOS CLI or the GUI to check the master volume location and the mount location. They should match\n$ kubectl -n kube-system exec -it cli -- storageos get volumes Exec into the pgbench container and run pgbench\n$ kubectl exec -it pgbench -- bash -c \u0026#39;/opt/cpm/bin/start.sh\u0026#39; Conclusion After completing these steps you will have benchmark scores for StorageOS. Please keep in mind that benchmarks are only part of the story and that there is no replacement for testing actual production or production like workloads.\nStorageOS invites you to provide feedback on your self-evaluation to the slack channel or by directly emailing us at info@storageos.com\n","excerpt":"If you have any specific or more complex requirements please contact StorageOS as we\u0026rsquo;d be …","ref":"/v2/docs/self-eval/","title":"Self Evaluation Guide"},{"body":"","excerpt":"","ref":"/v2/index.json","title":""},{"body":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: '#ohpen_swagger_ui', presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ] }) window.ui = ui } \t","excerpt":" \t window.onload = function() { const ui = SwaggerUIBundle({ url: \"/api-swagger-v2.yaml\", dom_id: …","ref":"/v2/docs/reference/api/","title":"API Reference"},{"body":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply licence --from-file \u0026lt;path-to-licence-file\u0026gt; $ echo \u0026#34;\u0026lt;licence file contents\u0026gt;\u0026#34; | storageos apply licence --from-stdin Flags: --from-file string reads a StorageOS product licence from a specified file path --from-stdin reads a StorageOS product licence from the standard input -h, --help help for licence Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos apply licence --help Usage: storageos apply licence [flags] Examples: $ storageos apply …","ref":"/v2/docs/reference/cli/apply/","title":"Apply licence"},{"body":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes.\nFundamentally, StorageOS uses the storage attached to the nodes in the StorageOS cluster to create and present virtual volumes into containers. Space on the host is consumed from the mount point /var/lib/storageos/data, so it is therefore recommended that disk devices are used exclusively for StorageOS, as described in Managing Host Storage \nStorageOS is agnostic to the underlying storage and runs equally well on bare metal, in virtual machines or on cloud providers.\nRead about the cloud native storage principles behind StorageOS.\nStorageOS on Kubernetes StorageOS is architected as a series of containers that fulfil separate, discrete functions.\n StorageOS Controlplane\nResponsible for monitoring and maintaining the state of volumes and nodes in the cluster\n StorageOS Dataplane\nResponsible for all I/O path related tasks; reading, writing, compression and caching\n StorageOS Scheduler\nResponsible for scheduling applications on the same node as applications volume\n CSI helper\nResponsible for registering StorageOS with Kubernetes as a CSI driver\n StorageOS Operator\nResponsible for the creation and maintenance of the StorageOS cluster\n StorageOS is deployed by the StorageOS operator. In Kubernetes, the StorageOS Controlplane and Dataplane are deployed in a single pod managed by a daemonset. This daemonset runs on every node in the cluster that will consume or present storage. The scheduler, CSI helper and Operator run as separate pods and are controlled as deployments.\nStorageOS is designed to feel familiar to Kubernetes and Docker users. Storage is managed through standard StorageClasses and PersistentVolumeClaims, and features are controlled by Kubernetes-style labels and selectors, prefixed with storageos.com/. By default, volumes are cached to improve read performance and compressed to reduce network traffic.\nAny pod may mount a StorageOS virtual volume from any node that is also running StorageOS, regardless of whether the pod and volume are collocated on the same node. Therefore, applications may be started or restarted on any node and access volumes transparently.\n","excerpt":"StorageOS is a software-defined storage platform for running stateful applications in Kubernetes. …","ref":"/v2/docs/concepts/architecture/","title":"Architecture"},{"body":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos attach --namespace my-namespace-name my-volume my-node Flags: -h, --help help for attach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos attach Attach a volume to a node Usage: storageos attach [flags] Examples: $ storageos …","ref":"/v2/docs/reference/cli/attach/","title":"Attach"},{"body":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our Kubernetes installation instructions page.\nAn EKS deployment of Kubernetes uses AWS Linux by default with an optimized kernel. As the requisite kernel modules are not available for StorageOS to use TCMU, FUSE will be used as a fallback. Using FUSE instead of TCMU has performance implications.\nFor more details about the OS Distributions check the System Configuration page.\n","excerpt":"StorageOS is fully compatible with AWS EKS. To install StorageOS on EKS, please follow our …","ref":"/v2/docs/platforms/aws-eks/","title":"AWS EKS"},{"body":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our Kubernetes installation instructions page.\nAn AKS deployment of Kubernetes uses Ubuntu by default with an optimized kernel. All versions of Ubuntu with a kernel version later than4.15.0-1029-azure meet the StorageOS prerequisites.\n","excerpt":"StorageOS is fully compatible with Azure AKS. To install StorageOS on AKS please follow our …","ref":"/v2/docs/platforms/azure-aks/","title":"Azure AKS"},{"body":"In this example use case we provide three different strategies for accessing files that have been written to a StorageOS persistent volume.\nIn the following examples the \u0026ldquo;application\u0026rdquo; container is the container main, which has a rsync, Nginx or sftp sidecar container. The StorageOS volume that the application is writing to will be mounted into the sidecar container so files written by the application are available for export. Files can be exported using Nginx as a web file server, transferred using rsync or accessed via SFTP.\nThe files create a stateful set that can be used AFTER a StorageOS cluster has been created. See our guide on how to install StorageOS on Kubernetes for more information.\nClone Repository In order to deploy the examples, clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/backup Before deploying the backup-example stateful set we recommend looking through the examples to understand how the different containers are configured\n Exfiltrating files through HTTP Deploy the Nginx example $ kubectl create -f nginx/ service/backup-example created configmap/nginx-config created statefulset.apps/backup-example created pod/busybox created Check that a backup-example pod is running $ kubectl get pods -w -l app=backup-example-nginx NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file $ kubectl exec -it backup-example-nginx-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-nginx ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 80/TCP 46s Use wget to access the files served by Nginx. Nginx is sharing files from the same volume that the main application container is writing to. The connection to the Nginx container is made via the backup-example service. $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;lost%2Bfound/\u0026#34;\u0026gt;lost+found/\u0026lt;/a\u0026gt; 12-Feb-2019 12:32 - \u0026lt;a href=\u0026#34;date.txt\u0026#34;\u0026gt;date.txt\u0026lt;/a\u0026gt; 12-Feb-2019 12:49 29 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; $ kubectl exec -it busybox -- /bin/wget -q -O- http://backup-example-nginx/date.txt Tue Feb 12 12:49:15 UTC 2019 Depending on what files have been written to the StorageOS volume the output of the index file will be different. In the example the date.txt file we created in Step 2 is present on the volume.\nExfiltrating files through Rsync Deploy the rsync example $ kubectl create -f rsync/ service/backup-example created configmap/rsync-config created secret/rsync-credentials created statefulset.apps/backup-example created pod/rsync created Check that a backup-example pod is running $ kubectl get pods -w -l app=backup-example-rsync NAME READY STATUS RESTARTS AGE backup-example-0 1/1 Running 0 1m Exec into the main container and write some data to a file $ kubectl exec -it backup-example-rsync-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-rsync NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-rsync ClusterIP 100.65.18.199 \u0026lt;none\u0026gt; 873/TCP 46s Use rsync to access the files shared by the rsync daemon. rsync is sharing files from the same volume that the main container is writing to. A username and password that are set in the rsync-credentials secret. The secret supplied in the example has the username and password set to username and password. $ kubectl exec -it rsync sh / # rsync --list-only rsync://username@backup-example-rsync/share Password: drwxr-xr-x 4,096 2019/02/12 12:49:15 . -rw-r--r-- 29 2019/02/12 12:49:15 date.txt drwx------ 16,384 2019/02/12 12:32:40 lost+found / # rsync -chavzP rsync://username@backup-example-rsync/share/date.txt . Password: receiving incremental file list date.txt 29 100% 28.32kB/s 0:00:00 (xfr#1, to-chk=0/1) sent 43 bytes received 135 bytes 50.86 bytes/sec total size is 29 speedup is 0.16 / # cat date.txt Tue Feb 12 12:49:15 UTC 2019 In the example above the list of available files was shown and a file called date.txt was synchronized to the rsync container.\nExfiltrating files through SFTP Deploy the sftp example $ kubectl create -f sftp/ Exec into the main container and write some data to a file $ kubectl exec -it backup-example-sftp-0 -c main bash root@backup-example-0:/# echo $(date) \u0026gt; /data/date.txt Check that the service exists $ kubectl get svc backup-example-sftp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE backup-example-sftp ClusterIP 100.70.50.56 \u0026lt;none\u0026gt; 22/TCP 2h Use SFTP to access the files shared by the SFTP container. If you have made no changes to the sftp-config secret the password is password. $ kubectl exec -it sftp -- bash root@sftp:/# sftp alex@backup-example-sftp alex@backup-example-sftp\u0026#39;s password: Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found sftp\u0026gt; get date.txt Fetching /date.txt to date.txt /date.txt 100% 29 15.9KB/s 00:00 sftp\u0026gt; bye root@sftp:/# cat date.txt Tue Feb 12 17:51:32 UTC 2019 In order to do this a SFTP user needs to be configured. The details for the user are stored in the sftp-config secret (see sftp/17-secret.yaml). The secret consists of base64 encoded username:password:uid:guid and the user is chroot\u0026rsquo;ed inside their home directory so the mount point for the StorageOS volume in the SFTP container in sftp/20-backup-pod.yaml needs to be configured.\nUsing custom SSH Keys The ConfigMap ssh-key-pub (see sftp/15-configmap.yaml) needs to be populated with a public key. The corresponding private key needs to be base64 encoded and put into the ssh-key-private secret (see sftp/17-secret.yaml). The user to connect as is determined by the user that is configured in the sftp-config configMap. To restrict logins to the SSH key edit the sftp-config secret so it contains no password (user::uid:guid).\n Connect to the sftp pod and connect through the service to the SFTP container running inside the backup-example pod. $ kubectl exec -it sftp -- bash root@sftp:/# sftp -i /home/alex/.ssh/id_rsa alex@backup-example-sftp Connected to backup-example-sftp. sftp\u0026gt; ls date.txt lost+found ","excerpt":"In this example use case we provide three different strategies for accessing files that have been …","ref":"/v2/docs/usecases/backups/","title":"Backing up files from StorageOS volumes"},{"body":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing StorageOS persistent volumes with Cassandra means that if a Cassandra pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up the pod data is immediately available. Should Kubernetes schedule the Cassandra pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not a StorageOS master is located on the same node.\nAs Cassandra has features to allow it to handle replication careful consideration of whether to allow StorageOS or Cassandra to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Cassandra on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiVersion:apps/v1kind:StatefulSetmetadata:name:cassandraspec:selector:matchLabels:app:cassandraserviceName:cassandrareplicas:3...spec:...volumeMounts:- name:cassandra-datamountPath:/var/lib/cassandra...volumeClaimTemplates:- metadata:name:cassandra-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClassresources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Move into the Cassandra examples folder and create the objects\ncd storageos-usecases kubectl create -f ./cassandra Confirm Cassandra is up and running.\n$ kubectl get pods -w -l app=cassandra NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 8m32s cassandra-1 1/1 Running 0 7m51s cassandra-2 1/1 Running 0 6m36s Connect to the Cassandra client pod and connect to the Cassandra server through the service\n$ kubectl exec -it cassandra-0 -- cqlsh cassandra-0.cassandra Connected to K8Demo at cassandra-0.cassandra:9042. [cqlsh 5.0.1 | Cassandra 3.11.3 | CQL spec 3.4.4 | Native protocol v4] Use HELP for help. cqlsh\u0026gt; SELECT cluster_name, listen_address FROM system.local; cluster_name | listen_address --------------+---------------- K8Demo | 100.96.7.124 (1 rows) ","excerpt":"Cassandra with StorageOS Cassandra is a popular distributed NoSQL open source database.\nUsing …","ref":"/v2/docs/usecases/cassandra/","title":"Cassandra"},{"body":"The storageos command line interface (CLI) is used to manage cluster-wide configuration.\nInstallation # linux/amd64 curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.0.0/storageos_linux_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ # MacOS curl -sSLo storageos https://github.com/storageos/go-cli/releases/download/v2.0.0/storageos_darwin_amd64 \u0026amp;\u0026amp; chmod +x storageos \u0026amp;\u0026amp; sudo mv storageos /usr/local/bin/ You will need to provide the correct credentials to connect to the API. The default installation uses the storageos-api Secret to generate the first admin user. By default, it creates a single user with username storageos and password storageos:\nexport STORAGEOS_USERNAME=storageos export STORAGEOS_PASSWORD=storageos export STORAGEOS_ENDPOINTS=10.1.5.249:5705 Usage $ storageos Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Formatting CLI Output StorageOS CLI output can be formatted using the --output option. The strings that are passed to --output are \u0026lsquo;json\u0026rsquo;, \u0026lsquo;yaml\u0026rsquo; or \u0026lsquo;text\u0026rsquo;.\nCheatsheet Command Subcommand Description apply licence Apply licence to cluster. attach Attach Volume to a node. create user, volume Create resources such as users or volumes. delete volume Delete resources. describe node, volume Show detailed view of resources. detach volume Detach volume from a node. get cluster, diagnostics, namespace, node, user, volume List resources. help Help version Show CLI version. Source is available on Github.\n","excerpt":"The storageos command line interface (CLI) is used to manage cluster-wide configuration. …","ref":"/v2/docs/reference/cli/","title":"Command line interface"},{"body":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS clusters, and assist with maintenance operations. We recommend its use for standard installations.\nThe operator acts as a Kubernetes controller that watches the StorageOSCluster CR (Custom Resource). Once the controller is ready, a StorageOS cluster definition can be created. The operator will deploy a StorageOS cluster based on the configuration specified in the cluster definition.\nYou can find the source code in the cluster-operator repository.\nInstall the operator following orchestrator specific procedure.\nTo deploy a StorageOS cluster you will need to fulfil the following steps.\n Install Etcd Install operator Create storageos-api Secret Create a CR (Custom Resource) to deploy StorageOS ","excerpt":"Our cluster operator is a Kubernetes native application developed to deploy and configure StorageOS …","ref":"/v2/docs/reference/cluster-operator/","title":"Cluster Operator"},{"body":"StorageOS clusters represent groups of nodes which run a common distributed control plane.\nTypically, a StorageOS cluster maps one-to-one to a Kubernetes (or similar orchestrator) cluster, and we expect our daemonset to run on all worker nodes within the cluster that will consume or present storage.\nClusters use etcd to maintain state and manage distributed consensus between nodes.\n","excerpt":"StorageOS clusters represent groups of nodes which run a common distributed control plane. …","ref":"/v2/docs/concepts/clusters/","title":"Clusters"},{"body":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is generally increased when compression is enabled due to fewer read/write operations taking place on the backend store (the volumes\u0026rsquo; blob files). Compression can be disabled by setting the label storageos.com/nocompress=true on a volume.\nStorageOS utilises the lz4 compression algorithm when writing to the backend store and when compressing replication traffic before it is sent across the network. Compression is granular per 4k block and data will remain compressed/uncompressed once written to a volume. Therefore, compression can be dynamically enabled and disabled by setting the storageos.com/nocompress label on a volume.\nStorageOS detects whether a block can be compressed or not by creating a heuristic that predicts the size of a compressed block. If the heuristic indicates that the compressed block is likely to be larger than the original block then the uncompressed block is stored. Block size increases post compression if the compression dictionary is added to a block that cannot be compressed. By verifying whether blocks can be compressed, disk efficiency is increased and CPU resources are not wasted on attempts to compress uncompressible blocks. StorageOS\u0026rsquo; patented on disk format is used to tell whether individual blocks are compressed without overhead. As such volume compression can be dynamically enabled/disabled even while a volume is in use.\nWhen compression and encryption are both enabled for a volume, blocks are compressed then encrypted.\n","excerpt":"StorageOS compression is handled on a per volume basis and is enabled by default, as performance is …","ref":"/v2/docs/concepts/compression/","title":"Compression"},{"body":"We are always looking to improve our documentation. If you like to help people and can write, read on for the process for submitting your contributions. If your guide is published, you\u0026rsquo;ll receive $250 per article by PayPal.\nContent guidelines A guide contains step by step instructions for how to accomplish a specific task using StorageOS. To be accepted, guides must be:\n Written in English. Relevant, accurate and complete. Technically correct and thoroughly tested. Follow the writing style guide. Guides should avoid:\n Duplicating an existing guide or other sources, such as blogs or forum posts. Including irrelevant material. Submission and review You should submit your guide as a pull request to the GitHub repo.\nYour submission will be left open for community review for two weeks. After this, your submission will be reviewed internally for about another week.\nIf accepted, your pull request will be approved and you will have 36 hours to send your submission title and PayPal account information. Non-response will be taken as a go-ahead to publish.\nLegal COPYRIGHT OWNERSHIP. The StorageOS Guides \u0026amp; Tutorials repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license.\nCREDIT. Nothing contained in this Agreement shall be deeded to require StorageOS to use the Work, or any part thereof, in connection with StorageOS Guides \u0026amp; Tutorials or otherwise. Credit for the Work shall read, “Contributed by writer’s name.”\nPAYMENT. Upon publication of a submission to the StorageOS Guides \u0026amp; Tutorials Repository, the writer will be paid the sum of USD $250.00 as an electronic payment.\n","excerpt":"We are always looking to improve our documentation. If you like to help people and can write, read …","ref":"/v2/docs/reference/contributing/","title":"Contributing to the docs"},{"body":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: user Create a new user account volume Provision a new volume Flags: -h, --help help for create Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos create [command] --help\u0026#34; for more information about a command. create volume $ storageos create volume --help Provision a new volume Usage: storageos create volume [flags] Examples: $ storageos create volume --description \u0026#34;This volume contains the data for my app\u0026#34; --fs-type \u0026#34;ext4\u0026#34; --labels env=prod,rack=db-1 --size 10GiB --namespace my-namespace-name my-app $ storageos create volume --replicas 1 --namespace my-namespace-name my-replicated-app Flags: --async perform the operation asynchronously, using the configured timeout duration --cache caches volume data (default true) --compress compress data stored by the volume at rest and during transit (default true) -d, --description string a human-friendly description to give the volume -f, --fs-type string the filesystem to format the new volume with once provisioned (default \u0026#34;ext4\u0026#34;) -h, --help help for volume -l, --labels strings an optional set of labels to assign to the new volume, provided as a comma-separated list of key=value pairs -r, --replicas uint the number of replicated copies of the volume to maintain -s, --size string the capacity to provision the volume with (default \u0026#34;5GiB\u0026#34;) --throttle deprioritises the volume\u0026#39;s traffic by reducing the rate of disk I/O Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) create user $ storageos create user --help Create a new user account Usage: storageos create user [flags] Examples: $ storageos create user --with-username=alice --with-admin=true Flags: -h, --help help for user --with-admin control whether the user is given administrative priviledges --with-groups stringArray the list of policy groups to assign to the user --with-password string the password to assign to the user. If not specified, this will be prompted for. --with-username string the username to assign Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos create --help Create new resources Usage: storageos create [command] Available Commands: …","ref":"/v2/docs/reference/cli/create/","title":"Create"},{"body":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available Commands: volume Delete a volume Flags: -h, --help help for delete Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos delete [command] --help\u0026#34; for more information about a command. delete volume $ storageos delete volume --help Delete a volume Usage: storageos delete volume [volume name] [flags] Examples: $ storageos delete volume my-test-volume my-unneeded-volume $ storagoes delete volume --namespace my-namespace my-old-volume Flags: --async perform the operation asynchronously, using the configured timeout duration --cas string make changes to a resource conditional upon matching a provided version -h, --help help for volume Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos delete -h Delete resources in the cluster Usage: storageos delete [command] Available …","ref":"/v2/docs/reference/cli/delete/","title":"Delete"},{"body":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] Available Commands: node Retrieve detailed information for nodes in the cluster volume Show detailed information for volumes Flags: -h, --help help for describe Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos describe [command] --help\u0026#34; for more information about a command. describe node $ storageos describe node --help Retrieve detailed information for nodes in the cluster Usage: storageos describe node [node names...] [flags] Aliases: node, nodes Examples: $ storageos describe node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) describe volume $ storageos describe volume --help Show detailed information for volumes Usage: storageos describe volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos describe volumes $ storageos describe volume --namespace my-namespace-name my-volume-name Flags: -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos describe --help Fetch extended details for resources Usage: storageos describe [command] …","ref":"/v2/docs/reference/cli/describe/","title":"Describe"},{"body":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] Examples: $ storageos detach volume --namespace my-namespace-name my-volume Flags: --cas string make changes to a resource conditional upon matching a provided version -h, --help help for detach Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos detach --help Detach a volume from its current location Usage: storageos detach [flags] …","ref":"/v2/docs/reference/cli/detach/","title":"Detach Volume"},{"body":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page.\nDocker EE and the Universal Control Plane can run on different Linux distributions. StorageOS supports RHEL, CentOS, Debian, and selected Ubuntu images. For more details, check out the supported OSs in the prerequisites page.\nStorageOS only supports Kubernetes nodes managed by Docker Enterprise Edition, not those nodes running Swarm. Mixed nodes (those running Kubernetes and Swarm workloads) are not supported. As a consequence, StorageOS volumes can only be provisioned on Kubernetes nodes, and only these nodes should be used for stateful workloads.\n","excerpt":"To install StorageOS on DockerEE, please follow our Kubernetes installation instructions page. …","ref":"/v2/docs/platforms/dockeree/","title":"Docker Enterprise Edition"},{"body":"","excerpt":"","ref":"/v2/docs/","title":"Documentation"},{"body":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, most popularly used to aggregate logs, but also to serve as a search backend to a number of different applications.\nUsing StorageOS persistent volumes with ElasticSearch (ES) means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Elasticsearch pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nElasticsearch has features to allow it to handle data replication, and as such careful consideration of whether to allow StorageOS or Elasticsearch to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Elasticsearch on Kubernetes Prerequisites Some OS tuning is required, which is done automatically when using our example from the use cases repository.\nElasticsearch requires vm.max_map_count to be increased to a minimum of 262144, which is a system wide setting. One way to achieve this is to run sysctl -w vm.max_map_count=262144 and update /etc/sysctl.conf to ensure it persists over a reboot. See ElasicSearch reference here.\nAdministrators should be aware that this impacts the behaviour of nodes and that there may be collisions with other application settings. Administrators are advised to centrally collate sysctl settings using the tooling of their choice.\nDeployment of the application StatefulSet defintion ---apiVersion:apps/v1kind:StatefulSetmetadata:name:esdata[...]spec:serviceAccountName:elasticsearchcontainers:- name:dataimage:elasticsearch:6.7.0imagePullPolicy:IfNotPresent[...]volumeMounts:- name:datamountPath:/usr/share/elasticsearch/data/data[...]volumeClaimTemplates:- metadata:name:\u0026#34;data\u0026#34;spec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- default StorageOS storage class nameresources:requests:storage:10Gi# \u0026lt;--- change this to the appropriate valueThis excerpt is from the StatefulSet definition (/elasticsearch/10-es-data.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nInstallation Clone the use cases repo You can find the latest files in the StorageOS use cases repostiory in /elasticsearch/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases Create the kubernetes objects\nThis will install an ES cluster with 3 master, 3 data and 3 coordinator nodes. Combined they will require ~ 14 GiB of available memory in your cluster, however, more may be used as the application is being used\nkubectl apply -f ./elasticsearch/ Once completed, an internal service object will have been created making the cluster available as http://elasticsearch:9200/ which is the default Kibana (when installed via Helm) will be using.\n Confirm Elasticsearch is up and running\nkubectl get pods -l component=elasticsearch NAME READY STATUS RESTARTS AGE elasticsearch-exporter-d86ffd94-zw45l 1/1 Running 0 5m44s es-coordinator-b7b984dd4-7wlz5 1/1 Running 0 5m44s es-coordinator-b7b984dd4-89w26 1/1 Running 0 5m44s es-coordinator-b7b984dd4-b4t6j 1/1 Running 0 5m44s es-master-78dfd5b49f-9gf5c 1/1 Running 0 5m44s es-master-78dfd5b49f-smsbw 1/1 Running 0 5m44s es-master-78dfd5b49f-z4qpj 1/1 Running 0 5m44s esdata-0 1/1 Running 0 5m44s esdata-1 1/1 Running 0 4m34s esdata-2 1/1 Running 0 3m22s Connect to ElasticSearch\nTo connect to ES directly, you can use the following port-forward command\nkubectl port-forward svc/elasticsearch 9200 and then access it via http://localhost:9200\n Kibana (optional) One of the most popular uses of ES is to use it for log aggregation and indexing, Kibana helps us visualize the data in these indices and can be easily used when installed via its Helm chart\n Install the helm chart.\nhelm install stable/kibana Once installed, use a port-foward to Kibana instead of directly to ES\nkubectl port-forward --namespace default $(kubectl get pods --namespace default -l \u0026#34;app=kibana\u0026#34; -o jsonpath=\u0026#34;{.items[0].metadata.name}\u0026#34;) 5601 and then access it via http://localhost:5601\n Monitoring (optional) As part of the example deployment, ES metrics are exposed and can be scraped by Prometheus on port 9108 (see 77-es-exporter.yaml). This is enabled by default, and should work with the default Prometheus install via Helm. If you\u0026rsquo;re using the Prometheus service monitors, you can monitor this installation by creating a monitor for the es-exporter service. For an example of how this is done to monitor StorageOS, please see prometheus-setup.\n","excerpt":"Elasticsearch with StorageOS Elasticsearch is a distributed, RESTful search and analytics engine, …","ref":"/v2/docs/usecases/elasticsearch/","title":"Elasticsearch"},{"body":"etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to durably persist the StorageOS cluster state. As the backing store for Kubernetes, StorageOS uses etcd for many of the same reasons.\nStorageOS uses etcd as the single source of truth for all StorageOS objects. Whenever a request is made to create, update or delete an object the result is written to etcd before the request is completed. Using etcd as a configuration store allows nodes to retrieve the current cluster state after being offlined, allowing offlined nodes to rejoin the cluster.\n N.B. StorageOS v2.0 does not provide an embedded etcd server as previous versions did. You will need to setup an etcd server for StorageOS to use prior to installation of StorageOS. Please see our etcd Operations page for more information on how to install and configure etcd.\n ","excerpt":"etcd is an open-source distributed, strongly consistent key value store that is used by StorageOS to …","ref":"/v2/docs/concepts/etcd/","title":"Etcd"},{"body":"StorageOS uses etcd to store cluster metadata. Because of the strong consistency model that etcd enforces, StorageOS metadata operations are guaranteed to be atomic and consistent.\nInstallation options Before installing StorageOS, an etcd cluster needs to be prepared. There are different topologies that fulfil this prerequisite.\n External etcd (Production) etcd as Pods (Testing) Production Testing External Etcd The production topology is designed to provide the highest stability for the etcd cluster. It is necessary for normal StorageOS function to have a reliable metadata cluster. Otherwise, central operations such as provisioning, attachment or failover of volumes cannot be performed. In the event that etcd becomes unavailable, StorageOS clusters become read only, allowing access to data but preventing metadata changes.\nIt is recommended to install etcd out of the scope of the orchestrator wherever possible. Following CoreOS best practices, a minimum of 3 independent nodes should be dedicated to etcd. StorageOS doesn\u0026rsquo;t require a high performance etcd cluster as the throughput of metadata to the cluster is low. Depending on the level of redundancy you feel comfortable with you can install etcd on the Kubernetes Master nodes. Take extreme care to avoid collisions of the StorageOS etcd installation with the Kubernetes etcd when using the Kubernetes Master nodes. Precautions such as changing the default configuration for the client and peer ports, and ensuring the etcd data directory is modified. The ansible playbook below will default the etcd installation directory to /var/lib/storageos-etcd.\nInstallation If you are familiar with etcd, you can proceed with the CoreOS instructions to install etcd, otherwise this section lays out out an example installation using Ansible.\n Clone StorageOS Helper repository\ngit clone https://github.com/storageos/deploy.git cd k8s/deploy-storageos/etcd-helpers/etcd-ansible-systemd Edit the inventory file\n Target the nodes that install etcd, where the file hosts.example serves as an example. The ip parameter is needed for each node.\n $ cat hosts.example [nodes] centos-1 ip=172.28.128.14 centos-2 ip=172.28.128.15 centos-3 ip=172.28.128.16 # Edit the inventory file $ vi hosts.example # Or your own inventory file Edit the etcd configuration\n If targeting Kubernetes Master nodes, you must change etcd_port_client, etcd_port_peers\n $ cat group_vars/all etcd_version: \u0026#34;3.3.18\u0026#34; etcd_port_client: \u0026#34;2379\u0026#34; etcd_port_peers: \u0026#34;2380\u0026#34; etcd_quota_bytes: 8589934592 # 8 GB etcd_auto_compaction_mode: \u0026#34;revision\u0026#34; etcd_auto_compaction_retention: \u0026#34;100\u0026#34; members: \u0026#34;{{ groups[\u0026#39;nodes\u0026#39;] }}\u0026#34; installation_dir: \u0026#34;/var/lib/storageos-etcd\u0026#34; $ vi group_vars/all Install\nansible-playbook -i hosts.example site.yml Verify installation\n The playbook installs the etcdctl binary on the nodes, at /usr/local/bin.\n $ ssh $NODE # Any node running the new etcd $ ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 member list 66946cff1224bb5, started, etcd-b94bqkb9rf, http://172.28.0.1:2380, http://172.28.0.1:2379 17e7256953f9319b, started, etcd-gjr25s4sdr, http://172.28.0.2:2380, http://172.28.0.2:2379 8b698843a4658823, started, etcd-rqdf9thx5p, http://172.28.0.3:2380, http://172.28.0.3:2379 Managed Services When running StorageOS on Managed Kubernetes services it may not be possible to deploy with the Production etcd topology described above. It is therefore recommended to deploy etcd on its own as much as possible, even if that means deploying 3 independent VMs for etcd to run on.\nAs managed services treat nodes as ephemeral resources, if the orchestration deletes the 3 nodes hosting etcd, the result will be catastrophic and a restore from a backup will be needed.\nIf it is not possible to deploy independent VMs for etcd, etcd can be deployed as pods, inside the cluster. This configuration requires an awareness of the stability that etcd requires. You can use the etcd-as-pods installation option, but be aware of the precautions that need to be taken.\nWhy External Etcd etcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. In the case that quorum is lost, an etcd node stops and marks its contents as read-only. It cannot guarantee that the data being held is valid. Another peer might have a newer version that has not been delivered. Quorum is fundamental for etcd operations.\nIn a Kubernetes environment, applications are scheduled across and in some scenarios such as \u0026ldquo;DiskPressure\u0026rdquo; they may need to be evicted from a node, and be scheduled onto a different node. With an application such as etcd, the scenario described can result in quorum being lost, making the cluster unable to recover automatically. Usually a 3 node etcd cluster can survive losing one node and recover. However, losing a second node at the same time or even having a network partition between them will result in quorum lost.\nBind Etcd IPs to Kubernetes Service Kubernetes external services use a DNS name to reference external endpoints. You can use the example from the helper github repository to deploy the external Service. That might be of use when monitoring etcd from Prometheus.\n Etcd as Pods etcd can be deployed in Kubernetes using the official etcd-operator.\nDeploying etcd in Kubernetes makes the etcd installation very easy, however be aware that even though the official etcd-operator is maintained by RedHat, it hasn\u0026rsquo;t been under active development since 2019. As such it may be considered an archived project. For an actively maintained etcd Operator you might want to check the Improbable etcd Operator.\nExamples of deploying etcd clusters using the etcd-operator on Kubernetes and OpenShift are available.\nSince Kubernetes 1.16 the deployment api uses \u0026ldquo;apps/v1\u0026rdquo;. Once you have cloned the coreos etcd operator repository, you will need to change the apiVersion of the file \u0026ldquo;examples/deployment.yaml\u0026rdquo; from extensions/v1beta1 to apps/v1.\nThe official etcd-operator repository also has a backup deployment operator that can help backup etcd data. Make sure you take frequent backups of the etcd cluster as it holds all the StorageOS cluster metadata.\nKnown etcd-operator issues This topology is only recommended for deployments where isolated nodes cannot be used.\netcd is a distributed key-value store database focused on strong consistency. That means that etcd nodes perform operations across the cluster to ensure quorum. If quorum is lost, etcd nodes stop and etcd marks its contents as read-only. This is because it cannot guarantee that new data will be valid. Quorum is fundamental for etcd operations. When running etcd in pods it is therefore important to consider that a loss of quorum could arise from etcd pods being evicted from nodes.\nOperations such as Kubernetes Upgrades with rolling node pools could cause a total failure of the etcd cluster as nodes are discarded in favor of new ones.\nA 3 etcd node cluster can survive losing one node and recover, a 5 node cluster can survive the loss of two nodes. Loss of further nodes will result in quorum being lost.\nThe etcd-operator doesn\u0026rsquo;t support a full stop of the cluster. Stopping the etcd cluster is not possible unless a backup is restored.\n StorageOS and Etcd When installing StorageOS, the etcd endpoints are passed in a StorageOSCluster Custom Resource.\nFor instance:\napiVersion: \u0026quot;storageos.com/v1\u0026quot; kind: StorageOSCluster metadata: name: \u0026quot;storageos\u0026quot; spec: secretRefName: \u0026quot;storageos-api\u0026quot; # Reference from the Secret created in the previous step secretRefNamespace: \u0026quot;default\u0026quot; # Namespace of the Secret (...) kvBackend: address: 'storageos-etcd-client.etcd:2379' # Example address, change for your etcd endpoint #address: '10.42.15.23:2379,10.42.12.22:2379,10.42.13.16:2379' # You can set etcd server ips backend: 'etcd' Note the kvBackend.address section.\n For full Custom Resource documentation check StorageOSCluster resource definition.\nBest practices StorageOS uses etcd as a service, whether it is deployed following the above instructions or as a custom installation. It is expected that the user maintains the availability and integrity of the etcd cluster.\nIt is highly recommended to keep the cluster backed up and ensure high availability of its data. It is also important to keep the latency between StorageOS nodes and the etcd replicas low. Deploying an etcd cluster in a different data center or region can make StorageOS detect etcd nodes as unavailable due to latency. A 10ms latency between StorageOS and etcd would be the maximum threshold for proper functioning of the system.\nMonitoring It is highly recommended to add monitoring to the etcd cluster. etcd serves Prometheus metrics on the client port http://etc-url:2379/metrics.\nYou can use StorageOS developed Grafana Dashboards for etcd. When using etcd for production, you can use the etcd-cluster-as-service, while the etcd-cluster-as-pod can be used when using etcd from the operator.\nDefragmentation etcd uses revisions to store multiple versions of keys. Compaction removes all key revision prior to a certain revision from etcd. Typically the etcd configuration enables the automatic compaction of keys to prevent performance degradation and limit the storage required. Compaction of revisions can create fragmentation that means space on disk is available for use by etcd but is unavailable for use by the file system. In order to reclaim this space, etcd can be defragmented.\nReclaiming space is important because when the etcd database file grows over the \u0026ldquo;DB_BACKEND_BYTES\u0026rdquo; parameter, the cluster triggers an alarm and sets itself read only and only allows reads and deletes. To avoid hitting the db backend bytes limit, compaction and defragmentation are required. How often defragmentation is required depends on the churn of key revisions in etcd.\nThe Grafana Dashboards mentioned above indicate when nodes require defragmentation. Be aware that defragmentation is a blocking operation that is performed per node, hence the etcd node will be locked for the duration of the defragmentation. Defragmentation usually takes a few milliseconds to complete.\n","excerpt":"StorageOS uses etcd to store cluster metadata. Because of the strong consistency model that etcd …","ref":"/v2/docs/operations/external-etcd/","title":"Etcd"},{"body":"StorageOS requires an etcd cluster in order to function.\nFor more information on why etcd is required please see our etcd concepts page and for production recommendations for etcd installations please see our Etcd Operations page.\n N.B. StorageOS does not recommend using the Kubernetes etcd cluster for StorageOS installations\n Using Etcd with StorageOS During installation of StorageOS the kvBackend parameters of the StorageOS operator are used to specify the address of the etcd cluster. See the StorageOS cluster operator configuration examples for more information.\n","excerpt":"StorageOS requires an etcd cluster in order to function.\nFor more information on why etcd is …","ref":"/v2/docs/prerequisites/etcd/","title":"Etcd"},{"body":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. StorageOS implements dynamic provisioning, so the creation of a PVC will automatically provision a PV (PersistentVolume) that can be used to persist data written by a Pod.\nCreate the PersistentVolumeClaim You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic PVC definition\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;fast\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC will dynamically provision a 5GB volume using the fast StorageClass. This StorageClass was created during the StorageOS install and triggers creation of a PeristentVolume by StorageOS.\nFor installations with CSI, you can create multiple StorageClasses in order to specify default labels.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels. For simplicities sake this example will use unreplicated volumes.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# Reference to the StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1labels:storageos.com/replicas:\u0026#34;1\u0026#34;spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:5GiThe above PVC has the storageos.com/replicas label set. This label tells StorageOS to add a replica for the volume that is created. For the sake of keeping this example simple an unreplicated volume will be used.\n Move into the examples folder and create a PVC using the PVC definition above.\n$ # from storageos-usecases/00-basic $ kubectl create -f ./pvc-basic.yaml You can view the PVC that you have created with the command below\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-1 Bound pvc-f8ffa027-e821-11e8-bc0b-0ac77ccc61fa 5Gi RWO fast 1m Create a pod that mounts the PVC created in step 2.\n$ kubectl create -f ./pod.yaml The command above creates a Pod that uses the PVC that was created in step 1.\napiVersion:v1kind:Podmetadata:name:d1spec:containers:- name:debianimage:debian:9-slimcommand:[\u0026#34;/bin/sleep\u0026#34;]args:[\u0026#34;3600\u0026#34;]volumeMounts:- mountPath:/mntname:v1volumes:- name:v1persistentVolumeClaim:claimName:pvc-1In the Pod definition above volume v1 references the PVC created in step 2, and is mounted in the pod at /mnt. In this example a debian image is used for the container but any container image with a shell would work for this example.\n Confirm that the pod is up and running\n$ kubectl get pods NAME READY STATUS RESTARTS AGE d1 1/1 Running 0 1m Execute a shell inside the container and write some contents to a file\n$ kubectl exec -it d1 -- bash root@d1:/# echo \u0026#34;Hello World!\u0026#34; \u0026gt; /mnt/helloworld root@d1:/# cat /mnt/helloworld Hello World! By writing to /mnt inside the container, the StorageOS volume created by the PVC is being written to. If you were to kill the pod and start it again on a new node, the helloworld file would still be avaliable.\nIf you wish to see more use cases with actual applications please see our Use Cases documentation.\n ","excerpt":"Follow the recipes on this page to create your first PVC (Persistent Volume Claim) using StorageOS. …","ref":"/v2/docs/operations/firstpvc/","title":"StorageOS Volume Guide"},{"body":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available Commands: cluster Fetch cluster-wide configuration details diagnostics Fetch a cluster diagnostic bundle namespace Retrieve basic details of cluster namespaces node Retrieve basic details of nodes in the cluster user Fetch user details volume Retrieve basic details of volumes Flags: -h, --help help for get Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Use \u0026#34;storageos get [command] --help\u0026#34; for more information about a command. get cluster $ storageos get cluster --help Fetch cluster-wide configuration details Usage: storageos get cluster [flags] Examples: $ storageos get cluster Flags: -h, --help help for cluster get diagnostics $ storageos get diagnostics --help Fetch a cluster diagnostic bundle Usage: storageos get diagnostics [flags] Examples: $ storageos get diagnostics $ storageos get diagnostics --output-file ~/my-diagnostics Flags: -h, --help help for diagnostics --output-file string writes the generated diagnostic bundle to a specified file path get namespace $ storageos get namespace -h Retrieve basic details of cluster namespaces Usage: storageos get namespace [namespace names...] [flags] Aliases: namespace, namespaces Examples: $ storageos get namespaces $ storageos get namespace my-namespace-name Flags: -h, --help help for namespace -l, --selector stringArray filter returned results by a set of comma-separated label selectors get node $ storageos get node -h Retrieve basic details of nodes in the cluster Usage: storageos get node [node names...] [flags] Aliases: node, nodes Examples: $ storageos get node my-node-name Flags: -h, --help help for node -l, --selector stringArray filter returned results by a set of comma-separated label selectors get user $ storageos get user -h Fetch user details Usage: storageos get user [user names...] [flags] Aliases: user, users Examples: $ storageos get user my-username $ storageos get user my-username-1 my-username-2 $ storageos get user --use-ids my-userid $ storageos get user --use-ids my-userid-1 my-userid-2 Flags: -h, --help help for user get volume $ storageos get volume -h Retrieve basic details of volumes Usage: storageos get volume [volume names...] [flags] Aliases: volume, volumes Examples: $ storageos get volumes --all-namespaces $ storageos get volume --namespace my-namespace-name my-volume-name Flags: --all-namespaces retrieves volumes from all accessible namespaces. This option overrides the namespace configuration -h, --help help for volume -l, --selector stringArray filter returned results by a set of comma-separated label selectors ","excerpt":"$ storageos get --help Fetch basic details for resources Usage: storageos get [command] Available …","ref":"/v2/docs/reference/cli/get/","title":"Get"},{"body":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any of the nodes in the cluster. Initially you can log in as the default administrator, with the username and password from the storageos-api Secret. By default storageos, storageos.\n You can access the GUI by either port-forwarding with kubectl or using an Ingress rule. i.e kubectl -n kube-system port-forward svc/storageos 5705.\n Nodes Volumes You can create volumes, including replicated volumes, and view volume details:\nLicensing Cluster info Namespaces Volumes can be namespaced across different projects or teams, and you can switch namespace using the left hand panel:\n","excerpt":"StorageOS provides a GUI for cluster and volume management.\nThe GUI is available at port 5705 on any …","ref":"/v2/docs/reference/gui/","title":"Graphical user interface (GUI)"},{"body":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the status of nodes in the cluster.\n$ storageos get nodes NAME HEALTH AGE LABELS node1 online 44 minutes ago node2 online 44 minutes ago node3 online 44 minutes ago node4 online 44 minutes ago node5 online 44 minutes ago ","excerpt":"Various tools are available for checking on the status of a cluster.\nThe StorageOS CLI displays the …","ref":"/v2/docs/operations/health/","title":"Cluster health"},{"body":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to the terms of the the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://storageos.com/legal/#eusa To be notified about stable releases and latest features, sign up at https://my.storageos.com. Usage: storageos [command] Available Commands: apply Make changes to existing resources attach Attach a volume to a node create Create new resources delete Delete resources in the cluster describe Fetch extended details for resources detach Detach a volume from its current location get Fetch basic details for resources help Help about any command version View version information for the StorageOS CLI Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -h, --help help for storageos -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) Additional help topics: storageos env View documentation for configuration settings which can be set in the environment storageos exitcodes View documentation for the exit codes used by the StorageOS CLI Use \u0026#34;storageos [command] --help\u0026#34; for more information about a command. ","excerpt":"$ storageos help Storage for Cloud Native Applications. By using this product, you are agreeing to …","ref":"/v2/docs/reference/cli/help/","title":"Help"},{"body":"StorageOS uses the storage available on the nodes where it is installed to present as available for volumes.\nIn order to mitigate against problems caused by filling the host root disk, we recommend mounting a separate device into the /var/lib/storageos directory. StorageOS is agnostic to the type of filesystem mounted in /var/lib/storageos.\nExtending Available Storage StorageOS uses subdirectories of /var/lib/storageos/data to hold user data. By default, the directory /var/lib/storageos/data/dev1 will be created when a node is bootstrapped, and used for pool data. It is possible to shard the data by creating more directories into this structure. StorageOS will save data in any directory that conforms to the pattern /var/lib/storageos/data/dev[0-9]+, such as /var/lib/storageos/data/dev2 or /var/lib/storageos/data/dev5. This functionality enables operators to mount different devices into devX directories and StorageOS will recognise them as available storage automatically.\nThere are two possible options to expand the available disk space for StorageOS to allocate:\n Mount filesystem in /var/lib/storageos/data/devX Use LVM to expand the logical volume available to StorageOS Option 1: Mount Additional Devices This option enables operators to expand the cluster\u0026rsquo;s available space at any time without having to stop applications or forcing operational downtime. The expansion of disk is transparent for applications and StorageOS Volumes. StorageOS will use the new available space to create new data files.\n Context\nWe assume that there is a disk available in our Linux system without formatting in addition to the root filesystem. StorageOS data dir dev1 (/var/lib/storageos/data/dev1) is using /dev/xvda1. We will use the device /dev/xvdf to expand StorageOS available space.\nList available block devices in the host.\nroot@node0:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Format device\nroot@node0:/var/lib/storageos/data# mkfs -t ext4 /dev/xvdf mke2fs 1.42.12 (29-Aug-2014) Creating filesystem with 26214400 4k blocks and 6553600 inodes Filesystem UUID: 380712fa-6f82-477a-81a5-d7466d4c6b7f Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 4096000, 7962624, 11239424, 20480000, 23887872 Allocating group tables: done Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done Mount filesystem\nroot@node0:~# mkdir -p /var/lib/storageos/data/dev2 root@node0:~# mount /dev/xvdf /var/lib/storageos/data/dev2 Verify available storage\nIn less than 30 seconds, StorageOS will see the new available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 244,491,013,324 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 137,438,953,472 Note that the node node0 has increased the TOTAL capacity in 100Gi.\n Persist the mount at boot by adding the mount endpoint to /etc/fstab\n Option 2: Expand Existing Devices Backed by LVM This option enables operators to take advantage of LVM to manage disks.\n Context\nWe assume that /var/lib/storageos is mounted onto an LVM volume. We are using a volumegroup named storageos and logical volume called data. There is a second physical disk /dev/xvdg unused.\nList available block devices in the host.\nroot@node2:~# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT xvda 202:0 0 128G 0 disk `-xvda1 202:1 0 128G 0 part / xvdf 202:80 0 100G 0 disk `-storageos-data 254:0 0 99G 0 lvm /var/lib/storageos xvdg 202:96 0 100G 0 disk Check StorageOS cluster\u0026rsquo;s available capacity.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 107,696,304,947 # --\u0026gt; LVM storageos/data volume Add physical disk to LVM\nroot@node2:~# vgextend storageos /dev/xvdg Volume group \u0026quot;storageos\u0026quot; successfully extended The volume group storageos must have 2 physical volumes (#PV)\nroot@node2:~# vgs VG #PV #LV #SN Attr VSize VFree storageos 2 1 0 wz--n- 199.99g 104.99g Extend logical volume data\nroot@node2:~# lvextend -L+100G /dev/storageos/data Size of logical volume storageos/data changed from 95.00 GiB (24320 extents) to 195.00 GiB (49920 extents). Logical volume data successfully resized Resize the FileSystem\n Your filesystem must support the option to be expanded, and to do so while in use. Otherwise, you need to unmount first.\n root@node2:~# resize2fs /dev/storageos/data resize2fs 1.42.12 (29-Aug-2014) Filesystem at /dev/storageos/data is mounted on /var/lib/storageos; on-line resizing required old_desc_blocks = 6, new_desc_blocks = 13 The filesystem on /dev/storageos/data is now 51118080 (4k) blocks long. Check new available space\nThe mounted file system to /var/lib/storageos has increased its size.\nroot@node2:~# df -h /dev/mapper/storageos-data Filesystem Size Used Avail Use% Mounted on /dev/mapper/storageos-data 192G 60M 183G 1% /var/lib/storageos StorageOS available storage has increased too.\n$ storageos get node -ojson | jq -r '.[] | { name: .name, capacity: .capacity.total }' \u0026quot;node0\u0026quot; 137,438,953,472 \u0026quot;node1\u0026quot; 137,438,953,472 \u0026quot;node2\u0026quot; 206,158,430,208 # --\u0026gt; 100G more available Persist the mount at boot by adding the mount point to /etc/fstab\n ","excerpt":"StorageOS uses the storage available on the nodes where it is installed to present as available for …","ref":"/v2/docs/operations/managing-host-storage/","title":"Managing Host Storage"},{"body":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init container that sets the system configuration for StorageOS. The container also manages configuration changes required when upgrading StorageOS versions.\nThe container belongs to the DaemonSet that the StorageOS Cluster Operator starts when a StorageOSCluster resource is created. The storageos-init container is executed as an initContainer as part of a Kubernetes Pod. Therefore, only successful execution of the storageos-init container processes will result in the main container starting.\nScript Framework The code responsible for fulfilling the requirements is based on a Script Framework.\nThe script framework executes a set of scripts, performing checks, verifications and other procedures needed for StorageOS to be able to start. The scripts stdout and stderr are written to the stdout and stderr of the init app. The container logs contain all the logs of the individual scripts that run. The exit statuses of the scripts are used to determine initialization failure or success. Any non-zero exit status is logged as an event in the Kubernetes Pod events.\nIf any of the scripts fail, the storageos-init container will propagate the failure to Kubernetes, showing the status of the Pod as Init:Err.\nTo view the output of all storageos-init containers the following command can be used:\nkubectl -n kube-system logs -l app=storageos,kind=daemonset -c storageos-init For more details, check the StorageOS init container project.\nScripts executed The storageos-init container executes the following scripts.\n enable-lio dbupgrade ","excerpt":"StorageOS has requirements for the configuration of host systems. As such, StorageOS starts an init …","ref":"/v2/docs/reference/init-container/","title":"Init container"},{"body":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a StorageOS persistent volume being mounted on /var/jenkins_home. Deploying Jenkins using StorageOS offers multiple benefits. Firstly Jenkins can spin up multiple build pods at once to allow concurrent builds of different projects. Secondly Jenkins configuration is on a PersistentVolume so even if the Jenkins pod is rescheduled the configuration will persist.\nUsing StorageOS volume replicas allows for failure of nodes holding the PersistentVolume without interrupting Jenkins. Lastly by enabling StorageOS fencing Jenkins time to recover, in case of node failures, is greatly reduced.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information\nDeploying Jenkins on Kubernetes You can find the latest files in the StorageOS example deployment repository.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases $ kubectl create -f ./jenkins Confirm that Jenkins is up and running\n$ kubectl get pods -w -l app=jenkins NAME READY STATUS RESTARTS AGE jenkins-0 1/1 Running 0 1m Connect to the Jenkins UI through the Jenkins service.\nYou can do this by port forwarding the Jenkins Kubernetes service to your localhost and accessing the UI via your browser. Alternatively if you have network access to your Kubernetes nodes then you can create a NodePort service and access Jenkins like that. A NodePort service has been left in 10-service.yaml commented out.\nTo port-foward the Jenkins service use the following command.\n$ kubectl port-foward svc/jenkins 8080 To login to the Jenkins UI use the credentials specified in 07-config.yaml, unless these have been changed from the defaults the username/password is admin/password.\n Create a Jenkins job.\nOnce you are logged into the UI you can create a job that will be farmed out to a Kubernetes plugin build agent. Click New Item, enter a name for the project and select Freestyle project. Next add an Execute shell build step. As a proof of concept you can use the bash below to have the pod execute a sleep.\n#!/bin/bash sleep 1000 Save the project and select Schedule a build of your project. You can watch for the appearance of a build pod using kubectl get pods -l jenkins=agent -w. Once the pod is created you should see the Build Executor status in the Jenkins UI display the pod.\nTo see multiple projects being built at once create another project and try scheduling a build of both projects at the same time.\n ","excerpt":"Jenkins with StorageOS This example shows an example of how to deploy Jenkins on Kubernetes with a …","ref":"/v2/docs/usecases/jenkins/","title":"Jenkins"},{"body":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from pub/sub and traditional queues.\nUsing StorageOS persistent volumes with Apache Kafka means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the kafka pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nKafka has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Kafka to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Apache Zookeeper is required by Kafka to function; we assume it to already exist and be accessible within the Kubernetes cluster as zookeeper, see how to run Zookeeper with StorageOS here StorageOS is assumed to have been installed; please check for the latest available version here Kafka pods require 1536 MB of memory for successful scheduling Helm To simplify the deployment of kafka, we\u0026rsquo;ve used this Kafka helm chart (incubator) (version 0.13.8, app version 5.0.1) and rendered it into the example deployment files you can find in our GitHub repo.\nClone the use cases repo You can find the latest files in the StorageOS use cases repository in /kafka/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases StatefulSet definition\n---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:kafkalabels:app:kafka...spec:serviceName:kafka-headlesspodManagementPolicy:OrderedReadyupdateStrategy:type:OnDeletereplicas:3# \u0026lt;--- number of kafa pods to runtemplate:...spec:serviceAccountName:kafkacontainers:...- name:kafka-brokerimage:\u0026#34;confluentinc/cp-kafka:5.0.1\u0026#34;imagePullPolicy:\u0026#34;IfNotPresent\u0026#34;...volumeMounts:- name:datadirmountPath:\u0026#34;/var/data\u0026#34;volumes:- name:jmx-configconfigMap:name:kafka-metricsterminationGracePeriodSeconds:60volumeClaimTemplates:- metadata:name:datadirspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]resources:requests:storage:10Gi# \u0026lt;--- storage requested for each podstorageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class.\nDynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\n Create the kubernetes objects\nkubectl apply -f ./kafka/ Confirm kafka is up and running\n$ kubectl get pods -l app=kafka NAME READY STATUS RESTARTS AGE kafka-0 2/2 Running 0 10m kafka-1 2/2 Running 0 9m26s kafka-2 2/2 Running 0 7m59s Connect to kafka\nConnect to the kafka test client pod and send some test data to kafka through its service endpoint\n Connect to the pod\nkubectl exec -it kafka-test-client /bin/bash Create a topic\n/usr/bin/kafka-topics --zookeeper zookeeper:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1 Send some test data\n/usr/bin/kafka-run-class org.apache.kafka.tools.ProducerPerformance --topic test-rep-one --num-records 5000 --record-size 100 --throughput -1 --print-metrics --producer-props acks=1 bootstrap.servers=kafka:9092 buffer.memory=67108864 batch.size=8196 ","excerpt":"Apache Kafka with StorageOS Kafka is a popular stream processing platform combining features from …","ref":"/v2/docs/usecases/kafka/","title":"Kafka"},{"body":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual machines (VMs) in Kubernetes pods.\nDeploying Kubevirt using StorageOS offers multiple benefits. Kubevirt can spin up VMs as Kubernetes pods, using images on StorageOS persistent volumes. Doing this allows the VM data to persist through restarts and rescheduling. Using StorageOS volume replicas also allows for failure of nodes holding the PersistentVolume without interrupting the VM running off the PersistentVolume. Containerized Data Importer (CDI) can also be used to prepare StorageOS volumes with disk images in an automated fashion. Simply by declaring that a VirtualMachine will use a DataVolume and providing the disk image URL, a StorageOS volume can be dynamically provisioned and automatically prepared with the disk image.\nThis usecase will guide you through installing KubeVirt and CDI on your Kubernetes cluster, and create a VM. By the end of the guide you\u0026rsquo;ll be able to launch a shell inside the KubeVirt VM that\u0026rsquo;s running as a Kubernetes pod.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nPrerequisites Please ensure you have met the Kubevirt prerequisites, please see the Kubevirt installation instructions for more information.\nAs part of this installation it is assumed that you are running a Kubernetes cluster on VMs. As such nested virtualization or hardware emulation need to be enabled.\nDeploying KubeVirt on Kubernetes For ease of installation we have enabled hardware emulation. If your VMs support nested virtualization then edit the Kubevirt ConfigMap ./kubevirt-install/10-cm.yaml, removing the line debug.useEmulation: \u0026quot;true\u0026quot;.\n In order to deploy Kubevirt you just need to clone this repository and use kubectl to create the Kubernetes objects.\n$ git clone https://github.com/storageos/use-cases.git storageos-usecases $ cd storageos-usecases/kubevirt $ kubectl create -f ./kubevirt-install Check that the Kubevirt pods are running.\n$ kubectl get pods -w -n kubevirt NAME READY STATUS RESTARTS AGE virt-api-57546d479b-p26d4 1/1 Running 0 1m virt-api-57546d479b-zs5dw 1/1 Running 0 1m virt-controller-56b5498854-7xsfz 1/1 Running 1 1m virt-controller-56b5498854-bz559 1/1 Running 1 1m virt-handler-6z4kq 1/1 Running 0 1m virt-handler-7szhl 1/1 Running 0 1m virt-handler-jmm6w 1/1 Running 0 1m virt-operator-79c9bdd859-8xq98 1/1 Running 0 1m virt-operator-79c9bdd859-kfjz6 1/1 Running 0 1m Once Kubevirt is running install CDI.\n$ kubectl create -f ./cdi Check that the CDI pods are running correctly.\n$ kubectl get pods -n cdi NAME READY STATUS RESTARTS AGE cdi-apiserver-8668f888df-s6pp4 1/1 Running 0 1m cdi-deployment-5cf794896b-whh4j 1/1 Running 0 1m cdi-operator-5887f96c-dz2hg 1/1 Running 0 1m cdi-uploadproxy-97fbbfcbf-6f9xs 1/1 Running 0 1m Now that CDI and Kubevirt are running, VMs can be created. In this example VMs running Cirros, a small and lightweight OS, will be created. The vm-cirros.yaml manifest creates a VirtualMachine that uses a DataVolume. This means that CDI will create a StorageOS backed PVC and download the image that the VirtualMachineInstance (VMI) will boot from onto the PVC.\n$ kubectl create -f ./vm-cirros.yaml Check that the VMI is running. Note that the VMI will only be created after CDI has downloaded the Cirros disk image onto a StorageOS persistent volume so depending on your connection speed this may take some time.\n$ kubectl get vmi NAME AGE PHASE IP NODENAME cirros 1m Running 10.244.2.12 ip-10-1-10-154.storageos.net $ kubectl get pods NAME READY STATUS RESTARTS AGE virt-launcher-cirros-drqhr 1/1 Running 0 1m Connect to the VM console.\nThis example uses the virtctl kubectl plugin in order to connect to the VMs console. The escape sequence ^] is ctrl + ]\n$ kubectl virt console cirros Successfully connected to cirros console. The escape sequence is ^] login as \u0026#39;cirros\u0026#39; user. default password: \u0026#39;gocubsgo\u0026#39;. use \u0026#39;sudo\u0026#39; for root. cirros login: cirros Password: $ Cloning Volumes CDI allows for images to be cloned using a DataVolume manifest. Verify that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n N.B. Ensure that the VMI is stopped before continuing!\n Verify that the VMI is stopped before continuing, and that the cirros pvc, created as part of the vm-cirros.yaml file, exists before attempting to clone the volume.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE cirros Bound pvc-f4833060-5a77-420c-927e-6bc518d9df3c 12Gi RWO fast 1m Once the PVC\u0026rsquo;s existence is confirmed then create a new DataVolume that uses the cirros PVC as its source.\n$ kubectl create -f ./cloned.yaml Watch as the CDI pods are created.\n$ kubectl get pods -w You\u0026rsquo;ll see that a cdi-upload-cloned-datavolume pod is created and then a cdi-clone-source pod is created. The cdi-source pod mounts the original cirros volume and sends the contents of the volume to the cdi-upload pod. The cdi-upload pod creates and mounts a new PVC and writes the contents of the original volume to it.\n ","excerpt":"Kubevirt with StorageOS Kubevirt is a CNCF sandbox project that allows the running of virtual …","ref":"/v2/docs/usecases/kubevirt/","title":"Kubevirt"},{"body":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to various StorageOS artefacts. Applying specific feature labels triggers compression, replication and other storage features. No feature labels are present by default.\nStorageOS Node labels Nodes do not have any feature labels present by default. When StorageOS is run within Kubernetes, the Cluster Operator syncs any node labels set on Kubernetes nodes within StorageOS. Node labels may also be set with the CLI or UI.\n Feature Label Values Description Compute only storageos.com/computeonly true / false Specifies whether a node should be computeonly where it only acts as a client and does not host volume data locally, otherwise the node is hyperconverged (the default), where the node can operate in both client and server modes. You can set the computeonly label on the Kubernetes node and the label will be sync\u0026rsquo;d to the StorageOS node (labels take an eventual consistency reconciliation time of ~1min).\nkubectl label node $NODE storageos.com/computeonly=true StorageOS Volume labels Volumes do not have any feature labels present by default. WARNING: The caching and compression labels can only apply during provision time, they can\u0026rsquo;t be changed during execution.\n Feature Label Values Description Caching storageos.com/nocache true / false Switches off caching. Compression storageos.com/nocompress true / false Switches off compression of data at rest and in transit. Replication storageos.com/replicas integers [0, 5] Replicates entire volume across nodes. Typically 1 replica is sufficient (2 copies of the data); more than 2 replicas is not recommended. Once applied, this setting can only be changed in the StorageOS UI or CLI. To create a volume with a feature label:\n Option 1: PVC Label\nAdd the label in the PVC definition, for instance:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:pvc-3labels:storageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;-----spec:storageClassName:\u0026#34;fast\u0026#34;accessModes:- ReadWriteOnceresources:requests:storage:1G Option 2: Set label in the StorageClass\nAny PVC using the StorageClass inherits the label. The PVC label takes precedence over the StorageClass parameters.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Label \u0026lt;--------provisioner:storageos# CSI driver (recommended) Option 3:\nOnce a PVC is created, you can update the Labels in StorageOS both in the UI or CLI. Those labels are going to be visible only for StorageOS and will not be synced to the Kubernetes resource.\n ","excerpt":"Feature labels are a powerful and flexible way to control storage features.\nLabels can be applied to …","ref":"/v2/docs/reference/labels/","title":"StorageOS Feature labels"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying a Free Developer licence.\nTo learn how to apply a licence to your cluster, check the operations licensing page.\nFree Developer licence The developer licence only requires the user to register and issue a licence through StorageOS.\nThe developer licence is free and grants a licence for a 5 node StorageOS cluster with 5TB of provisioned capacity. It is designed to enable basic cloud native workflows in Kubernetes that require the persistence of stateful application data. Dynamic provisioning, distributed access to data and high availability of volumes through synchronous replication and automatic failover are some of the features that are available under the developer licence.\nThe free developer licence includes community support through our public Slack channel..\nCommercial licences For information on our commercial offerings, including support, please contact sales@storageos.com.\nNote about capacity limits Some StorageOS licences have limits on capacity. StorageOS allows provisioning volumes until the limit of the licence is reached. Only the size of the volume requested by the Persistent Volume Claim counts for the licence limit, regardless whether of whether the volume has replication enabled.\nOnce the licence limit is reached, new volumes are not able to provision unless provisioned capacity is released, i.e deleting volumes. That behaviour is not tight to the capacity of the backend disks on your nodes.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2/docs/reference/licence/","title":"Licence"},{"body":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 hours. After that, new operations such as volume provisioning or adding nodes are not permitted. Normal functioning of the cluster can be unlocked by applying for a Free Developer licence.\nObtaining a Developer licence via the GUI You can obtain and apply a free Developer licence in the StorageOS web GUI by creating or logging in with a StorageOS account on the StorageOS portal via the licence page of the StorageOS web GUI:\n You can access the GUI on any node running the StorageOS Daemonset on the port 5705. Whether using an Ingress route or port-forwarding.\n Wait a few seconds for the licence generation process to complete, at which point your licence will be visible.\nApplying a previously obtained licence via the GUI Commercial licences are delivered through contact with the StorageOS team. To apply such keys, via the web GUI, visit the licence section of the GUI and click on the tab \u0026ldquo;Upgrade\u0026rdquo;, for the specific licence level you purchased. Then paste the licence key and click on \u0026ldquo;UPLOAD KEY TO CLUSTER\u0026rdquo;. Note that you can also view your cluster ID on the same page.\nApplying a licence via the CLI Before getting a licence, you need to know the ID of your StorageOS cluster.\nThis CLI command can print the cluster ID:\n$ storageos get cluster ID: 704dd165-9580-4da4-a554-0acb96d328cb Licence: expiration: 2021-03-25T13:48:46Z (1 year from now) capacity: 5.0 TiB kind: professional customer name: storageos Created at: 2020-03-25T13:48:33Z (1 hour ago) Updated at: 2020-03-25T13:48:46Z (1 hour ago) Given the Cluster ID, the StorageOS team can generate a licence. Once, given the key, you can apply the licence by using the following command.\n$ echo PASTE-THE-LICENCE-KEY-HERE | storageos apply licence --from-stdin Read the licence CLI command reference for further information.\nObtaining an Enterprise licence Please contact sales@storageos.com to discuss pricing for commercial licences.\n","excerpt":"A newly installed StorageOS cluster does not include a licence. A cluster can run unlicensed for 24 …","ref":"/v2/docs/operations/licensing/","title":"Licensing"},{"body":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on linux.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MS SQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:mssqlspec:selector:matchLabels:app:mssqlenv:prodserviceName:mssqlreplicas:1...spec:serviceAccountName:mssql...volumeMounts:- name:datamountPath:/var/opt/mssql...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MS SQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mssql Confirm MS SQL is up and running.\n$ kubectl get pods -w -l app=mssql NAME READY STATUS RESTARTS AGE mssql-0 1/1 Running 0 1m Connect to the MS SQL client pod and connect to the MS SQL server through the service\n$ kubectl exec -it mssql-0 -- /opt/mssql-tools/bin/sqlcmd -S mssql-0.mssql -U SA -P \u0026#39;Password15\u0026#39; 1\u0026gt; USE master; 2\u0026gt; GO Changed database context to \u0026#39;master\u0026#39;. 1\u0026gt; SELECT name, database_id, create_date FROM sys.databases ; 2\u0026gt; GO name database_id create_date --------------------------- ----------- ----------------------- master 1 2003-04-08 09:13:36.390 tempdb 2 2018-11-02 16:30:37.907 model 3 2003-04-08 09:13:36.390 msdb 4 2018-10-19 01:18:57.300 (4 rows affected) Configuration If you need custom startup options, you can edit the ConfigMap file 15-mssql-configmap.yaml with your desired MS SQL configuration settings.\n","excerpt":"MS SQL with StorageOS Beginning with Microsoft SQL Server 2017, Microsoft has supported MSSQL on …","ref":"/v2/docs/usecases/mssql/","title":"MS SQL"},{"body":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With mTLS both StorageOS and etcd authenticate each other ensuring that communication only happens between mutually authenticated end points, and that all communication is encrypted.\nStorageOS uses environment variables to specify the location of the necessary certificates and keys. This allows Kubernetes secrets to be mounted as volumes, directly into pods.\nSetting up mTLS with Etcd Below is an example StorageOSCluster resource that can be used to setup StorageOS with etcd using mTLS.\napiVersion:storageos.com/v1kind:StorageOSClustermetadata:name:storageos-clusternamespace:\u0026#34;default\u0026#34;spec:images:nodeContainer:\u0026#34;storageos/node:v2.0.0\u0026#34;secretRefName:\u0026#34;storageos-api\u0026#34;secretRefNamespace:\u0026#34;default\u0026#34;namespace:\u0026#34;storageos\u0026#34;csi:enable:truedeploymentStrategy:\u0026#34;deployment\u0026#34;enableControllerPublishCreds:trueenableNodePublishCreds:trueenableProvisionCreds:true# External mTLS secured etcd cluster specific propertiestlsEtcdSecretRefName:\u0026#34;etcd-client-tls\u0026#34;# Secret containing etcd client certificatestlsEtcdSecretRefNamespace:\u0026#34;etcd\u0026#34;# Namespace of the client certificates secretkvBackend:address:\u0026#34;https://storageos-etcd-cluster-client.storageos-etcd.svc:2379\u0026#34;# Etcd client service address.backend:\u0026#34;etcd\u0026#34;# Backend typetlsEtcdSecretRefName and tlsEtcdSecretRefNamespace are used to pass a reference to a secret that should contain the following key:values:\n etcd-client-ca.crt - containing the etcd Certificate Authority certificate etcd-client.crt - containing the etcd Client certificate etcd-client.key - cotaining the etcd Client key The StorageOS operator uses the etcd secret that contains the client certificates, to build a secret in the StorageOS installation namespace. This secret contains the certificate filenames and certificate file contents. The StorageOS daemonset that is created by the operator mounts the secret as a volume so that the certificate files are avaliable inside the pod. Environment variables containing the file paths are passed to the StorageOS process in order to use the files from the mounted path.\nA worked example of settings up StorageOS with external etcd using mTLS is avaliable here. For ease of use the example uses the CoreOS etcd operator and the CoreOS guide for Cluster TLS is followed.\n","excerpt":"StorageOS supports secure communication with an external etcd cluster using mutual TLS (mTLS). With …","ref":"/v2/docs/operations/external-etcd/etcd-tls/","title":"Encrypting communication with Etcd"},{"body":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular web-based applications including WordPress.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying MySQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\napiversion:apps/v1kind:statefulsetmetadata:name:mysqlspec:selector:matchlabels:app:mysqlenv:prodservicename:mysqlreplicas:1...spec:serviceaccountname:mysql...volumemounts:- name:datamountpath:/var/lib/mysqlsubpath:mysql- name:confmountpath:/etc/mysql/mysql.conf.d...volumeclaimtemplates:- metadata:name:datalabels:env:prodspec:accessmodes:[\u0026#34;readwriteonce\u0026#34;]storageclassname:\u0026#34;fast\u0026#34;# storageos storageclass resources:requests:storage:5giThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the MySQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./mysql Confirm MySQL is up and running.\n$ kubectl get pods -w -l app=mysql NAME READY STATUS RESTARTS AGE mysql-0 1/1 Running 0 1m Connect to the MySQL client pod and connect to the MySQL server through the service\n$ kubectl exec client -- mysql -h mysql-0.mysql -e \u0026#34;show databases;\u0026#34; Database information_schema mysql performance_schema Configuration If you need custom startup options, you can edit the ConfigMap file 15-mysqld-configmap.yaml with your desired MySQL configuration settings.\n","excerpt":"MySQL with StorageOS MySQL is a popular SQL open source database for a wide range of popular …","ref":"/v2/docs/usecases/mysql/","title":"MySQL"},{"body":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a StorageOS cluster to be used by multiple teams across multiple projects.\nIt is not necessary to create StorageOS namespaces manually, as StorageOS maps Kubernetes namespaces on a one-to-one basis when PersistentVolumeClaims using the StorageOS StorageClass are created.\nAccess to Namespaces is controlled through user or group level policies.\n","excerpt":"StorageOS namespaces are an identical concept to Kubernetes namespaces. They are intended to allow a …","ref":"/v2/docs/concepts/namespaces/","title":"Namespaces"},{"body":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is created by default.\nNamespaces apply to volumes.\nManaging Namespaces In order to create a new namespace navigate to \u0026ldquo;Namespaces\u0026rdquo; in the GUI, and select \u0026ldquo;Create Namespace\u0026rdquo;.\nWhen a Kubernetes PVC is created in a namespace, StorageOS automatically maps the Volume in the same namespace. Namespaces are created by StorageOS to fulfil the RBAC rules enforced by Kubernetes roles.\nIn order to delete a namespace, all volumes must be deleted from the namespace before the namespace can be deleted.\n","excerpt":"Namespaces help different projects or teams share a StorageOS cluster. Only the default namespace is …","ref":"/v2/docs/operations/namespaces/","title":"Namespaces"},{"body":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load balancer or even as a Kubernetes ingress controller.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Nginx on Kubernetes You can find the latest files in the StorageOS use cases repostiory\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet definition\napiVersion:apps/v1kind:StatefulSetmetadata:name:nginxspec:serviceName:nginxspec:serviceAccountName:nginxcontainers:- name:nginximage:nginxports:- containerPort:80volumeMounts:- name:nginx-datamountPath:/usr/share/nginx/htmlsubPath:html- name:nginx-configmountPath:/etc/nginx/conf.dvolumes:- name:nginx-configconfigMap:name:nginxvolumeClaimTemplates:- metadata:name:nginx-dataspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Nginx examples folder and create the objects\ncd storageos-usecases kubectl create -f ./nginx Confirm Nginx is up and running.\n$ kubectl get pods -w -l app=nginx NAME READY STATUS RESTARTS AGE nginx-0 1/1 Running 0 1m Connect to the nginx pod and write a file to /usr/share/nginx/html that Nginx will serve.\n$ kubectl exec nginx-0 -it -- bash root@nginx-0:/# echo Hello world! \u0026gt; /usr/share/nginx/html/greetings.txt Connect to the Busybox pod and connect to the Nginx server through the service and retrieve the directory index from Nginx.\n$ kubectl exec -it busybox -- /bin/sh / # wget -q -O- nginx \u0026lt;html\u0026gt; \u0026lt;head\u0026gt;\u0026lt;title\u0026gt;Index of /\u0026lt;/title\u0026gt;\u0026lt;/head\u0026gt; \u0026lt;body\u0026gt; \u0026lt;h1\u0026gt;Index of /\u0026lt;/h1\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;pre\u0026gt;\u0026lt;a href=\u0026#34;../\u0026#34;\u0026gt;../\u0026lt;/a\u0026gt; \u0026lt;a href=\u0026#34;greetings.txt\u0026#34;\u0026gt;greetings.txt\u0026lt;/a\u0026gt; 27-Feb-2019 12:04 13 \u0026lt;/pre\u0026gt;\u0026lt;hr\u0026gt;\u0026lt;/body\u0026gt; \u0026lt;/html\u0026gt; Retrieve and display the contents of the greetings.txt file\n/ # wget -q -O- nginx/greetings.txt Hello world! Configuration If you need custom startup options, you can edit the ConfigMap file 15-nginx-configmap.yaml with your desired Nginx configuration settings.\n","excerpt":"Nginx with StorageOS Nginx is a popular web server that can be used as a reverse proxy, load …","ref":"/v2/docs/usecases/nginx/","title":"Nginx"},{"body":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A node must be running a daemonset pod in order to consume and/or present storage.\nBy default StorageOS nodes run in hyperconverged mode. This means that the node hosts data from StorageOS volumes and can present volumes to applications.\nAlternatively, a node can run in computeonly mode, which means no storage is consumed on the node itself and the node only presents volumes hosted by other nodes. Volumes presented to applications running on compute only nodes are therefore all remote. Compute only nodes can be very useful for topologies where nodes are ephemeral and should not host data, but the ephemeral nodes host applications that require StorageOS volumes. The nodes that are not intended to hold data, but just to present StorageOS volumes, can be set as computeonly.\nA node can be marked as compute only at any point in time by adding the label storageos.com/computeonly=true, following the labels reference.\n","excerpt":"A StorageOS node is any machine (virtual or physical) that is running the StorageOS daemonset pod. A …","ref":"/v2/docs/concepts/nodes/","title":"Nodes"},{"body":"The following document lists the open source software attributions in the StorageOS Control Plane, Data Plane and CLI.\nCNats The MIT License (MIT) Copyright (c) 2015 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CppUTest Copyright (c) 2007, Michael Feathers, James Grenning and Bas Vodde All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the \u0026lt;organization\u0026gt; nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE EARLIER MENTIONED AUTHORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL \u0026lt;copyright holder\u0026gt; BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Hayai Copyright (c) 2011 - Nick Bruun. This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. If you meet (any of) the author(s), you're encouraged to buy them a beer, a drink or whatever is suited to the situation, given that you like the software. This notice may not be removed or altered from any source distribution. LZ4 LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. LibreSSL LICENSE ISSUES ============== The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the OpenSSL License and the original SSLeay license apply to the toolkit. See below for the actual license texts. Actually both licenses are BSD-style Open Source licenses. In case of any license issues related to OpenSSL please contact openssl-core@openssl.org. OpenSSL License --------------- ==================================================================== Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit. (http://www.openssl.org/)\u0026quot; 4. The names \u0026quot;OpenSSL Toolkit\u0026quot; and \u0026quot;OpenSSL Project\u0026quot; must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact openssl-core@openssl.org. 5. Products derived from this software may not be called \u0026quot;OpenSSL\u0026quot; nor may \u0026quot;OpenSSL\u0026quot; appear in their names without prior written permission of the OpenSSL Project. 6. Redistributions of any form whatsoever must retain the following acknowledgment: \u0026quot;This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)\u0026quot; THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ==================================================================== This product includes cryptographic software written by Eric Young (eay@cryptsoft.com). This product includes software written by Tim Hudson (tjh@cryptsoft.com). Original SSLeay License ----------------------- Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) All rights reserved. This package is an SSL implementation written by Eric Young (eay@cryptsoft.com). The implementation was written so as to conform with Netscapes SSL. This library is free for commercial and non-commercial use as long as the following conditions are aheared to. The following conditions apply to all code found in this distribution, be it the RC4, RSA, lhash, DES, etc., code; not just the SSL code. The SSL documentation included with this distribution is covered by the same copyright terms except that the holder is Tim Hudson (tjh@cryptsoft.com). Copyright remains Eric Young's, and as such any Copyright notices in the code are not to be removed. If this package is used in a product, Eric Young should be given attribution as the author of the parts of the library used. This can be in the form of a textual message at program startup or in documentation (online or textual) provided with the package. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. All advertising materials mentioning features or use of this software must display the following acknowledgement: \u0026quot;This product includes cryptographic software written by Eric Young (eay@cryptsoft.com)\u0026quot; The word 'cryptographic' can be left out if the rouines from the library being used are not cryptographic related :-). 4. If you include any Windows specific code (or a derivative thereof) from the apps directory (application code) you must include an acknowledgement: \u0026quot;This product includes software written by Tim Hudson (tjh@cryptsoft.com)\u0026quot; THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The licence and distribution terms for any publically available version or derivative of this code cannot be changed. i.e. this code cannot simply be copied and put under another distribution licence [including the GNU Public Licence.] protobuf This license applies to all parts of Protocol Buffers except the following: - Atomicops support for generic gcc, located in src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Code generated by the Protocol Buffer compiler is owned by the owner of the input file used when generating it. This code is not standalone and requires a support library to be linked with it. This support library is itself covered by the above license. rocksdb BSD License For rocksdb software Copyright (c) 2011-present, Facebook, Inc. All rights reserved. --------------------------------------------------------------------- Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/armon/go-metrics The MIT License (MIT) Copyright (c) 2013 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/armon/go-radix The MIT License (MIT) Copyright (c) 2014 Armon Dadgar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/bgentry/speakeasy Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \u0026quot;[]\u0026quot; replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \u0026quot;printed page\u0026quot; as the copyright notice for easier identification within third-party archives. Copyright [2013] [the CloudFoundry Authors] Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/dgrijalva/jwt-go Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgrijalva/jwt-go/request Copyright (c) 2012 Dave Grijalva Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/dgryski/go-metro MetroHash This package is a mechanical translation of the reference C++ code for MetroHash, available at https://github.com/jandrewrogers/MetroHash I claim no additional copyright over the original implementation. The MIT License (MIT) Copyright (c) 2015 J. Andrew Rogers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/docker/docker/pkg/mount Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/docker/libkv/store Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \u0026quot;License\u0026quot; shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \u0026quot;Licensor\u0026quot; shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \u0026quot;Legal Entity\u0026quot; shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \u0026quot;control\u0026quot; means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) shall mean an individual or Legal Entity exercising permissions granted by this License. \u0026quot;Source\u0026quot; form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \u0026quot;Object\u0026quot; form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \u0026quot;Work\u0026quot; shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \u0026quot;Derivative Works\u0026quot; shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \u0026quot;Contribution\u0026quot; shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \u0026quot;submitted\u0026quot; means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \u0026quot;Not a Contribution.\u0026quot; \u0026quot;Contributor\u0026quot; shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \u0026quot;NOTICE\u0026quot; text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2014-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/golang/protobuf/proto Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/context Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/handlers Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/mux Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/gorilla/websocket Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/consul/api Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/errwrap Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-cleanhttp Mozilla Public License, version 2.0 1. Definitions 1.1. \u0026quot;Contributor\u0026quot; means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. \u0026quot;Contributor Version\u0026quot; means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. \u0026quot;Contribution\u0026quot; means Covered Software of a particular Contributor. 1.4. \u0026quot;Covered Software\u0026quot; means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. \u0026quot;Incompatible With Secondary Licenses\u0026quot; means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. \u0026quot;Executable Form\u0026quot; means any form of the work other than Source Code Form. 1.7. \u0026quot;Larger Work\u0026quot; means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. \u0026quot;License\u0026quot; means this document. 1.9. \u0026quot;Licensable\u0026quot; means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. \u0026quot;Modifications\u0026quot; means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. \u0026quot;Patent Claims\u0026quot; of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. \u0026quot;Secondary License\u0026quot; means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. \u0026quot;Source Code Form\u0026quot; means the form of the work preferred for making modifications. 1.14. \u0026quot;You\u0026quot; (or \u0026quot;Your\u0026quot;) means an individual or a legal entity exercising rights under this License. For legal entities, \u0026quot;You\u0026quot; includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, \u0026quot;control\u0026quot; means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an \u0026quot;as is\u0026quot; basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - \u0026quot;Incompatible With Secondary Licenses\u0026quot; Notice This Source Code Form is \u0026quot;Incompatible With Secondary Licenses\u0026quot;, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/go-msgpack/codec Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/hashicorp/go-multierror Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/memberlist Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/coordinate Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/hashicorp/serf/serf Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/kelseyhightower/envconfig Copyright (c) 2013 Kelsey Hightower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/mattn/go-isatty Copyright (c) Yasuhiro MATSUMOTO \u0026lt;mattn.jp@gmail.com\u0026gt; MIT License (Expat) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/miekg/dns Extensions of the original work are copyright (c) 2011 Miek Gieben As this is fork of the official Go code the same license applies: Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/mitchellh/cli Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/conf The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/logger The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/server/pse The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/gnatsd/util The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nats/encoders/builtin The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/nats-io/nuid The MIT License (MIT) Copyright (c) 2012-2016 Apcera Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/pborman/uuid Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. github.com/Sirupsen/logrus The MIT License (MIT) Copyright (c) 2014 Simon Eskildsen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/thejerf/suture Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. github.com/urfave/negroni The MIT License (MIT) Copyright (c) 2014 Jeremy Saenz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \u0026quot;Software\u0026quot;), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED \u0026quot;AS IS\u0026quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang.org/x/crypto/bcrypt Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/crypto/blowfish Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/net/context Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/unix Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/registry Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/sys/windows/svc/eventlog Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang.org/x/time/rate Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \u0026quot;AS IS\u0026quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. gopkg.in/yaml.v2 Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the \u0026quot;License\u0026quot;); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \u0026quot;AS IS\u0026quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ","excerpt":"The following document lists the open source software attributions in the StorageOS Control Plane, …","ref":"/v2/docs/reference/open_source_attribution/","title":"StorageOS Open Source Software Attribution Notice"},{"body":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform actions such as creation, deletion and mounting of volumes through CSI. The CSI container running in the StorageOS Daemonset creates a Linux socket that allows the communication between OpenShift and StorageOS\nCSI (Container Storage Interface) Note CSI is the standard that enables storage drivers to release on their own schedule. This allows storage vendors to upgrade, update, and enhance their drivers without the need to update Kubernetes source code, or follow Kubernetes release cycles.\nStorageOS v2 implements communication with the OpenShift controlplane with CSI only. The native driver has been deprecated\nStorageOS PersistentVolumeClaims The user can provide standard PVC definitions and StorageOS will dynamically provision them. StorageOS presents volumes to containers with standard POSIX mount targets. This enables the Kubelet to mount StorageOS volumes using standard linux device files. Checkout device presentation for more details.\nInstallation StorageOS v2 supports OpenShift 4.0, 4.1, 4.2 and 4.3.\nTo install StorageOS on OpenShift, please follow our installation instructions page.\n","excerpt":"StorageOS V2 supports Openshift v4.\nOpenShift and StorageOS communicate with each other to perform …","ref":"/v2/docs/platforms/openshift/","title":"OpenShift"},{"body":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a user or group access to a namespace, a policy needs to be created mapping the user or group to the namespace.\n Note: Users always have access to the default namespace\n For more information on how to use policies, see the Policies operations page.\n","excerpt":"StorageOS policies are a way to control user and group access to StorageOS Namespaces. To grant a …","ref":"/v2/docs/concepts/policies/","title":"Policies"},{"body":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user level so access can be controlled granularly.\nUsers can belong to one or more groups to control their namespace permissions. Additionally user specific policies can be created to grant a user access to a namespace. Users can belong to any number of groups and have any number of user level policies configured.\n Note: Users are created with access to the default namespace. Policies cannot be applied to the default namespace.\n Managing Policies To start creating policies, at least one custom namespace and user are required. For more information on how to create namespaces see our Namespace guide, for users see our Users CLI reference.\nIn order to create a policy navigate to \u0026ldquo;Policies\u0026rdquo; in the GUI and select \u0026ldquo;Create Policy\u0026rdquo;. A policy controls access to a variety of StorageOS resources and is applied to a user, by placing the user in the policies group.\nIn order to delete a policy, all users must be removed from the policy group before deletion of the policy can be completed.\n","excerpt":"Policies control access to StorageOS namespaces. Policies can be configured at the group or user …","ref":"/v2/docs/operations/policies/","title":"Policies"},{"body":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational database management system (ORDBMS).\nPostgres is deployed across a wide variety of platforms with a mix of workloads ranging from small, single-node use cases to large Internet-facing clusters with many concurrent users.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. If you need to setup StorageOS on Kubernetes then please see our guide to [installing StorageOS on Kubernetes]({% link \u0026ldquo;docs/install/kubernetes.md\u0026rdquo; \u0026gt;}}).\nDeploying PostgreSQL on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases PersistentVolumeClaim and Pod definition excerpts\nkind:PersistentVolumeClaimmetadata:name:pg-dataannotations:volume.beta.kubernetes.io/storage-class:fast...kind:Podmetadata:name:postgressspec:securityContext:fsGroup:26containers:- name:pgimage:crunchydata/crunchy-postgres:centos7-10.4-1.8.3volumeMounts:- mountPath:/pgdataname:data...volumes:- name:datapersistentVolumeClaim:claimName:pg-dataThis excerpt is from the PersistentVolumeClaim and Pod definition. The pod definition references the pg-data VolumeClaim so storage is dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the PostgreSQL examples folder and create the objects\ncd storageos-usecases kubectl create -f ./postgres Confirm PostgreSQL is up and running.\n$ kubectl get pod postgres-0 -w NAME READY STATUS RESTARTS AGE postgres-0 1/1 Running 0 1m Connect to the PostgreSQL client pod and connect to the PostgreSQL server through the service.\n$ kubectl exec -it postgres-0 -- psql -h postgres-0.postgres -U primaryuser postgres -c \u0026#34;\\l\u0026#34; Password for user primaryuser: password List of Databases Name | Owner | Encoding | Collate | Ctype | Access privileges +=========================================================================+ postgres | postgres | SQL_ASCII | C | C | template0 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres template1 | postgres | SQL_ASCII | C | C | =c/postgres + | | | | | postgres=CTc/postgres userdb | postgres | SQL_ASCII | C | C | =Tc/postgres + | | | | | postgres=CTc/postgres+ | | | | | testuser=CTc/postgres (4 rows) The password for the primary user is password. You can see this is set in the ConfigMap file.\n Configuration If you need custom startup options, you can edit the ConfigMap file 15-postgresd-configmap.yaml with your desired PostgreSQL configuration settings.\n","excerpt":"PostgreSQL with StorageOS PostgreSQL or \u0026ldquo;Postgres\u0026rdquo; is an open source object-relational …","ref":"/v2/docs/usecases/postgres/","title":"PostgreSQL"},{"body":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting in Kubernetes.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Prometheus on Kubernetes This is the Prometheus use case for StorageOS. Following are the steps for creating a Prometheus instance and using StorageOS to handle its persistent storage.\n You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases Prometheus Custom Resource defintion\napiVersion:monitoring.coreos.com/v1kind:Prometheusmetadata:name:prometheus-storageoslabels:app:prometheus-operatorspec:...storage:volumeClaimTemplate:metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:storageos-replicatedresources:requests:storage:1GiThis excerpt is from the Prometheus Custom Resource definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaim.\n Move into the Prometheus examples folder and create the objects\n$ cd storageos-usecases/prometheus $ ./install-prometheus.sh Confirm Prometheus is up and running.\n$ kubectl get pods -w -l app=prometheus NAME READY STATUS RESTARTS AGE prometheus-prometheus-storageos-0 3/3 READY 0 1m You can see the created PVC using.\n$ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-prometheus-prometheus-storageos-0 Bound pvc-b6c17c0a-e76b-4a0b-8fc6-46c0e1629210 1Gi RWO storageos-replicated 65m In the Prometheus deployment script, a service monitor is also created. The new Prometheus instance will use the storageos-etcd service monitor to start scraping metrics from the storageos-etcd pods. (Assuming the storageos cluster was setup using ETCD as pods) For more information about service monitors, have a look at the upstream documentation.\n$ kubectl get servicemonitor NAME AGE storageos-etcd 5d1h Port forward in the prometheus pod to access the prom webapp.\n$ kubectl port-forward prometheus-prometheus-storageos-0 9090 Then launch a web browser and type the url localhost:9090 to access the prometheus webapp. Confirm that prometheus is up and running there.\n Configuration In the storageos-usecases/prometheus/manifests/prometheus directory exist examples of Service Monitors. For more information at operating Prometheus, check out the prometheus documentation.\n","excerpt":"Prometheus with StorageOS Prometheus is a popular application used for event monitoring and alerting …","ref":"/v2/docs/usecases/prometheus/","title":"Prometheus"},{"body":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS transparently supports Rancher deployments using CentOS, RHEL or Debian and can support other Linux distributions detailed in the systems supported page if the appropriate kernel modules are present.\n","excerpt":"To install StorageOS on Rancher, please follow our installation instructions page.\nStorageOS …","ref":"/v2/docs/platforms/rancher/","title":"Rancher"},{"body":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional durability to disk.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Redis on Kubernetes You can find the latest files in the StorageOS use cases repository\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion\nkind:StatefulSetmetadata:name:redisspec:selector:matchLabels:app:redisenv:prodserviceName:redisreplicas:1...spec:serviceAccountName:redis...volumeMounts:- name:datamountPath:/bitnami/redis/data...volumeClaimTemplates:- metadata:name:datalabels:env:prodspec:accessModes:[\u0026#34;ReadWriteOnce\u0026#34;]storageClassName:\u0026#34;fast\u0026#34;# StorageOS storageClass resources:requests:storage:5GiThis excerpt is from the StatefulSet definition. This file contains the VolumeClaim template that will dynamically provision storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a Volume Claim.\n Move into the Redis examples folder and create the objects\ncd storageos-usecases kubectl create -f ./redis Confirm Redis is up and running.\n$ kubectl get pods -w -l app=redis NAME READY STATUS RESTARTS AGE redis-0 1/1 Running 0 1m Connect to the Redis client pod and connect to the Redis server through the service\n$ kubectl exec -it redis-0 -- redis-cli -a password Warning: Using a password with \u0026#39;-a\u0026#39; option on the command line interface may not be safe. 127.0.0.1:6379\u0026gt; CONFIG GET maxmemory 1) \u0026#34;maxmemory\u0026#34; 2) \u0026#34;0\u0026#34; Configuration If you need custom startup options, you can edit the ConfigMap file 15-redis-configmap.yaml with your desired Redis configuration settings.\n","excerpt":"Redis with StorageOS Redis is a popular networked, in-memory, key-value data store with optional …","ref":"/v2/docs/usecases/redis/","title":"Redis"},{"body":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than \u0026ldquo;latest\u0026rdquo;, and to perform upgrades only after reading the release notes.\nThe latest tagged release is v2.0.0. For installation instructions please see our Install page.\nThe latest CLI release is v2.0.0, available from Github\nUpgrading To upgrade from version 1.x to 2.x, please contact support for assistance.\nv2.2.0 - Released 2020-08-18 This release focuses on performance. We analysed existing performance characteristics across a variety of real-world use cases and ended up with improvements across the board. Of particular note:\n Sequential reads have improved by up to 130% Sequential writes have improved by up to 737% Random reads have improved by up to 45% Random writes have improved by up to 135% I/O for large block sizes (128K) has improved by up to 353% We are extremely proud of our performance and we love to talk about it. Have a look at the Benchmarking section of the self-evaluation guide and consider sharing your results. Our PRE engineers are available to discuss in our slack channel.\nNew Data engine revamp focused on provable consistency and performance. Key characteristics:\n Metadata is stored in an optimised index, lowering I/O latency and improving performance for all workloads. Large block reads/writes are now be handled in a single operation. Applications like Kafka will go much faster. On-disk compression is now disabled by default as in most scenarios this offers better performance. To enable on-disk compression for a specific workload, see compression.\n Improved dataplane: The number of I/O threads are now determined by the number of processing cores available. This improves scalability and performance on larger servers. ha: Improve partition tolerance behaviour when a volume master that has lost its connection to etcd rejoins. ha: Allow replicas in unhealthy states to be remediated and re-used while maintaining partition tolerance. ha: When a master fails and the new master is not yet available, introduce a back-off to the redirection logic to avoid spamming the logs with connection failure errors. ha: Ignore health advertisements for local node. Local nodes are handled directly. node delete: Only refuse to delete a node if the node health can be authoritatively verified to be in use. api: Increase HTTP server write timeout. cli/ui: Allow partial diagnostic bundle downloads. ui: Namespace dropdown can now be scrolled. ui: Add \u0026ldquo;Job title\u0026rdquo; to UI licence form. logging: Log version at startup at INFO level. logging: Lower verbosity of SCSI warnings that do not apply to StorageOS. diagnostics: Include logs that have been rotated. diagnostics: Bundle collection across providers is now done in parallel. build: Update base image to RHEL 8.2. operator: Removed DB migration utility required for v1.3 -\u0026gt; v1.4 upgrades. operator: Automatically refreshes StorageOS API token without failing requests when the token expires. operator: Updated CSI attacher and provisioner to latest upstream. operator: Remove cluster.local suffix on Pod scheduler service address. This allows the scheduler to work in clusters with custom DNS configuration. operator: Defaults are now set for most CSI configuration options in the StorageOSCluster custom resource. Fixed csi: When unmount request is received for a volume that has already been unmounted, return success. csi: Verify volume is attached on the node before mounting it. xfs: Support older RHEL kernels which have an XFS library that does not allow reflinks/dedupe. dataplane: Reserve 1GiB of capacity on the target disk to allow manual cleanup operations, rather than filling target disk to capacity. operator: In some cases /var/lib/storageos could fail to unmount cleanly after a restart. This resulted in multiple entries in /proc/mounts. v2.1.0 - Released 2020-06-26 New csi: Volume expansion now supported in offline mode. To expand a volume, stop any workloads accessing the volume, then edit the PVC to increase the capacity. For more information, see our Volume Resize operations page and the CSI Volume Expansion page. api: Volume configuration including replica count can now be updated while the volume is in use. Other updateable fields include labels and description. failover: Before determining that a node is offline and performing recovery operations, the I/O path is also verified. This provides more robust failure detection and ensures that nodes that are still responding to I/O do not get replaced. This I/O path verification is in addition to the gossip-based failure detection. operator: Default tolerations are now set for the StorageOS node container. This helps ensure that the StorageOS node container does not get evicted when the node is running low on resources. Improved api: Added checks to prevent deletion of a node with active volumes, or if it is the master of at least one volume. This helps prevent orphaned volumes. cli: Add an --offline-delete flag to allow removal of volumes whose master and replica nodes are offline. This allows cleanup of orphaned volumes. ui: Add an offline volume delete option. ui: Volumes can now be detached from the UI. cli: Labels are no longer truncated. api: When a new node is added to the cluster, its capacity is available to use immediately. Fixed ui: Favicon was missing. ui: Duplicate volumes could be shown on the node details page. operator: During uninstall a ClusterRoleBinding was not removed. v2.0.0 - Released 2020-05-05 New operator: StorageOS containers now run in the kube-system namespace by default to allow the system-node-critical priority class to be set. This instructs Kubernetes to start StorageOS before application Pods, and to evict StorageOS only after application Pods have finished. This setting was previously recommended in documentation; it is now the default. operator: StorageOS CSI helper containers now run as privileged. This ensures that the CSI endpoint can be seen on systems with SELinux enabled. ui: replication progress for new or re-joining replicas is now displayed. ui: show warning for unlicensed clusters. cli: new commands: licence management get policy create namespace create policy describe user describe namespace describe policy delete user delete namespace delete policy licence: removed the default licence expiry date added for v2.0.0-rc.1. Improved dataplane: improved retry behaviour for network I/O. cli: \u0026ldquo;get volumes\u0026rdquo; for all namespaces should be done in parallel. cli: help text document config file ui: link node name and get to node details on the volume details page. ui: node details add available capacity spinner. ui: node list remove capacity values / address port. ui: node list show master/replica counts. ui: node list remove edit action. ui: format entity labels. ui: node details link volumes. ui: align buttons for licences. ui: k8s warning in \u0026ldquo;create volume\u0026rdquo; modal. ui: node list remove \u0026ldquo;API\u0026rdquo; from \u0026ldquo;API Address\u0026rdquo; ui: add some details about the Licence on the licence page. api: include valid for duration in login response. licence: restrict nodes which are unregistered after 24 hours. scheduler: return error for namespace/volume not found dataplane: start gRPC threads separately from rest of the supervisor. Fixed ui: centre licence types. ui: capacity in ui is per namespace. cli: fail gracefully if missing some output details (i.e. no node exists for id). v2.0.0-rc.1 - Released 2020-03-31 Initial release of version 2.x. See StorageOS v2.0 Release Blog for details.\n","excerpt":"We recommend always using \u0026ldquo;tagged\u0026rdquo; versions of StorageOS rather than …","ref":"/v2/docs/reference/release_notes/","title":"Release notes"},{"body":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous replication ensures strong consistency for applications such as databases and Elasticsearch, incurring one network round trip on writes.\nThe basic model for StorageOS replication is of a master volume with distributed replicas. Each volume can be replicated between 0 and 5 times, which are provisioned to 0 to 5 nodes, up to the number of remaining nodes in the cluster.\nIn this diagram, the master volume D was created on node 1, and two replicas, D2 and D3 on nodes 3 and 5.\nWrites that come into D (step 1) are written in parallel to D2 and D3 (step 2). When both replicas and the master acknowledge that the data has been written (step 3), the write operation return successfully to the application (step 4).\nFor most applications, one replica is sufficient (storageos.com/replicas=1).\nAll replication traffic on the wire is compressed using the lz4 algorithm, then streamed over tcp/ip to target port tcp/5703.\nIf the master volume is lost, a replica is promoted to master (D2 or D3 above) and a new replica is created and synced on an available node (Node 2 or 4). This is transparent to the application and does not cause downtime.\nIf a replica volume is lost and there are enough remaining nodes, a new replica is created and synced on an available node. While a new replica is created and being synced, the volume\u0026rsquo;s health will be marked as degraded.\nIf the lost replica comes back online before the new replica has finished synchronizing, then StorageOS will calculate which of the two synchronizing replicas has the smallest difference compared to the master volume and keep that replica. The same holds true if a master volume is lost and a replica is promoted to be the new master. If possible, a new replica will be created and begin to sync. Should the former master come back online it will be demoted to a replica and the replica will the smallest difference to the current master will be kept.\nWhile the replica count is controllable on a per-volume basis, some environments may prefer to set default labels on the StorageClass.\nDelta Sync StorageOS implements a delta sync between a volume master and its replicas. This means that if a replica for a volume goes offline, that when the replica comes back online only the regions with changed blocks need to be synchronized. This optimization reduces the time it takes for replicas to catch up, improving volume resilience. Additionally, it reduces network and IO bandwidth which can reduce costs when running in public clouds.\n","excerpt":"StorageOS replicates volumes across nodes for data protection and high availability. Synchronous …","ref":"/v2/docs/concepts/replication/","title":"Replication"},{"body":"","excerpt":"","ref":"/v2/search/","title":"Search Results"},{"body":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for instance, StorageOS. A StorageClass defines parameters to pass to the provisioner, which in the case of StorageOS can be translated into behaviour applied to the Volumes. Many StorageClasses can be provisioned to apply different feature labels to the StorageOS Volumes.\nBy default the StorageOS Cluster Operator installs the fast StorageClass at bootstrap of StorageOS. You can define its name in the StorageOS Cluster Resource.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:fastprovisioner:csi.storageos.comparameters:csi.storage.k8s.io/fstype:ext4pool:default# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretStorageClasses can be created to define default labels for StorageOS volumes, but also to map to any semantic aggregation of volumes that suits your use case, whether there are different roles (dev, staging, prod), or a StorageClass maps to a team or customer using the cluster.\nExamples You can find the basic examples in the StorageOS use-cases repository, in the 00-basic directory.\ngit clone https://github.com/storageos/use-cases.git storageos-usecases cd storageos-usecases/00-basic StorageClass definition in v2-storageclass-replicated.yaml.\napiVersion:storage.k8s.io/v1kind:StorageClassmetadata:name:storageos-replicatedprovisioner:csi.storageos.com# Provisioner when using CSIparameters:fsType:ext4pool:defaultstorageos.com/replicas:\u0026#34;1\u0026#34;# Enforces 1 replica for the Volume# Change the Namespace below if StorageOS doesn\u0026#39;t run in kube-systemcsi.storage.k8s.io/node-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/provisioner-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/controller-publish-secret-namespace:kube-system# Namespace that runs StorageOS Daemonsetcsi.storage.k8s.io/node-publish-secret-name:csi-node-publish-secretcsi.storage.k8s.io/provisioner-secret-name:csi-provisioner-secretcsi.storage.k8s.io/controller-publish-secret-name:csi-controller-publish-secretThat StorageClass can be used by a PVC:\napiVersion:v1kind:PersistentVolumeClaimmetadata:name:my-vol-1spec:storageClassName:\u0026#34;storageos-replicated\u0026#34;# StorageOS StorageClassaccessModes:- ReadWriteOnceresources:requests:storage:5GiThe above StorageClass has the storageos.com/replicas label set. This label tells StorageOS to create a volume with a replica. Adding StorageOS feature labels to the StorageClass ensures all volumes created with the StorageClass have the same labels.\nYou can also choose to add the label in the PVC definition rather than the StorageClass. The PVC definition takes precedence over the SC.\n","excerpt":"StorageClassess in Kubernetes are used to link PVCs with a backend storage provisioner - for …","ref":"/v2/docs/operations/storageclasses/","title":"Kubernetes StorageClasses"},{"body":"","excerpt":"","ref":"/v2/","title":"StorageOS"},{"body":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for two different purposes.\nTelemetry Telemetry is made up a DNS version check query and a once per day report. Error reporting is the sentry.io crash dump reporting. sentry.io StorageOS sends crash reports to sentry.io. This information helps our developers monitor and fix crashes. Information is sent to sentry.io when a process inside the StorageOS container crashes.\n The crash report contains the signal that triggered the shutdown (e.g. SIGSEGV), the exit code and whether or not the crash generated a core dump. All StorageOS clusters with a routable connection to the internet will send crash reports to sentry.io over tcp/443. StorageOS respects environment variables that ProxyFromEnvironment uses.\nAn exhaustive list of information included in the crash report is below:\n StorageOS version Crash description string Anonymized Cluster ID Anonymized Node ID DNS Query StorageOS will perform a \u0026ldquo;latest version check\u0026rdquo; using a DNS query in order to inform administrators that a new version is available. StorageOS will also send anonymized node ids, cluster id and StorageOS version information to StorageOS using a DNS query. The information that we send in the query is encoded as well as being anonymized. This query allows us to inform Cluster admins when StorageOS upgrades are available in the StorageOS GUI and in the logs.\nThe DNS query includes:\n Anonymized StorageOS Cluster ID Anonymized StorageOS node ID StorageOS version number Once Per Day Report The once per day report contains information about the StorageOS cluster and Kubernetes versions to help StorageOS focus our development efforts on the most popular platforms. The once per day data is encrypted and sent to a StorageOS telemetry server so it is never processed outside of StorageOS assets.\nAn exhaustive list of information included in the once per day report is below:\n api_call_metrics cluster_disable_crash_reporting cluster_disable_version_check cluster_log_format cluster_log_level cluster_tls_provided k8s_distribution k8s_in_k8s k8s_scheduler_extender_enabled k8s_version node_available_bytes node_capacity node_crash_files_on_disk node_created_at_time node_etcd_config node_etcd_namespacing_enabled node_etcd_tls_enabled node_free_bytes node_health node_http_tls_enabled node_id node_labels node_status node_storageos_version node_system_clock_time node_total_bytes node_version volume_fs_type volume_id volume_labels volume_master_attach volume_master_delete_deployment volume_master_detach volume_master_failover_deployment volume_master_promote volume_master_provision volume_master_recover_replica volume_master_trigger_rejoin volume_metrics volume_placement_strategy volume_provision_source_user_agent volume_replica_delete_deployment volume_replica_failover_deployment volume_replica_promote volume_replica_provision volume_replicas volume_replica_trigger_rejoin volume_size_bytes Disable Telemetry It is possible to disable telemetry using the GUI, CLI, API, environment variables or the StorageOS Cluster Spec.\nStorageOS Cluster Spec Disable telemetry explicitly through the configurable spec parameters of the StorageOSCluster custom resource.\nEnvironment Variables You can use the following environmental variables to disable or enable telemetry.\nDISABLE_VERSION_CHECK # Disable the DNS version check DISABLE_TELEMETRY # Disable the once per day reporting DISABLE_ERROR_REPORTING # Disable sentry.io crash reports ","excerpt":"StorageOS collects telemetry and error reports from StorageOS clusters via two different methods for …","ref":"/v2/docs/reference/telemetry/","title":"StorageOS Telemetry"},{"body":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The bundle packages the information needed for the engineering team to understand the context of your cluster and begin troubleshooting any issues you may be experiencing.\nGenerate Bundle CLI The StorageOS CLI can generate the bundle.\n$ storageos get diagnostics The file generated is in the form of diagnostics-${TIME_STAMP}.gz\n GUI Or you can use the StoregeOS GUI.\n Go to section \u0026ldquo;Cluster\u0026rdquo; Press the button \u0026ldquo;DOWNLOAD DIAGNOSTICS\u0026rdquo;. Data collected in the bundle Most of the data collected in the bundle is regarding the state of the StorageOS cluster, however some other information regarding the infrastructure is also gathered. The information is used to have a clear view of the cluster where StorageOS is running.\nThe bundle incorporates for each node:\n StorageOS Daemonset Pod logs lshw dmesg (kernel logs) StorageOS metadata for the ControlPlane and DataPlane StorageOS metadata collected cluster metadata namespaces metadata nodes metadata volumes metadata capacity stats environment variables health Privacy StorageOS can only obtain the bundle if it is downloaded by the user and given to our engineering team, or uploaded for analysis. The data received by StorageOS is private and never leaves nor will leave StorageOS Inc.\nThe data contained in the cluster diagnostic bundle has the sole purpose of helping customers troubleshoot their issues.\n","excerpt":"StorageOS can generate a cluster diagnostic bundle from the GUI or CLI under user request. The …","ref":"/v2/docs/reference/bundle/","title":"Support bundle"},{"body":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base directory for storing configuration and blob files. Recommended host filesystem types are ext4 or xfs. If you require a specific filesystem please contact StorageOS.\nPersistent Volume Filesystems StorageOS provides a block device on which a file system can be created. The creation of the filesystem is either handled by StorageOS or by Kubernetes which affects what filesystems can be created.\nCSI Driver When using StorageOS with the CSI driver, StorageOS is responsible for running mkfs against the block device that pods mount. StorageOS is able to create ext2, ext3, ext4 and xfs file systems.\n","excerpt":"Host Filesystems StorageOS will automatically use /var/lib/storageos on each host as a base …","ref":"/v2/docs/reference/filesystems/","title":"Supported File systems"},{"body":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain node taints. Taints can be thought of as the opposite of Node affinity in that taints repel pods. Node taints are automatically applied by Kubernetes in response to node resources coming under contention. As StorageOS provides storage to pods it should not be evicted during periods of resource contention, as any pods using StorageOS volumes on the same node would need to be restarted.\nAs StorageOS runs as a daemonset some Kubernetes tolerations are added by Kubernetes while others are automatically added by the StorageOS operator.\nFor more information about tolerations please see the Kubernetes documentation.\ntolerations:# The unreachable and not-ready tolerations are added by Kubernetes to daemonsets automatically- key:\u0026#34;node.kubernetes.io/unreachable\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;- key:\u0026#34;node.kubernetes.io/not-ready\u0026#34;operator:\u0026#34;Exists\u0026#34;effect:\u0026#34;NoExecute\u0026#34;# The following tolerations are added to the StorageOS daemonset by the StorageOS operator- key:node.kubernetes.io/disk-pressureoperator:Exists- key:node.kubernetes.io/memory-pressureoperator:Exists- key:node.kubernetes.io/network-unavailableoperator:Exists- key:node.kubernetes.io/out-of-diskoperator:Exists- key:node.kubernetes.io/pid-pressureoperator:Exists- key:node.kubernetes.io/unschedulableoperator:ExistsAdding Custom Tolerations To add custom tolerations to the StorageOS daemonset configure them in the StorageOSCluster resource.\n","excerpt":"Kubernetes Tolerations Tolerations are a Kubernetes pod property that allow pods to tolerate certain …","ref":"/v2/docs/reference/tolerations/","title":"Tolerations"},{"body":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to the StorageOS installation, integration with orchestrators or common misconfigurations.\nTools To be able to troubleshoot issues the StorageOS cli is required.\nPod in pending because of mount error Issue: The output of kubectl describe pod $POD_ID contains no such file or directory and references the StorageOS volume device file.\nroot@node1:~# kubectl -n kube-system describe $POD_ID (...) Events: (...) Normal Scheduled 11s default-scheduler Successfully assigned default/d1 to node3 Warning FailedMount 4s (x4 over 9s) kubelet, node3 MountVolume.SetUp failed for volume \u0026#34;pvc-f2a49198-c00c-11e8-ba01-0800278dc04d\u0026#34; : stat /var/lib/storageos/volumes/d9df3549-26c0-4cfc-62b4-724b443069a1: no such file or directory Reason: There are two main reasons this issue may arise:\n The StorageOS DEVICE_DIR location is wrongly configured when using Kubelet as a container Mount Propagation is not enabled (Option 1) Misconfiguration of the DeviceDir/SharedDir\nSome Kubernetes distributions such as Rancher, DockerEE or some installations of OpenShift deploy the Kubelet as a container, because of this, the device files that StorageOS creates to mount into the containers need to be visible to the kubelet. StorageOS can be configured to share the device directory.\nModern installations use CSI, which handles the complexity internally.\nAssert: root@node1:~# kubectl -n default describe stos | grep \u0026#34;Shared Dir\u0026#34; Shared Dir: # \u0026amp;lt;-- Shouldn\u0026#39;t be blank Solution: The Cluster Operator Custom Definition should specify the SharedDir option as follows.\nspec: sharedDir: \u0026#39;/var/lib/kubelet/plugins/kubernetes.io~storageos\u0026#39; # Needed when Kubelet as a container ... See example on how to configure the StorageOS Custom Resource.\n (Option 2) Mount propagation is not enabled.\n Applies only if Option 1 is configured properly.\n Assert: If not using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/storageos/volumes is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/storageos/volumes/ root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/storageos/volumes bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nIf using the Kubelet as a container, SSH into one of the nodes and check if /var/lib/kubelet/plugins/kubernetes.io~storageos/devices is empty. If so, exec into any StorageOS pod and check the same directory.\nroot@node1:~# ls /var/lib/kubelet/plugins/kubernetes.io~storageos/devices root@node1:~# # \u0026amp;lt;-- Shouldn\u0026#39;t be blank root@node1:~# kubectl exec $POD_ID -c storageos -- ls -l /var/lib/kubelet/plugins/kubernetes.io~storageos/devices bst-196004 d529b340-0189-15c7-f8f3-33bfc4cf03fa ff537c5b-e295-e518-a340-0b6308b69f74 If the directory inside the container and the device files are visible, disabled mount propagation is the cause.\nSolution: Older versions of Kubernetes need to enable mount propagation as it is not enabled by default. Most Kubernetes distributions allow MountPropagation to be enabled using FeatureGates. Rancher specifically, needs to enable it in the \u0026ldquo;View in API\u0026rdquo; section of your cluster. You need to edit the section \u0026ldquo;rancherKubernetesEngineConfig\u0026rdquo; to enable the Kubelet feature gate.\nPVC pending state - Failed to dial StorageOS A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: root@node1:~/# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE vol-1 Pending fast 7s kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 7s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: Get http://storageos-cluster/version: failed to dial all known cluster members, (10.233.59.206:5705) Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount volume can\u0026rsquo;t be transmitted to StorageOS, hence the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nIn this case, the Event message indicates that StorageOS API is not responding, implying that StorageOS is not running. For Kubernetes to define StorageOS pods ready, the health check must pass.\nAssert: Check the status of StorageOS pods.\nroot@node1:~/# kubectl -n kube-system get pod --selector app=storageos # for CSI add --selector kind=daemonset NAME READY STATUS RESTARTS AGE storageos-qrqkj 0/1 Running 0 1m storageos-s4bfv 0/1 Running 0 1m storageos-vcpfx 0/1 Running 0 1m storageos-w98f5 0/1 Running 0 1m If the pods are not READY, the service will not forward traffic to the API they serve hence PVC will remain in pending state until StorageOS pods are available.\n Kubernetes keeps trying to execute the action until it succeeds. If a PVC is created before StorageOS finish starting, the PVC will be created eventually.\n Solution: StorageOS health check takes 60 seconds of grace before reporting as READY. If StorageOS is starting properly after that period, the volume will be created when StorageOS finishes its bootstrap. If StorageOS is not running or is not starting properly, the solution would be to troubleshoot the installation. PVC pending state - Secret Missing A created PVC remains in pending state making pods that need to mount that PVC unable to start.\nIssue: kubectl describe pvc $PVC (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning ProvisioningFailed 13s (x2 over 28s) persistentvolume-controller Failed to provision volume with StorageClass \u0026#34;fast\u0026#34;: failed to get secret from [\u0026#34;storageos\u0026#34;/\u0026#34;storageos-api\u0026#34;] Reason: For non CSI installations of StorageOS, Kubernetes uses the StorageOS API endpoint to communicate. If that communication fails, relevant actions such as create or mount a volume can\u0026rsquo;t be transmitted to StorageOS, and the PVC will remain in pending state. StorageOS never received the action to perform, so it never sent back an acknowledgement.\nThe StorageClass provisioned for StorageOS references a Secret from where it retrieves the API endpoint and the authentication parameters. If that secret is incorrect or missing, the connections won\u0026rsquo;t be established. It is common to see that the Secret has been deployed in a different namespace where the StorageClass expects it or that is has been deployed with a different name.\nAssert: Check the StorageClass parameters to know where the Secret is expected to be found.\nkubectl get storageclass fast -o yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: 2018-09-25T08:44:57Z labels: app: storageos name: fast resourceVersion: \u0026#34;108853\u0026#34; selfLink: /apis/storage.k8s.io/v1/storageclasses/fast uid: 48490a9b-c09f-11e8-ba01-0800278dc04d parameters: adminSecretName: storageos-api adminSecretNamespace: storageos description: Kubernetes volume fsType: ext4 pool: default provisioner: kubernetes.io/storageos reclaimPolicy: Delete Note that the parameters specify adminSecretName and adminSecretNamespace.\n Check if the secret exists according to those parameters\nkubectl -n kube-system get secret storageos-api No resources found. Error from server (NotFound): secrets \u0026#34;storageos-api\u0026#34; not found If no resources are found, it is clear that the Secret doesn\u0026rsquo;t exist or it is not deployed in the right location.\n Solution: Deploy StorageOS following the installation procedures. If you are using the manifests provided for Kubernetes to deploy StorageOS rather than using automated provisioners, make sure that the StorageClass parameters and the Secret reference match.\nPeer discovery - Pod allocation Issue: StorageOS nodes can\u0026rsquo;t join the cluster and show the following log entries.\ntime=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;not first cluster node, joining first node\u0026#34; action=create address=172.28.128.5 category=etcd host=node3 module=cp target=172.28.128.6 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;could not retrieve cluster config from api\u0026#34; status_code=503 time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=error msg=\u0026#34;failed to join existing cluster\u0026#34; action=create category=etcd endpoint=\u0026#34;172.28.128.3,172.28.128.4,172.28.128.5,172.28.128.6\u0026#34; error=\u0026#34;503 Service Unavailable\u0026#34; module=cp time=\u0026#34;2018-09-24T13:40:20Z\u0026#34; level=info msg=\u0026#34;retrying cluster join in 5 seconds...\u0026#34; action=create category=etcd module=cp Reason: StorageOS uses a gossip protocol to discover the nodes in the cluster. When StorageOS starts, one or more active nodes must be referenced so new nodes can query existing nodes for the list of members. This error indicates that the node can\u0026rsquo;t connect to any of the nodes in the known list. The known list is defined in the JOIN variable.\nIf there are no active StorageOS nodes, the bootstrap process will elect the first node in the JOIN variable as master, and the rest will try to discover from it. In case of that node not starting, the whole cluster will remain unable to bootstrap.\nInstallations of StorageOS use a DaemonSet, and by default do not schedule StorageOS pods to master nodes, due to the presence of the node-role.kubernetes.io/master:NoSchedule taint that is typically present. In such cases the JOIN variable must not contain master nodes or the StorageOS cluster will remain unable to start.\nAssert: Check that the first node of the JOIN variable started properly.\nroot@node1:~/# kubectl -n kube-system describe ds/storageos | grep JOIN JOIN: 172.28.128.3,172.28.128.4,172.28.128.5 root@node1:~/# kubectl -n kube-system get pod -o wide | grep 172.28.128.3 storageos-8zqxl 1/1 Running 0 2m 172.28.128.3 node1 Solution: Make sure that the JOIN variable doesn\u0026rsquo;t specify the master nodes. In case you are using the discovery service, it is necessary to ensure that the DaemonSet won\u0026rsquo;t allocate Pods on the masters. This can be achieved with taints, node selectors or labels.\nFor installations with the StorageOS operator you can specify which nodes to deploy StorageOS on using nodeSelectors. See examples in the Cluster Operator Examples page.\nFor more advanced installations using compute-only and storage nodes, check the storageos.com/deployment=computeonly label that can be added to the nodes through Kubernetes node labels, or StorageOS in the Labels page.\nLIO Init:Error Issue: StorageOS pods not starting with Init:Error\nkubectl -n kube-system get pod NAME READY STATUS RESTARTS AGE storageos-2kwqx 0/3 Init:Err 0 6s storageos-cffcr 0/3 Init:Err 0 6s storageos-d4f69 0/3 Init:Err 0 6s storageos-nhq7m 0/3 Init:Err 0 6s Reason: This indicates that since the Linux open source SCSI drivers are not enabled, StorageOS cannot start. The StorageOS DaemonSet enables the required kernel modules on the host system. If you are seeing these errors it is because that container couldn\u0026rsquo;t load the modules.\nAssert Check the logs of the init container.\nkubectl -n kube-system logs $ANY_STORAGEOS_POD -c storageos-init In case of failure, it will show the following output, indicating which kernel modules couldn\u0026rsquo;t be loaded or that they are not properly configured:\nChecking configfs configfs mounted on sys/kernel/config Module target_core_mod is not running executing modprobe -b target_core_mod Module tcm_loop is not running executing modprobe -b tcm_loop modprobe: FATAL: Module tcm_loop not found. Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and delete StorageOS pods, allowing the DaemonSet to create the pods again.\nLIO not enabled Issue: StorageOS node can\u0026rsquo;t start and shows the following log entries.\ntime=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;liocheck returned error\u0026#34; category=liocheck error=\u0026#34;exit status 1\u0026#34; module=dataplane stderr=\u0026#34;Sysfs root \u0026#39;/sys/kernel/config/target\u0026#39; is missing, is kernel configfs present and target_core_mod loaded? category=fslio level=warn\\nRuntime error checking stage \u0026#39;target_core_mod\u0026#39;: SysFs root missing category=fslio level=warn\\nliocheck: FAIL (lio_capable_system() returns failure) category=fslio level=fatal\\n\u0026#34; stdout= time=\u0026#34;2018-09-24T14:34:40Z\u0026#34; level=error msg=\u0026#34;failed to start dataplane services\u0026#34; error=\u0026#34;system dependency check failed: exit status 1\u0026#34; module=command Reason: This indicates that one or more kernel modules required for StorageOS are not loaded.\nAssert The following kernel modules must be enabled in the host.\nlsmod | egrep \u0026#34;^tcm_loop|^target_core_mod|^target_core_file|^configfs\u0026#34; Solution: Install the required kernel modules (usually found in the linux-image-extra-$(uname -r) package of your distribution) on your nodes following this prerequisites page and restart the container.\n(OpenShift) StorageOS pods missing \u0026ndash; DaemonSet error StorageOS DaemonSet doesn\u0026rsquo;t have any pod replicas. The DaemonSet couldn\u0026rsquo;t allocate any Pod due to security issues.\nIssue: [root@master02 standard]# oc get pod No resources found. [root@master02 standard]# oc describe daemonset storageos (...) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedCreate 0s (x12 over 10s) daemonset-controller Error creating: pods \u0026#34;storageos-\u0026#34; is forbidden: unable to validate against any security context constraint: [provider restricted: .spec.securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used provider restricted: .spec.securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.volumes[0]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[1]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[2]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.volumes[3]: Invalid value: \u0026#34;hostPath\u0026#34;: hostPath volumes are not allowed to be used spec.initContainers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.initContainers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.initContainers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.initContainers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed capabilities.add: Invalid value: \u0026#34;SYS_ADMIN\u0026#34;: capability may not be added spec.containers[0].securityContext.hostNetwork: Invalid value: true: Host network is not allowed to be used spec.containers[0].securityContext.containers[0].hostPort: Invalid value: 5705: Host ports are not allowed to be used spec.containers[0].securityContext.hostPID: Invalid value: true: Host PID is not allowed to be used] Reason: The OpenShift cluster has security context constraint policies enabled that forbid any pod, without an explicitly set policy for the service account, to be allocated.\nAssert: Check if the StorageOS ServiceAccount can create pods with enough permissions\noc get scc privileged -o yaml # Or custom scc with enough privileges (...) users: - system:admin - system:serviceaccount:openshift-infra:build-controller - system:serviceaccount:management-infra:management-admin - system:serviceaccount:management-infra:inspector-admin - system:serviceaccount:storageos:storageos \u0026amp;lt;-- - system:serviceaccount:tiller:tiller If the StorageOS sa system:serviceaccount:storageos:storageos is in the privileged scc it will be able to create pods.\nSolution: Add the ServiceAccount system:serviceaccount:storageos:storageos to a scc with enough privileges.\noc adm policy add-scc-to-user privileged system:serviceaccount:storageos:storageos Getting Help If our troubleshooting guides do not help resolve your issue, please see our support section for details on how to get in touch with us.\n","excerpt":"This section is aimed to help you troubleshoot issues in your cluster, whether they are related to …","ref":"/v2/docs/operations/troubleshooting/","title":"Troubleshooting"},{"body":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces using policies.\n Note: Users are created with access to the default namespace. This access is only revoked when a policy is created for the user or their group.\n Managing users Users can be created and deleted by navigating to the \u0026ldquo;Users\u0026rdquo; section of the GUI.\nAlternatively users can also be managed using the CLI.\nTo create a user with the CLI, run:\n$ storageos user create jim The above command will create a user named jim. The command will also prompt you to enter a password for the newly created user.\n","excerpt":"A StorageOS cluster admin can create users and restrict their access rights to StorageOS namespaces …","ref":"/v2/docs/operations/users/","title":"User Management"},{"body":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version [flags] Flags: -h, --help help for version Global Flags: --endpoints stringArray set the list of endpoints which are used when connecting to the StorageOS API (default [http://localhost:5705]) -n, --namespace string specifies the namespace to operate within for commands that require one (default \u0026#34;default\u0026#34;) -o, --output string specifies the output format (one of [json yaml text]) (default \u0026#34;text\u0026#34;) --password string set the StorageOS account password to authenticate with (default \u0026#34;storageos\u0026#34;) --timeout duration set the timeout duration to use for execution of the command (default 5s) --use-ids specify existing StorageOS resources by their unique identifiers instead of by their names --username string set the StorageOS account username to authenticate as (default \u0026#34;storageos\u0026#34;) ","excerpt":"$ storageos version --help View version information for the StorageOS CLI Usage: storageos version …","ref":"/v2/docs/reference/cli/version/","title":"Version"},{"body":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating the volume config via the CLI or UI. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure the volume is not in use. Please note that StorageOS only supports increasing volume size. For more information about how the resize works please see our Resize concepts page.\nResizing a Volume In order to resize a PVC the storage request field must be updated.\napiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-1 spec: storageClassName: fast accessModes: - ReadWriteOnce resources: requests: storage: 5Gi In order to edit a PVC you can use kubectl edit or kubectl apply to make changes.\nStorageOS supports offline resize of volumes through editing a PVC storage request. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This requires that any pods using a volume must be scaled down for the resize to take place.\n N.B. Resizing a volume without updating the PVC directly will NOT result in the PVC being updated. The methods below are included for completeness, in Kubernetes environments editing the PVC is the preferred method for resizing a volume.\n To resize a volume using the StorageOS CLI use the volume update command\n$ storageos update volume size pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa 10GiB Name: pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa ID: 925e667f-91d3-465a-9391-8fdb56d0c9ff Size: 10 GiB Description: AttachedOn: Replicas: 1x ready Labels: - csi.storage.k8s.io/pv/name pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa - csi.storage.k8s.io/pvc/name pvc-1 - csi.storage.k8s.io/pvc/namespace default - pool default - storageos.com/replicas 1 Volume pvc-a47cfa03-cc92-4ec9-84ab-00e5516c64fa (925e667f-91d3-465a-9391-8fdb56d0c9ff) updated. Size changed. To resize a volume using the StorageOS UI, navigate to the volumes section and click the edit pencil in order to update the volume config.\n","excerpt":"StorageOS supports offline resize of volumes through editing a PVC storage request, or by updating …","ref":"/v2/docs/operations/resize/","title":"Volume Resize"},{"body":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard POSIX semantics. StorageOS presents volumes as mounts into containers via the Linux LIO subsystem.\nConceptually, StorageOS volumes have a frontend presentation, which is the what the application sees, and a backend presentation, which is the actual on-disk format. Depending on the configuration, frontend and backend components may be on the same or different hosts.\nVolumes are formatted using the linux standard ext4 filesystem by default. Kubernetes users may change the default filesystem type to ext2, ext3, ext4, or xfs by setting the fsType parameter in their StorageClass (See Supported Filesystems for more information). Different filesystems may be supported in the future.\nStorageOS volumes are represented on disk in two parts. Actual volume data is written to blob files in /var/lib/storageos/data/dev[\\d+]. Inside these directories, each StorageOS block device gets two blob files of the form vol.xxxxxx.y.blob, where x is the inode number for the device, and y is an index between 0 and 1. We provide two blob files in order to ensure that certain operations which require locking do not impede in-flight writes to the volume.\nIn systems which have multiple /var/lib/storageos/data/dev[\\d+] directories, two blob files are created per block device. This allows us to load-balance writes across multiple devices. In cases where dev directories are added after a period of run time, later directories are favoured for writes until the data is distributed evenly across the blob files.\nMetadata is kept in directories named /var/lib/storageos/data/db[\\d+]. We maintain an index of all blocks written to the blob file inside the metadata store, including checksums. These checksums allow us to detect bitrot, and return errors on reads, rather than serve bad data. In future versions we may implement recovery from replicas for volumes with one or more replicas defined.\nStorageOS metadata requires approximately 2.7GB of storage per 1TB of allocated blocks in the associated volume. This size is consistent irrespective of data compression defined on the volume.\nTo ensure deterministic performance, individual StorageOS volumes must fit on a single node.\nVolume Resize StorageOS v2.1 supports offline resize of volumes. This means that a volume cannot be resized while it is in use. Furthermore, in order for a resize operation to take place the volume must not be attached to a node. This is to ensure that the volume is not in use.\nThis means that if a Kubernetes pod is currently consuming a volume that a resize request has been issued for, the resize will not be actioned until the pod is terminated and the volume is detached from the node. The StorageOS controlplane will then attach the volume to the node that holds the master deployment and resize the underlying block device and then run resize2fs to expand the filesystem.\nFor a walk through of how to resize a volume please see the Volume Resize operations page.\n","excerpt":"StorageOS volumes are a logical construct which represent a writeable volume and exhibit standard …","ref":"/v2/docs/concepts/volumes/","title":"Volumes"},{"body":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.\nUsing StorageOS persistent volumes with Apache Zookeeper means that if a pod fails, the cluster is only in a degraded state for as long as it takes Kubernetes to restart the pod. When the pod comes back up, the pod data is immediately available. Should Kubernetes schedule the Zookeeper pod on a new node, StorageOS allows for the data to be available to the pod, irrespective of whether or not the original StorageOS master volume is located on the same node.\nAs Zookeeper has features to allow it to handle replication, and as such careful consideration of whether to allow StorageOS or Zookeeper to handle replication is required.\nBefore you start, ensure you have StorageOS installed and ready on a Kubernetes cluster. See our guide on how to install StorageOS on Kubernetes for more information.\nDeploying Zookeeper on Kubernetes Pre-requisites StorageOS is assumed to have been installed; please check for the latest available version here. Helm To simplify the deployment of Zookeeper, we\u0026rsquo;ve used this Zookeeper helm chart (incubator) (version 1.2.2, app version 3.4.10) and rendered it into the example deployment files you can find in our GitHub repo.\nDeployment Clone the use cases repo You can find the latest files in the StorageOS use cases repository in /zookeeper/\ngit clone https://github.com/storageos/use-cases.git storageos-usecases StatefulSet defintion ---apiVersion:apps/v1beta1kind:StatefulSetmetadata:name:zookeeper...spec:replicas:3# \u0026lt;--- number of zookeeper pods...containers:- name:zookeeperimage:\u0026#34;gcr.io/google_samples/k8szk:v3\u0026#34;imagePullPolicy:IfNotPresent...volumeMounts:- name:datamountPath:/var/lib/zookeepervolumeClaimTemplates:- metadata:name:dataspec:accessModes:- \u0026#34;ReadWriteOnce\u0026#34;storageClassName:\u0026#34;fast\u0026#34;# \u0026lt;--- the StorageClass to useresources:requests:storage:\u0026#34;5Gi\u0026#34;# \u0026lt;--- storage requested per podThis excerpt is from the StatefulSet definition (10-statefulset.yaml). The file contains the PersistentVolumeClaim template that will dynamically provision the necessary storage, using the StorageOS storage class. Dynamic provisioning occurs as a volumeMount has been declared with the same name as a VolumeClaimTemplate.\nCreate the kubernetes objects cd storageos-usecases kubectl apply -f ./zookeeper/ Confirm Zookeeper is up and running $ kubectl get pods NAME READY STATUS RESTARTS AGE zookeeper-0 1/1 Running 0 2m30s zookeeper-1 1/1 Running 0 112s zookeeper-2 1/1 Running 0 56s zookeeper-test-client 1/1 Running 0 2m30s Connect to Zookeeper Connect to the zookeeper client pod and list existing topics using the service endpoint\nkubectl exec -it zookeeper-test-client /bin/bash and issue a command to the zookeeper service\nzkCli.sh -server zookeeper ls /zookeeper ","excerpt":"Apache Zookeeper with StorageOS ZooKeeper is a centralized service for maintaining configuration …","ref":"/v2/docs/usecases/zookeeper/","title":"Zookeeper"}]
\ No newline at end of file
diff --git a/v2/search/index.html b/v2/search/index.html
index 8039321fb..21c5e569b 100644
--- a/v2/search/index.html
+++ b/v2/search/index.html
@@ -187,7 +187,7 @@
aria-label="Search this site…"
autocomplete="off"
- data-offline-search-index-json-src="/v2/offline-search-index.77cab3d35f3a2d565d80b5860f1a344c43421ff1e8308a9a4413b5ca250aeec444ab7216406c8d2c55bb2c972c49852ce12ea01aaa2ca2a7c75e027222226be3.json"
+ data-offline-search-index-json-src="/v2/offline-search-index.045a6006b6b08e51b08cdf6eafc46226c816717b26d47e1bd4c2058c4e2c97243f8ad6e348d2ed3601c727409d72cb2cb4aa379adb7bb0443e3ade34df5f97fd.json"
data-offline-search-base-href="/"
>
diff --git a/v2/yaml/bundle-configuration.yaml b/v2/yaml/bundle-configuration.yaml
index f1c1004a0..5b21e3ed3 100644
--- a/v2/yaml/bundle-configuration.yaml
+++ b/v2/yaml/bundle-configuration.yaml
@@ -40,11 +40,18 @@ spec:
# InternalAPIPort = 5710 # Grpc API
# GossipPort = 5711 # Gossip+Healthcheck
- echo \"Source node for the test:\";
+ date;
+ echo \"Source node:\";
hostname -f -I; echo;
+ echo \"TCP:\";
parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
- ::: 5703 5704 5705 5710 5711
+ ::: 5703 5704 5705 5710 5711;
+ echo;
+
+ echo \"UDP:\";
+ parallel -j2 nc -vunz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
+ ::: 5711;
"
timeout: 90s
- exec:
diff --git a/yaml/bundle-configuration.yaml b/yaml/bundle-configuration.yaml
index f1c1004a0..5b21e3ed3 100644
--- a/yaml/bundle-configuration.yaml
+++ b/yaml/bundle-configuration.yaml
@@ -40,11 +40,18 @@ spec:
# InternalAPIPort = 5710 # Grpc API
# GossipPort = 5711 # Gossip+Healthcheck
- echo \"Source node for the test:\";
+ date;
+ echo \"Source node:\";
hostname -f -I; echo;
+ echo \"TCP:\";
parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
- ::: 5703 5704 5705 5710 5711
+ ::: 5703 5704 5705 5710 5711;
+ echo;
+
+ echo \"UDP:\";
+ parallel -j2 nc -vunz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \
+ ::: 5711;
"
timeout: 90s
- exec: