Test 2 Ceph clusters to ensure both are read/writable

src/manifests_base.py

@@ -1,7 +1,8 @@
 import logging
 import pickle
 from hashlib import md5
-from typing import Any, Dict, Generator, List, Optional
+from pathlib import Path
+from typing import Any, Dict, Generator, List, Optional, Tuple
 
 from lightkube.codecs import AnyResource
 from lightkube.core.resource import NamespacedResource
@@ -251,3 +252,65 @@ def from_space_separated(cls, tolerations: str) -> List["CephToleration"]:
             return [cls._from_string(toleration) for toleration in tolerations.split()]
         except ValueError as e:
             raise ValueError(f"Invalid tolerations: {e}") from e
+
+
+class ProvisionerAdjustments(Patch):
+    """Update provisioner manifest objects."""
+
+    PROVISIONER_NAME: str
+    PLUGIN_NAME: str
+    PLUGIN_PATH: str
+
+    def tolerations(self) -> Tuple[List[CephToleration], bool]:
+        return [], False
+
+    def __call__(self, obj: AnyResource) -> None:
+        """Use the provisioner-replicas and enable-host-networking to update obj."""
+        tolerations, legacy = self.tolerations()
+        if (
+            obj.kind == "Deployment"
+            and obj.metadata
+            and obj.metadata.name == self.PROVISIONER_NAME
+        ):
+            obj.spec.replicas = replica = self.manifests.config.get("provisioner-replicas")
+            log.info(f"Updating deployment replicas to {replica}")
+
+            obj.spec.template.spec.tolerations = tolerations
+            log.info("Updating deployment tolerations")
+
+            obj.spec.template.spec.hostNetwork = host_network = self.manifests.config.get(
+                "enable-host-networking"
+            )
+            log.info(f"Updating deployment hostNetwork to {host_network}")
+
+        if obj.kind == "DaemonSet" and obj.metadata and obj.metadata.name == self.PLUGIN_NAME:
+            log.info("Updating daemonset tolerations")
+            obj.spec.template.spec.tolerations = (
+                tolerations if not legacy else [CephToleration(operator="Exists")]
+            )
+
+            log.info("Updating plugin paths")
+            kubelet_dir = self.manifests.config.get("kubelet_dir", "/var/lib/kubelet")
+            app = self.manifests.model.app.name
+
+            for c in obj.spec.template.spec.containers:
+                c.args = [arg.replace("/var/lib/kubelet", kubelet_dir) for arg in c.args]
+                for m in c.volumeMounts:
+                    m.mountPath = m.mountPath.replace("/var/lib/kubelet", kubelet_dir)
+                if c.name == "driver-registrar":
+                    path = Path(
+                        f"/{kubelet_dir}/plugins/{self.PLUGIN_PATH}/{app}/csi.sock"
+                    ).resolve()
+                    for match_idx, arg in enumerate(c.args):
+                        if arg.startswith("--kubelet-registration-path="):
+                            break
+                    else:
+                        raise ValueError("Missing --kubelet-registration-path argument")
+                    c.args[match_idx] = f"--kubelet-registration-path={path}"
+
+            for v in obj.spec.template.spec.volumes:
+                if v.hostPath:
+                    v.hostPath.path = v.hostPath.path.replace("/var/lib/kubelet", kubelet_dir)
+                if v.name == "socket-dir":
+                    v.hostPath.path += f"/{app}"
+            log.info(f"Updating daemonset kubeletDir to {kubelet_dir}")

src/manifests_cephfs.py

@@ -9,14 +9,15 @@
 from lightkube.codecs import AnyResource
 from lightkube.resources.core_v1 import Secret
 from lightkube.resources.storage_v1 import StorageClass
-from ops.manifests import Addition, ConfigRegistry, Patch
+from ops.manifests import Addition, ConfigRegistry
 from ops.manifests.manipulations import Subtraction
 
 from manifests_base import (
     AdjustNamespace,
     CephToleration,
     ConfigureLivenessPrometheus,
     ManifestLabelExcluder,
+    ProvisionerAdjustments,
     RbacAdjustments,
     SafeManifest,
     StorageClassFactory,
@@ -200,53 +201,19 @@ def __call__(self) -> List[AnyResource]:
         return [self.create(class_param) for class_param in parameter_list]
 
 
-class ProvisionerAdjustments(Patch):
+class FSProvAdjustments(ProvisionerAdjustments):
     """Update Cephfs provisioner."""
 
     PROVISIONER_NAME = "csi-cephfsplugin-provisioner"
     PLUGIN_NAME = "csi-cephfsplugin"
+    PLUGIN_PATH = "cephfs.csi.ceph.com"
 
     def tolerations(self) -> Tuple[List[CephToleration], bool]:
         cfg = self.manifests.config.get("cephfs-tolerations") or ""
         if cfg == "$csi-cephfsplugin-legacy$":
             return [], True
         return CephToleration.from_space_separated(cfg), False
 
-    def __call__(self, obj: AnyResource) -> None:
-        """Use the provisioner-replicas and enable-host-networking to update obj."""
-        tolerations, legacy = self.tolerations()
-        if (
-            obj.kind == "Deployment"
-            and obj.metadata
-            and obj.metadata.name == self.PROVISIONER_NAME
-        ):
-            obj.spec.replicas = replica = self.manifests.config.get("provisioner-replicas")
-            log.info(f"Updating deployment replicas to {replica}")
-
-            obj.spec.template.spec.tolerations = tolerations
-            log.info("Updating deployment tolerations")
-
-            obj.spec.template.spec.hostNetwork = host_network = self.manifests.config.get(
-                "enable-host-networking"
-            )
-            log.info(f"Updating deployment hostNetwork to {host_network}")
-        if obj.kind == "DaemonSet" and obj.metadata and obj.metadata.name == self.PLUGIN_NAME:
-            obj.spec.template.spec.tolerations = (
-                tolerations if not legacy else [CephToleration(operator="Exists")]
-            )
-            log.info("Updating daemonset tolerations")
-
-            kubelet_dir = self.manifests.config.get("kubelet_dir", "/var/lib/kubelet")
-
-            for c in obj.spec.template.spec.containers:
-                c.args = [arg.replace("/var/lib/kubelet", kubelet_dir) for arg in c.args]
-                for m in c.volumeMounts:
-                    m.mountPath = m.mountPath.replace("/var/lib/kubelet", kubelet_dir)
-            for v in obj.spec.template.spec.volumes:
-                if v.hostPath:
-                    v.hostPath.path = v.hostPath.path.replace("/var/lib/kubelet", kubelet_dir)
-            log.info(f"Updating daemonset kubeletDir to {kubelet_dir}")
-
 
 class RemoveCephFS(Subtraction):
     """Remove all Cephfs resources when disabled."""
@@ -268,7 +235,7 @@ def __init__(self, charm: "CephCsiCharm"):
                 StorageSecret(self),
                 ManifestLabelExcluder(self),
                 ConfigRegistry(self),
-                ProvisionerAdjustments(self),
+                FSProvAdjustments(self),
                 CephStorageClass(self, STORAGE_TYPE),
                 RbacAdjustments(self),
                 RemoveCephFS(self),

src/manifests_rbd.py

@@ -3,18 +3,19 @@
 """Implementation of rbd specific details of the kubernetes manifests."""
 
 import logging
-from typing import TYPE_CHECKING, Dict, List, Optional, cast
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast
 
 from lightkube.codecs import AnyResource
 from lightkube.resources.core_v1 import Secret
 from lightkube.resources.storage_v1 import StorageClass
-from ops.manifests import Addition, ConfigRegistry, Patch
+from ops.manifests import Addition, ConfigRegistry
 
 from manifests_base import (
     AdjustNamespace,
     CephToleration,
     ConfigureLivenessPrometheus,
     ManifestLabelExcluder,
+    ProvisionerAdjustments,
     RbacAdjustments,
     SafeManifest,
     StorageClassFactory,
@@ -104,49 +105,16 @@ def __call__(self) -> Optional[AnyResource]:
         )
 
 
-class ProvisionerAdjustments(Patch):
+class RBDProvAdjustments(ProvisionerAdjustments):
     """Update RBD provisioner."""
 
     PROVISIONER_NAME = "csi-rbdplugin-provisioner"
     PLUGIN_NAME = "csi-rbdplugin"
+    PLUGIN_PATH = "rbd.csi.ceph.com"
 
-    def tolerations(self) -> List[CephToleration]:
+    def tolerations(self) -> Tuple[List[CephToleration], bool]:
         cfg = self.manifests.config.get("ceph-rbd-tolerations") or ""
-        return CephToleration.from_space_separated(cfg)
-
-    def __call__(self, obj: AnyResource) -> None:
-        """Mutates CSI RBD Provisioner Deployment replicas/hostNetwork and DaemonSet kubelet_dir paths."""
-        tolerations = self.tolerations()
-        if (
-            obj.kind == "Deployment"
-            and obj.metadata
-            and obj.metadata.name == self.PROVISIONER_NAME
-        ):
-            obj.spec.replicas = replica = self.manifests.config.get("provisioner-replicas")
-            log.info(f"Updating deployment replicas to {replica}")
-
-            obj.spec.template.spec.tolerations = tolerations
-            log.info("Updating deployment tolerations")
-
-            obj.spec.template.spec.hostNetwork = host_network = self.manifests.config.get(
-                "enable-host-networking"
-            )
-            log.info(f"Updating deployment hostNetwork to {host_network}")
-
-        if obj.kind == "DaemonSet" and obj.metadata and obj.metadata.name == self.PLUGIN_NAME:
-            obj.spec.template.spec.tolerations = tolerations
-            log.info("Updating daemonset tolerations to operator=Exists")
-
-            kubelet_dir = self.manifests.config.get("kubelet_dir", "/var/lib/kubelet")
-
-            for c in obj.spec.template.spec.containers:
-                c.args = [arg.replace("/var/lib/kubelet", kubelet_dir) for arg in c.args]
-                for m in c.volumeMounts:
-                    m.mountPath = m.mountPath.replace("/var/lib/kubelet", kubelet_dir)
-            for v in obj.spec.template.spec.volumes:
-                if v.hostPath:
-                    v.hostPath.path = v.hostPath.path.replace("/var/lib/kubelet", kubelet_dir)
-            log.info(f"Updating RBD daemonset kubeletDir to {kubelet_dir}")
+        return CephToleration.from_space_separated(cfg), False
 
 
 class RBDManifests(SafeManifest):
@@ -161,7 +129,7 @@ def __init__(self, charm: "CephCsiCharm"):
                 StorageSecret(self),
                 ManifestLabelExcluder(self),
                 ConfigRegistry(self),
-                ProvisionerAdjustments(self),
+                RBDProvAdjustments(self),
                 CephStorageClass(self, "ceph-xfs"),  # creates ceph-xfs
                 CephStorageClass(self, "ceph-ext4"),  # creates ceph-ext4
                 RbacAdjustments(self),

terraform/versions.tf

@@ -7,7 +7,7 @@ terraform {
   required_providers {
     juju = {
       source  = "juju/juju"
-      version = "~> 0.14.0"
+      version = ">= 0.14.0, < 1.0.0"
     }
   }
 }

tests/functional/overlay.yaml

@@ -11,23 +11,25 @@ applications:
       allow-privileged: "true"
   kubernetes-worker:
     expose: true
+
+  # Primary Ceph Cluster
   ceph-fs:
     charm: ceph-fs
     channel: "quincy/stable"
     num_units: 1
-    constraints: "cores=2 mem=4G root-disk=16G"
+    constraints: "cores=2 mem=4G root-disk=4G"
   ceph-mon:
     charm: ceph-mon
     channel: "quincy/stable"
     num_units: 1
-    constraints: "cores=2 mem=4G root-disk=16G"
+    constraints: "cores=2 mem=4G root-disk=4G"
     options:
       monitor-count: '1'
   ceph-osd:
     charm: ceph-osd
     channel: "quincy/stable"
     num_units: 2
-    constraints: "cores=2 mem=4G root-disk=16G"
+    constraints: "cores=2 mem=4G root-disk=4G"
     options:
       osd-devices: /srv/osd
     storage:
@@ -39,6 +41,30 @@ applications:
       provisioner-replicas: 1
       namespace: {{ namespace }}
       ceph-rbac-name-formatter: '{name}-formatter'
+
+  # Secondary Ceph Cluster
+  ceph-fs-alt:
+    charm: ceph-fs
+    channel: "quincy/stable"
+    num_units: 1
+    constraints: "cores=2 mem=4G root-disk=4G"
+  ceph-mon-alt:
+    charm: ceph-mon
+    channel: "quincy/stable"
+    num_units: 1
+    constraints: "cores=2 mem=4G root-disk=4G"
+    options:
+      monitor-count: '1'
+  ceph-osd-alt:
+    charm: ceph-osd
+    channel: "quincy/stable"
+    num_units: 2
+    constraints: "cores=2 mem=4G root-disk=4G"
+    options:
+      osd-devices: /srv/osd
+    storage:
+      osd-devices: 1G,2
+      osd-journals: 1G,1
   ceph-csi-alt:
     # This is an alternative ceph-csi charm that is used to test
     # the ability to deploy multiple ceph-csi charms in the same
@@ -55,5 +81,9 @@ relations:
 - [ceph-osd,     ceph-mon]
 - [ceph-fs,      ceph-mon]
 - [ceph-csi,     ceph-mon:client]
-- [ceph-csi-alt, ceph-mon:client]
+
+- [ceph-osd-alt, ceph-mon-alt]
+- [ceph-fs-alt,  ceph-mon-alt]
+- [ceph-csi-alt, ceph-mon-alt:client]
+
 - [ceph-csi,     kubernetes-control-plane]

tests/functional/test_ceph_csi.py

@@ -309,6 +309,11 @@ async def test_duplicate_ceph_csi(ops_test: OpsTest, namespace: str, kube_config
     try:
         await app.relate("kubernetes", "kubernetes-control-plane")
         await ops_test.model.wait_for_idle(apps=[CEPH_CSI_ALT], status="active", timeout=5 * 60)
+
+        await run_test_storage_class(kube_config, f"ceph-xfs-{namespace}")
+        await run_test_storage_class(kube_config, "ceph-xfs")
+        await run_test_storage_class(kube_config, f"ceph-ext4-{namespace}")
+        await run_test_storage_class(kube_config, "ceph-ext4")
     finally:
         await app.destroy_relation("kubernetes", "kubernetes-control-plane:juju-info")
         await ops_test.model.wait_for_idle(

tests/unit/test_manifests_cephfs.py

@@ -13,7 +13,7 @@
     CephFilesystem,
     CephFSManifests,
     CephStorageClass,
-    ProvisionerAdjustments,
+    FSProvAdjustments,
     StorageSecret,
 )
 
@@ -36,8 +36,8 @@
 upstream_path = Path(__file__).parent.parent.parent / "upstream"
 cephfs_path = upstream_path / "cephfs"
 current_path = cephfs_path / "manifests" / (cephfs_path / "version").read_text().strip()
-provisioner_path = current_path / (ProvisionerAdjustments.PROVISIONER_NAME + ".yaml")
-plugin_path = current_path / (ProvisionerAdjustments.PLUGIN_NAME + ".yaml")
+provisioner_path = current_path / (FSProvAdjustments.PROVISIONER_NAME + ".yaml")
+plugin_path = current_path / (FSProvAdjustments.PLUGIN_NAME + ".yaml")
 
 
 def test_storage_secret_modelled(caplog):
@@ -82,7 +82,7 @@ def test_ceph_provisioner_adjustment_modelled(caplog):
         "enable-host-networking": False,
         "kubelet_dir": alt_path,
     }
-    cpa = ProvisionerAdjustments(manifest)
+    cpa = FSProvAdjustments(manifest)
     resources = list(yaml.safe_load_all(provisioner_path.read_text()))
     resource = Deployment.from_dict(resources[1])
     assert cpa(resource) is None