Merge pull request #180 from dceara/explicit-nodes-refactor

Refactor cluster bringup.

ovn-fake-multinode-utils/translate_yaml.py

@@ -26,28 +26,6 @@ class GlobalConfig:
     cms_name: str = ''
 
 
-def calculate_node_remotes(
-    node_net: str, clustered_db: bool, n_relays: int, enable_ssl: bool
-) -> str:
-    net = netaddr.IPNetwork(node_net)
-
-    ip_gen = net.iter_hosts()
-    # The first IP is assigned to the tester, skip it.
-    next(ip_gen)
-    if n_relays > 0:
-        skip = 3 if clustered_db else 1
-        for _ in range(0, skip):
-            next(ip_gen)
-        ip_range = range(0, n_relays)
-    else:
-        ip_range = range(0, 3 if clustered_db else 1)
-    if enable_ssl:
-        remotes = ["ssl:" + str(next(ip_gen)) + ":6642" for _ in ip_range]
-    else:
-        remotes = ["tcp:" + str(next(ip_gen)) + ":6642" for _ in ip_range]
-    return ','.join(remotes)
-
-
 DEFAULT_N_VIPS = 2
 DEFAULT_VIP_PORT = 80
 
@@ -121,7 +99,6 @@ class ClusterConfig:
     db_inactivity_probe: int = 60000
     node_net: str = "192.16.0.0/16"
     enable_ssl: bool = True
-    node_remote: str = None
     node_timeout_s: int = 20
     internal_net: str = "16.0.0.0/16"
     internal_net6: str = "16::/64"
@@ -143,14 +120,6 @@ class ClusterConfig:
 
     def __post_init__(self, **kwargs):
         # Some defaults have to be calculated
-        if not self.node_remote:
-            self.node_remote = calculate_node_remotes(
-                self.node_net,
-                self.clustered_db,
-                self.n_relays,
-                self.enable_ssl,
-            )
-
         if self.vips is None:
             self.vips = calculate_vips(self.vip_subnet)
 

ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py

@@ -1,5 +1,5 @@
 from ovn_context import Context
-from ovn_workload import WorkerNode, Cluster
+from ovn_workload import WorkerNode
 from ovn_utils import DualStackSubnet
 
 
@@ -8,30 +8,38 @@
 
 class OVNKubernetes:
     @staticmethod
-    def create_nodes(cluster_config, workers):
-        mgmt_net = cluster_config.node_net
-        mgmt_ip = mgmt_net.ip + 2
-        internal_net = cluster_config.internal_net
-        external_net = cluster_config.external_net
-        gw_net = cluster_config.gw_net
-        worker_nodes = [
-            WorkerNode(
-                workers[i % len(workers)],
-                f'ovn-scale-{i}',
-                mgmt_net,
-                mgmt_ip + i,
-                DualStackSubnet.next(internal_net, i),
-                DualStackSubnet.next(external_net, i),
-                gw_net,
-                i,
-            )
-            for i in range(cluster_config.n_workers)
-        ]
-        return worker_nodes
+    def add_cluster_worker_nodes(cluster, workers):
+        cluster_cfg = cluster.cluster_cfg
+
+        # Allocate worker IPs after central and relay IPs.
+        mgmt_ip = (
+            cluster_cfg.node_net.ip
+            + 2
+            + len(cluster.central_nodes)
+            + len(cluster.relay_nodes)
+        )
+
+        protocol = "ssl" if cluster_cfg.enable_ssl else "tcp"
+        internal_net = cluster_cfg.internal_net
+        external_net = cluster_cfg.external_net
+        gw_net = cluster_cfg.gw_net
+        cluster.add_workers(
+            [
+                WorkerNode(
+                    workers[i % len(workers)],
+                    f'ovn-scale-{i}',
+                    mgmt_ip + i,
+                    protocol,
+                    DualStackSubnet.next(internal_net, i),
+                    DualStackSubnet.next(external_net, i),
+                    gw_net,
+                    i,
+                )
+                for i in range(cluster_cfg.n_workers)
+            ]
+        )
 
     @staticmethod
-    def prepare_test(central_node, worker_nodes, cluster_cfg, brex_cfg):
-        ovn = Cluster(central_node, worker_nodes, cluster_cfg, brex_cfg)
-        with Context(ovn, 'prepare_test'):
-            ovn.start()
-        return ovn
+    def prepare_test(cluster):
+        with Context(cluster, 'prepare_test'):
+            cluster.start()

ovn-tester/cms/ovn_kubernetes/tests/base_cluster_bringup.py

@@ -7,8 +7,8 @@
 
 
 class BaseClusterBringup(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('base_cluster_bringup', dict())
         self.config = ClusterBringupCfg(
             n_pods_per_node=test_config.get('n_pods_per_node', 0),

ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py

@@ -17,8 +17,8 @@
 
 
 class ClusterDensity(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('cluster_density', dict())
         self.config = ClusterDensityCfg(
             n_runs=test_config.get('n_runs', 0),

ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py

@@ -23,8 +23,8 @@
 
 
 class DensityHeavy(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('density_heavy', dict())
         pods_vip_ratio = test_config.get(
             'pods_vip_ratio', DENSITY_PODS_VIP_RATIO

ovn-tester/cms/ovn_kubernetes/tests/density_light.py

@@ -10,8 +10,8 @@
 
 
 class DensityLight(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('density_light', dict())
         self.config = DensityCfg(
             n_pods=test_config.get('n_pods', 0),

ovn-tester/cms/ovn_kubernetes/tests/netpol.py

@@ -8,8 +8,8 @@
 
 
 class NetPol(ExtCmd):
-    def __init__(self, name, config, central_node, worker_nodes):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, name, config, cluster):
+        super().__init__(config, cluster)
         test_config = config.get(name, dict())
         self.config = NpCfg(
             n_ns=test_config.get('n_ns', 0),

ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py

@@ -7,8 +7,8 @@
 
 
 class NetpolCrossNs(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('netpol_cross', dict())
         self.config = NpCrossNsCfg(
             n_ns=test_config.get('n_ns', 0),

ovn-tester/cms/ovn_kubernetes/tests/netpol_large.py

@@ -2,8 +2,8 @@
 
 
 class NetpolLarge(NetPol):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__('netpol_large', config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__('netpol_large', config, cluster)
 
     def run(self, ovn, global_cfg):
         self.init(ovn, global_cfg)

ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py

@@ -15,8 +15,8 @@
 
 
 class NetpolMultitenant(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('netpol_multitenant', dict())
         ranges = [
             NsRange(

ovn-tester/cms/ovn_kubernetes/tests/netpol_small.py

@@ -2,8 +2,8 @@
 
 
 class NetpolSmall(NetPol):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__('netpol_small', config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__('netpol_small', config, cluster)
 
     def run(self, ovn, global_cfg):
         self.init(ovn, global_cfg)

ovn-tester/cms/ovn_kubernetes/tests/service_route.py

@@ -16,8 +16,8 @@
 
 
 class ServiceRoute(ExtCmd):
-    def __init__(self, config, central_node, worker_nodes, global_cfg):
-        super().__init__(config, central_node, worker_nodes)
+    def __init__(self, config, cluster, global_cfg):
+        super().__init__(config, cluster)
         test_config = config.get('service_route', dict())
         self.config = ServiceRouteCfg(
             n_lb=test_config.get('n_lb', 16),

ovn-tester/ovn_ext_cmd.py

@@ -1,18 +1,10 @@
 from collections import defaultdict
 from fnmatch import fnmatch
 from io import StringIO
-from ovn_sandbox import Sandbox
-
-
-# The wrapper allows us to execute the command on all
-# matching central containers
-class CentralNodeWrapper(Sandbox):
-    def __init__(self, central_node, container):
-        super().__init__(central_node.phys_node, container)
 
 
 class ExtCmdUnit:
-    def __init__(self, conf, central_node, worker_nodes):
+    def __init__(self, conf, cluster):
         self.iteration = conf.get('iteration')
         self.cmd = conf.get('cmd')
         self.test = conf.get('test')
@@ -21,13 +13,11 @@ def __init__(self, conf, central_node, worker_nodes):
         self.pid_opt = conf.get('pid_opt', '')
 
         node = conf.get('node')
-        self.nodes = [n for n in worker_nodes if fnmatch(n.container, node)]
+        self.nodes = [
+            n for n in cluster.worker_nodes if fnmatch(n.container, node)
+        ]
         self.nodes.extend(
-            [
-                CentralNodeWrapper(central_node, c)
-                for c in central_node.central_containers()
-                if fnmatch(c, node)
-            ]
+            [n for n in cluster.central_nodes if fnmatch(n.container, node)]
         )
 
     def is_valid(self):
@@ -58,10 +48,10 @@ def _node_exec(self, node):
 
 
 class ExtCmd:
-    def __init__(self, config, central_node, worker_nodes):
+    def __init__(self, config, cluster):
         self.cmd_map = defaultdict(list)
         for ext_cmd in config.get('ext_cmd', list()):
-            cmd_unit = ExtCmdUnit(ext_cmd, central_node, worker_nodes)
+            cmd_unit = ExtCmdUnit(ext_cmd, cluster)
             if cmd_unit.is_valid():
                 self.cmd_map[(cmd_unit.iteration, cmd_unit.test)].append(
                     cmd_unit

ovn-tester/ovn_tester.py

@@ -11,7 +11,13 @@
 
 from collections import namedtuple
 from ovn_sandbox import PhysicalNode
-from ovn_workload import BrExConfig, CentralNode, ClusterConfig
+from ovn_workload import (
+    BrExConfig,
+    CentralNode,
+    Cluster,
+    ClusterConfig,
+    RelayNode,
+)
 from ovn_utils import DualStackSubnet
 from ovs.stream import Stream
 
@@ -68,7 +74,6 @@ def read_config(config):
         node_net=netaddr.IPNetwork(cluster_args['node_net']),
         n_relays=cluster_args['n_relays'],
         enable_ssl=cluster_args['enable_ssl'],
-        node_remote=cluster_args['node_remote'],
         northd_probe_interval=cluster_args['northd_probe_interval'],
         db_inactivity_probe=cluster_args['db_inactivity_probe'],
         node_timeout_s=cluster_args['node_timeout_s'],
@@ -167,7 +172,7 @@ def load_cms(cms_name):
     return cls()
 
 
-def configure_tests(yaml, central_node, worker_nodes, global_cfg):
+def configure_tests(yaml, cluster, global_cfg):
     tests = []
     for section, cfg in yaml.items():
         if section in RESERVED:
@@ -178,25 +183,30 @@ def configure_tests(yaml, central_node, worker_nodes, global_cfg):
         )
         class_name = ''.join(s.title() for s in section.split('_'))
         cls = getattr(mod, class_name)
-        tests.append(cls(yaml, central_node, worker_nodes, global_cfg))
+        tests.append(cls(yaml, cluster, global_cfg))
     return tests
 
 
-def create_central_nodes(cluster_config, central):
-    mgmt_net = cluster_config.node_net
-    mgmt_ip = mgmt_net.ip + 2
+def create_cluster(cluster_cfg, central, brex_cfg):
+    protocol = "ssl" if cluster_cfg.enable_ssl else "tcp"
+    mgmt_ip = cluster_cfg.node_net.ip + 2
     db_containers = (
         ['ovn-central-1', 'ovn-central-2', 'ovn-central-3']
-        if cluster_config.clustered_db
+        if cluster_cfg.clustered_db
         else ['ovn-central']
     )
-    relay_containers = [
-        f'ovn-relay-{i + 1}' for i in range(cluster_config.n_relays)
+
+    central_nodes = [
+        CentralNode(central, c, mgmt_ip + i, protocol)
+        for i, c in enumerate(db_containers)
     ]
-    central_node = CentralNode(
-        central, db_containers, relay_containers, mgmt_net, mgmt_ip
-    )
-    return central_node
+    mgmt_ip += len(central_nodes)
+
+    relay_nodes = [
+        RelayNode(central, f'ovn-relay-{i + 1}', mgmt_ip + i, protocol)
+        for i in range(cluster_cfg.n_relays)
+    ]
+    return Cluster(central_nodes, relay_nodes, cluster_cfg, brex_cfg)
 
 
 def set_ssl_keys(cluster_cfg):
@@ -225,14 +235,14 @@ def set_ssl_keys(cluster_cfg):
     cms = load_cms(global_cfg.cms_name)
 
     central, workers = read_physical_deployment(sys.argv[1], global_cfg)
-    central_node = create_central_nodes(cluster_cfg, central)
-    worker_nodes = cms.create_nodes(cluster_cfg, workers)
-    tests = configure_tests(config, central_node, worker_nodes, global_cfg)
+    cluster = create_cluster(cluster_cfg, central, brex_cfg)
+    cms.add_cluster_worker_nodes(cluster, workers)
+    tests = configure_tests(config, cluster, global_cfg)
 
     if cluster_cfg.enable_ssl:
         set_ssl_keys(cluster_cfg)
 
-    ovn = cms.prepare_test(central_node, worker_nodes, cluster_cfg, brex_cfg)
+    cms.prepare_test(cluster)
     for test in tests:
-        test.run(ovn, global_cfg)
+        test.run(cluster, global_cfg)
     sys.exit(0)