From 3bb9c69f5406bd08bdcbc3f06edb6768ba85af22 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 20:53:27 +0200 Subject: [PATCH 1/8] ovn-tester: Remove unused mgmt_net fields. Signed-off-by: Dumitru Ceara --- ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py | 4 +--- ovn-tester/ovn_tester.py | 5 ++--- ovn-tester/ovn_workload.py | 12 ++++-------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index db672b55..8d893c25 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -9,8 +9,7 @@ class OVNKubernetes: @staticmethod def create_nodes(cluster_config, workers): - mgmt_net = cluster_config.node_net - mgmt_ip = mgmt_net.ip + 2 + mgmt_ip = cluster_config.node_net.ip + 2 internal_net = cluster_config.internal_net external_net = cluster_config.external_net gw_net = cluster_config.gw_net @@ -18,7 +17,6 @@ def create_nodes(cluster_config, workers): WorkerNode( workers[i % len(workers)], f'ovn-scale-{i}', - mgmt_net, mgmt_ip + i, DualStackSubnet.next(internal_net, i), DualStackSubnet.next(external_net, i), diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index 17e355f9..0b909f00 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -183,8 +183,7 @@ def configure_tests(yaml, central_node, worker_nodes, global_cfg): def create_central_nodes(cluster_config, central): - mgmt_net = cluster_config.node_net - mgmt_ip = mgmt_net.ip + 2 + mgmt_ip = cluster_config.node_net.ip + 2 db_containers = ( ['ovn-central-1', 'ovn-central-2', 'ovn-central-3'] if cluster_config.clustered_db @@ -194,7 +193,7 @@ def create_central_nodes(cluster_config, central): f'ovn-relay-{i + 1}' for i in range(cluster_config.n_relays) ] central_node = CentralNode( - central, db_containers, relay_containers, mgmt_net, mgmt_ip + central, db_containers, relay_containers, mgmt_ip ) return central_node diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 2a531e14..781d9f2c 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -53,18 +53,15 @@ class Node(ovn_sandbox.Sandbox): - def __init__(self, phys_node, container, mgmt_net, mgmt_ip): + def __init__(self, phys_node, container, mgmt_ip): super().__init__(phys_node, container) self.container = container - self.mgmt_net = mgmt_net self.mgmt_ip = mgmt_ip class CentralNode(Node): - def __init__( - self, phys_node, db_containers, relay_containers, mgmt_net, mgmt_ip - ): - super().__init__(phys_node, db_containers[0], mgmt_net, mgmt_ip) + def __init__(self, phys_node, db_containers, relay_containers, mgmt_ip): + super().__init__(phys_node, db_containers[0], mgmt_ip) self.db_containers = db_containers self.relay_containers = relay_containers @@ -155,14 +152,13 @@ def __init__( self, phys_node, container, - mgmt_net, mgmt_ip, int_net, ext_net, gw_net, unique_id, ): - super().__init__(phys_node, container, mgmt_net, mgmt_ip) + super().__init__(phys_node, container, mgmt_ip) self.int_net = int_net self.ext_net = ext_net self.gw_net = gw_net From 5a97782cd16afc24a8fb619bc3b40f892c7bae17 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 23:04:51 +0200 Subject: [PATCH 2/8] treewide: Use cluster_cfg instead of cluster_config. It was confusing to have both and the majority of cases used 'cluster_cfg' instead of 'cluster_config' (100 vs 10). Signed-off-by: Dumitru Ceara --- ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py | 12 ++++++------ ovn-tester/ovn_tester.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 8d893c25..3fe8fdd8 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -8,11 +8,11 @@ class OVNKubernetes: @staticmethod - def create_nodes(cluster_config, workers): - mgmt_ip = cluster_config.node_net.ip + 2 - internal_net = cluster_config.internal_net - external_net = cluster_config.external_net - gw_net = cluster_config.gw_net + def create_nodes(cluster_cfg, workers): + mgmt_ip = cluster_cfg.node_net.ip + 2 + internal_net = cluster_cfg.internal_net + external_net = cluster_cfg.external_net + gw_net = cluster_cfg.gw_net worker_nodes = [ WorkerNode( workers[i % len(workers)], @@ -23,7 +23,7 @@ def create_nodes(cluster_config, workers): gw_net, i, ) - for i in range(cluster_config.n_workers) + for i in range(cluster_cfg.n_workers) ] return worker_nodes diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index 0b909f00..1aac5d8c 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -182,15 +182,15 @@ def configure_tests(yaml, central_node, worker_nodes, global_cfg): return tests -def create_central_nodes(cluster_config, central): - mgmt_ip = cluster_config.node_net.ip + 2 +def create_central_nodes(cluster_cfg, central): + mgmt_ip = cluster_cfg.node_net.ip + 2 db_containers = ( ['ovn-central-1', 'ovn-central-2', 'ovn-central-3'] - if cluster_config.clustered_db + if cluster_cfg.clustered_db else ['ovn-central'] ) relay_containers = [ - f'ovn-relay-{i + 1}' for i in range(cluster_config.n_relays) + f'ovn-relay-{i + 1}' for i in range(cluster_cfg.n_relays) ] central_node = CentralNode( central, db_containers, relay_containers, mgmt_ip From 87e6653aa4e7f7de0a98634206effd730d7dff60 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 22:42:37 +0200 Subject: [PATCH 3/8] ovn-tester: Store a copy of the mgmt IP when creating new nodes. Storing a reference will have unexpected results as we might end up sharing it between different nodes. Signed-off-by: Dumitru Ceara --- ovn-tester/ovn_workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 781d9f2c..00a109f7 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -56,7 +56,7 @@ class Node(ovn_sandbox.Sandbox): def __init__(self, phys_node, container, mgmt_ip): super().__init__(phys_node, container) self.container = container - self.mgmt_ip = mgmt_ip + self.mgmt_ip = netaddr.IPAddress(mgmt_ip) class CentralNode(Node): From ec0a73cfd0daaa93ebb540a86e8a34d53f0ca057 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 23:36:51 +0200 Subject: [PATCH 4/8] ovn-tester: Fix WorkerNode mgmt IP allocation. The IPs were skewed by 2 for clustered DB deployments. Instead of mangling the connection string generation just allocate the correct IP. Signed-off-by: Dumitru Ceara --- ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py | 4 ++++ ovn-tester/ovn_workload.py | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 3fe8fdd8..0728da1e 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -9,7 +9,11 @@ class OVNKubernetes: @staticmethod def create_nodes(cluster_cfg, workers): + # Allocate worker IPs after central and relay IPs. mgmt_ip = cluster_cfg.node_net.ip + 2 + mgmt_ip += 3 if cluster_cfg.clustered_db else 1 + mgmt_ip += cluster_cfg.n_relays + internal_net = cluster_cfg.internal_net external_net = cluster_cfg.external_net gw_net = cluster_cfg.gw_net diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 00a109f7..a6fad775 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -433,10 +433,7 @@ def ping_ports(self, cluster, ports): def get_connection_string(self, cluster_cfg, port): protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - offset = 0 - offset += 3 if cluster_cfg.clustered_db else 1 - offset += cluster_cfg.n_relays - return f"{protocol}:{self.mgmt_ip + offset}:{port}" + return f"{protocol}:{self.mgmt_ip}:{port}" ACL_DEFAULT_DENY_PRIO = 1 From 348028ae70d998924f1b061e906555306a013f15 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 00:47:28 +0200 Subject: [PATCH 5/8] ovn-tester: Add explicit RelayNode class. It removes the need to artificially compute remote values. Instead just use the correct RelayNode IPs. Signed-off-by: Dumitru Ceara --- ovn-fake-multinode-utils/translate_yaml.py | 31 ---------- .../cms/ovn_kubernetes/ovn_kubernetes.py | 8 ++- ovn-tester/ovn_tester.py | 22 ++++--- ovn-tester/ovn_workload.py | 61 +++++++++++++------ 4 files changed, 62 insertions(+), 60 deletions(-) diff --git a/ovn-fake-multinode-utils/translate_yaml.py b/ovn-fake-multinode-utils/translate_yaml.py index 56b0b312..a1fcdfa2 100755 --- a/ovn-fake-multinode-utils/translate_yaml.py +++ b/ovn-fake-multinode-utils/translate_yaml.py @@ -26,28 +26,6 @@ class GlobalConfig: cms_name: str = '' -def calculate_node_remotes( - node_net: str, clustered_db: bool, n_relays: int, enable_ssl: bool -) -> str: - net = netaddr.IPNetwork(node_net) - - ip_gen = net.iter_hosts() - # The first IP is assigned to the tester, skip it. - next(ip_gen) - if n_relays > 0: - skip = 3 if clustered_db else 1 - for _ in range(0, skip): - next(ip_gen) - ip_range = range(0, n_relays) - else: - ip_range = range(0, 3 if clustered_db else 1) - if enable_ssl: - remotes = ["ssl:" + str(next(ip_gen)) + ":6642" for _ in ip_range] - else: - remotes = ["tcp:" + str(next(ip_gen)) + ":6642" for _ in ip_range] - return ','.join(remotes) - - DEFAULT_N_VIPS = 2 DEFAULT_VIP_PORT = 80 @@ -121,7 +99,6 @@ class ClusterConfig: db_inactivity_probe: int = 60000 node_net: str = "192.16.0.0/16" enable_ssl: bool = True - node_remote: str = None node_timeout_s: int = 20 internal_net: str = "16.0.0.0/16" internal_net6: str = "16::/64" @@ -143,14 +120,6 @@ class ClusterConfig: def __post_init__(self, **kwargs): # Some defaults have to be calculated - if not self.node_remote: - self.node_remote = calculate_node_remotes( - self.node_net, - self.clustered_db, - self.n_relays, - self.enable_ssl, - ) - if self.vips is None: self.vips = calculate_vips(self.vip_subnet) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 0728da1e..126d27a6 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -32,8 +32,12 @@ def create_nodes(cluster_cfg, workers): return worker_nodes @staticmethod - def prepare_test(central_node, worker_nodes, cluster_cfg, brex_cfg): - ovn = Cluster(central_node, worker_nodes, cluster_cfg, brex_cfg) + def prepare_test( + central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + ): + ovn = Cluster( + central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + ) with Context(ovn, 'prepare_test'): ovn.start() return ovn diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index 1aac5d8c..0cdc0713 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -11,7 +11,7 @@ from collections import namedtuple from ovn_sandbox import PhysicalNode -from ovn_workload import BrExConfig, CentralNode, ClusterConfig +from ovn_workload import BrExConfig, CentralNode, ClusterConfig, RelayNode from ovn_utils import DualStackSubnet from ovs.stream import Stream @@ -68,7 +68,6 @@ def read_config(config): node_net=netaddr.IPNetwork(cluster_args['node_net']), n_relays=cluster_args['n_relays'], enable_ssl=cluster_args['enable_ssl'], - node_remote=cluster_args['node_remote'], northd_probe_interval=cluster_args['northd_probe_interval'], db_inactivity_probe=cluster_args['db_inactivity_probe'], node_timeout_s=cluster_args['node_timeout_s'], @@ -189,13 +188,14 @@ def create_central_nodes(cluster_cfg, central): if cluster_cfg.clustered_db else ['ovn-central'] ) - relay_containers = [ - f'ovn-relay-{i + 1}' for i in range(cluster_cfg.n_relays) + central_node = CentralNode(central, db_containers, mgmt_ip) + mgmt += 1 + + relay_nodes = [ + RelayNode(central, f'ovn-relay-{i + 1}', mgmt_ip + i) + for i in range(cluster_cfg.n_relays) ] - central_node = CentralNode( - central, db_containers, relay_containers, mgmt_ip - ) - return central_node + return central_node, relay_nodes def set_ssl_keys(cluster_cfg): @@ -224,14 +224,16 @@ def set_ssl_keys(cluster_cfg): cms = load_cms(global_cfg.cms_name) central, workers = read_physical_deployment(sys.argv[1], global_cfg) - central_node = create_central_nodes(cluster_cfg, central) + central_node, relay_nodes = create_central_nodes(cluster_cfg, central) worker_nodes = cms.create_nodes(cluster_cfg, workers) tests = configure_tests(config, central_node, worker_nodes, global_cfg) if cluster_cfg.enable_ssl: set_ssl_keys(cluster_cfg) - ovn = cms.prepare_test(central_node, worker_nodes, cluster_cfg, brex_cfg) + ovn = cms.prepare_test( + central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + ) for test in tests: test.run(ovn, global_cfg) sys.exit(0) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index a6fad775..fecca058 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -28,7 +28,6 @@ 'db_inactivity_probe', 'node_net', 'enable_ssl', - 'node_remote', 'node_timeout_s', 'internal_net', 'external_net', @@ -60,10 +59,9 @@ def __init__(self, phys_node, container, mgmt_ip): class CentralNode(Node): - def __init__(self, phys_node, db_containers, relay_containers, mgmt_ip): + def __init__(self, phys_node, db_containers, mgmt_ip): super().__init__(phys_node, db_containers[0], mgmt_ip) self.db_containers = db_containers - self.relay_containers = relay_containers def start(self, cluster_cfg): log.info('Configuring central node') @@ -110,12 +108,6 @@ def enable_trim_on_compaction(self): f'/run/ovn/ovnsb_db.ctl ' f'ovsdb-server/memory-trim-on-compaction on' ) - for relay_container in self.relay_containers: - self.phys_node.run( - f'podman exec {relay_container} ovs-appctl -t ' - f'/run/ovn/ovnsb_db.ctl ' - f'ovsdb-server/memory-trim-on-compaction on' - ) def enable_txns_db_logging(self): log.info('Enable DB txn logging') @@ -147,6 +139,27 @@ def central_containers(self): return self.db_containers +class RelayNode(Node): + def __init__(self, phys_node, container, mgmt_ip): + super().__init__(phys_node, container, mgmt_ip) + + def start(self): + log.info(f'Configuring relay node {self.container}') + self.enable_trim_on_compaction() + + def get_connection_string(self, cluster_cfg, port): + protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" + return f'{protocol}:{self.mgmt_ip}:{port}' + + def enable_trim_on_compaction(self): + log.info('Setting DB trim-on-compaction') + self.phys_node.run( + f'podman exec {self.container} ovs-appctl -t ' + f'/run/ovn/ovnsb_db.ctl ' + f'ovsdb-server/memory-trim-on-compaction on' + ) + + class WorkerNode(Node): def __init__( self, @@ -178,14 +191,11 @@ def start(self, cluster_cfg): ) @ovn_stats.timeit - def connect(self, cluster_cfg): + def connect(self, remote): log.info( - f'Connecting worker {self.container}: ' - f'ovn-remote = {cluster_cfg.node_remote}' - ) - self.vsctl.set_global_external_id( - 'ovn-remote', f'{cluster_cfg.node_remote}' + f'Connecting worker {self.container}: ' f'ovn-remote = {remote}' ) + self.vsctl.set_global_external_id('ovn-remote', f'{remote}') def configure_localnet(self, physical_net): log.info(f'Creating localnet on {self.container}') @@ -210,7 +220,7 @@ def wait(self, sbctl, timeout_s): @ovn_stats.timeit def provision(self, cluster): - self.connect(cluster.cluster_cfg) + self.connect(cluster.get_relay_connection_string()) self.wait(cluster.sbctl, cluster.cluster_cfg.node_timeout_s) # Create a node switch and connect it to the cluster router. @@ -737,9 +747,12 @@ def provision_vips_to_load_balancers(self, backend_lists, version): class Cluster: - def __init__(self, central_node, worker_nodes, cluster_cfg, brex_cfg): + def __init__( + self, central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + ): # In clustered mode use the first node for provisioning. self.central_node = central_node + self.relay_nodes = relay_nodes self.worker_nodes = worker_nodes self.cluster_cfg = cluster_cfg self.brex_cfg = brex_cfg @@ -769,6 +782,10 @@ def start(self): self.sbctl = ovn_utils.OvnSbctl( self.central_node, sb_conn, inactivity_probe ) + + for r in self.relay_nodes: + r.start() + for w in self.worker_nodes: w.start(self.cluster_cfg) w.configure(self.brex_cfg.physical_net) @@ -782,6 +799,16 @@ def start(self): self.nbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe) self.sbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe) + def get_relay_connection_string(self): + if len(self.relay_nodes) > 0: + return ','.join( + [ + db.get_connection_string(self.cluster_cfg, 6642) + for db in self.relay_nodes + ] + ) + return self.central_node.get_connection_string(self.cluster_cfg, 6642) + def create_cluster_router(self, rtr_name): self.router = self.nbctl.lr_add(rtr_name) self.nbctl.lr_set_options( From 55928fa1f963c4655921f480cacae92e7da72e53 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 00:47:28 +0200 Subject: [PATCH 6/8] ovn_tester: Make the CentralNode instance handle a single DB container. It makes it easier to encapsulate things in the Cluster class. Signed-off-by: Dumitru Ceara --- .../cms/ovn_kubernetes/ovn_kubernetes.py | 4 +- ovn-tester/ovn_ext_cmd.py | 14 +-- ovn-tester/ovn_tester.py | 20 +++-- ovn-tester/ovn_workload.py | 90 ++++++++++--------- 4 files changed, 64 insertions(+), 64 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 126d27a6..ae6c5874 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -33,10 +33,10 @@ def create_nodes(cluster_cfg, workers): @staticmethod def prepare_test( - central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg ): ovn = Cluster( - central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg ) with Context(ovn, 'prepare_test'): ovn.start() diff --git a/ovn-tester/ovn_ext_cmd.py b/ovn-tester/ovn_ext_cmd.py index 2209a3dd..6b7fca3d 100644 --- a/ovn-tester/ovn_ext_cmd.py +++ b/ovn-tester/ovn_ext_cmd.py @@ -1,14 +1,6 @@ from collections import defaultdict from fnmatch import fnmatch from io import StringIO -from ovn_sandbox import Sandbox - - -# The wrapper allows us to execute the command on all -# matching central containers -class CentralNodeWrapper(Sandbox): - def __init__(self, central_node, container): - super().__init__(central_node.phys_node, container) class ExtCmdUnit: @@ -23,11 +15,7 @@ def __init__(self, conf, central_node, worker_nodes): node = conf.get('node') self.nodes = [n for n in worker_nodes if fnmatch(n.container, node)] self.nodes.extend( - [ - CentralNodeWrapper(central_node, c) - for c in central_node.central_containers() - if fnmatch(c, node) - ] + [n for n in central_nodes if fnmatch(n.container, node)] ) def is_valid(self): diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index 0cdc0713..e5fdb992 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -166,7 +166,7 @@ def load_cms(cms_name): return cls() -def configure_tests(yaml, central_node, worker_nodes, global_cfg): +def configure_tests(yaml, central_nodes, worker_nodes, global_cfg): tests = [] for section, cfg in yaml.items(): if section in RESERVED: @@ -177,7 +177,7 @@ def configure_tests(yaml, central_node, worker_nodes, global_cfg): ) class_name = ''.join(s.title() for s in section.split('_')) cls = getattr(mod, class_name) - tests.append(cls(yaml, central_node, worker_nodes, global_cfg)) + tests.append(cls(yaml, central_nodes, worker_nodes, global_cfg)) return tests @@ -188,14 +188,18 @@ def create_central_nodes(cluster_cfg, central): if cluster_cfg.clustered_db else ['ovn-central'] ) - central_node = CentralNode(central, db_containers, mgmt_ip) - mgmt += 1 + + central_nodes = [ + CentralNode(central, c, mgmt_ip + i) + for i, c in enumerate(db_containers) + ] + mgmt_ip += len(central_nodes) relay_nodes = [ RelayNode(central, f'ovn-relay-{i + 1}', mgmt_ip + i) for i in range(cluster_cfg.n_relays) ] - return central_node, relay_nodes + return central_nodes, relay_nodes def set_ssl_keys(cluster_cfg): @@ -224,15 +228,15 @@ def set_ssl_keys(cluster_cfg): cms = load_cms(global_cfg.cms_name) central, workers = read_physical_deployment(sys.argv[1], global_cfg) - central_node, relay_nodes = create_central_nodes(cluster_cfg, central) + central_nodes, relay_nodes = create_central_nodes(cluster_cfg, central) worker_nodes = cms.create_nodes(cluster_cfg, workers) - tests = configure_tests(config, central_node, worker_nodes, global_cfg) + tests = configure_tests(config, central_nodes, worker_nodes, global_cfg) if cluster_cfg.enable_ssl: set_ssl_keys(cluster_cfg) ovn = cms.prepare_test( - central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg ) for test in tests: test.run(ovn, global_cfg) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index fecca058..513045c8 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -59,13 +59,13 @@ def __init__(self, phys_node, container, mgmt_ip): class CentralNode(Node): - def __init__(self, phys_node, db_containers, mgmt_ip): - super().__init__(phys_node, db_containers[0], mgmt_ip) - self.db_containers = db_containers + def __init__(self, phys_node, container, mgmt_ip): + super().__init__(phys_node, container, mgmt_ip) - def start(self, cluster_cfg): + def start(self, cluster_cfg, update_election_timeout=False): log.info('Configuring central node') - self.set_raft_election_timeout(cluster_cfg.raft_election_to) + if cluster_cfg.clustered_db and update_election_timeout: + self.set_raft_election_timeout(cluster_cfg.raft_election_to) self.enable_trim_on_compaction() self.set_northd_threads(cluster_cfg.northd_threads) if cluster_cfg.log_txns_db: @@ -73,12 +73,11 @@ def start(self, cluster_cfg): def set_northd_threads(self, n_threads): log.info(f'Configuring northd to use {n_threads} threads') - for container in self.db_containers: - self.phys_node.run( - f'podman exec {container} ovn-appctl -t ' - f'ovn-northd parallel-build/set-n-threads ' - f'{n_threads}' - ) + self.phys_node.run( + f'podman exec {self.container} ovn-appctl -t ' + f'ovn-northd parallel-build/set-n-threads ' + f'{n_threads}' + ) def set_raft_election_timeout(self, timeout_s): for timeout in range(1000, (timeout_s + 1) * 1000, 1000): @@ -97,17 +96,16 @@ def set_raft_election_timeout(self, timeout_s): def enable_trim_on_compaction(self): log.info('Setting DB trim-on-compaction') - for db_container in self.db_containers: - self.phys_node.run( - f'podman exec {db_container} ovs-appctl -t ' - f'/run/ovn/ovnnb_db.ctl ' - f'ovsdb-server/memory-trim-on-compaction on' - ) - self.phys_node.run( - f'podman exec {db_container} ovs-appctl -t ' - f'/run/ovn/ovnsb_db.ctl ' - f'ovsdb-server/memory-trim-on-compaction on' - ) + self.phys_node.run( + f'podman exec {self.container} ovs-appctl -t ' + f'/run/ovn/ovnnb_db.ctl ' + f'ovsdb-server/memory-trim-on-compaction on' + ) + self.phys_node.run( + f'podman exec {self.container} ovs-appctl -t ' + f'/run/ovn/ovnsb_db.ctl ' + f'ovsdb-server/memory-trim-on-compaction on' + ) def enable_txns_db_logging(self): log.info('Enable DB txn logging') @@ -130,13 +128,7 @@ def enable_txns_db_logging(self): def get_connection_string(self, cluster_cfg, port): protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - ip = self.mgmt_ip - num_conns = 3 if cluster_cfg.clustered_db else 1 - conns = [f"{protocol}:{ip + idx}:{port}" for idx in range(num_conns)] - return ",".join(conns) - - def central_containers(self): - return self.db_containers + return f'{protocol}:{self.mgmt_ip}:{port}' class RelayNode(Node): @@ -748,10 +740,10 @@ def provision_vips_to_load_balancers(self, backend_lists, version): class Cluster: def __init__( - self, central_node, relay_nodes, worker_nodes, cluster_cfg, brex_cfg + self, central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg ): # In clustered mode use the first node for provisioning. - self.central_node = central_node + self.central_nodes = central_nodes self.relay_nodes = relay_nodes self.worker_nodes = worker_nodes self.cluster_cfg = cluster_cfg @@ -767,20 +759,20 @@ def __init__( self.n_ns = 0 def start(self): - self.central_node.start(self.cluster_cfg) - nb_conn = self.central_node.get_connection_string( - self.cluster_cfg, 6641 - ) + for c in self.central_nodes: + c.start( + self.cluster_cfg, + update_election_timeout=(c is self.central_nodes[0]), + ) + nb_conn = self.get_nb_connection_string() inactivity_probe = self.cluster_cfg.db_inactivity_probe // 1000 self.nbctl = ovn_utils.OvnNbctl( - self.central_node, nb_conn, inactivity_probe + self.central_nodes[0], nb_conn, inactivity_probe ) - sb_conn = self.central_node.get_connection_string( - self.cluster_cfg, 6642 - ) + sb_conn = self.get_sb_connection_string() self.sbctl = ovn_utils.OvnSbctl( - self.central_node, sb_conn, inactivity_probe + self.central_nodes[0], sb_conn, inactivity_probe ) for r in self.relay_nodes: @@ -799,6 +791,22 @@ def start(self): self.nbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe) self.sbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe) + def get_nb_connection_string(self): + return ','.join( + [ + db.get_connection_string(self.cluster_cfg, 6641) + for db in self.central_nodes + ] + ) + + def get_sb_connection_string(self): + return ','.join( + [ + db.get_connection_string(self.cluster_cfg, 6642) + for db in self.central_nodes + ] + ) + def get_relay_connection_string(self): if len(self.relay_nodes) > 0: return ','.join( @@ -807,7 +815,7 @@ def get_relay_connection_string(self): for db in self.relay_nodes ] ) - return self.central_node.get_connection_string(self.cluster_cfg, 6642) + return self.get_sb_connection_string def create_cluster_router(self, rtr_name): self.router = self.nbctl.lr_add(rtr_name) From 82809c8c638e94d94469c67be167b4550e14cb7f Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 28 Sep 2023 22:49:47 +0200 Subject: [PATCH 7/8] ovn_tester: Factor out protocol as member of Node. All nodes run the same OVSDB protocol. Signed-off-by: Dumitru Ceara --- .../cms/ovn_kubernetes/ovn_kubernetes.py | 2 + ovn-tester/ovn_tester.py | 5 +- ovn-tester/ovn_workload.py | 48 ++++++++----------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index ae6c5874..9c518aae 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -14,6 +14,7 @@ def create_nodes(cluster_cfg, workers): mgmt_ip += 3 if cluster_cfg.clustered_db else 1 mgmt_ip += cluster_cfg.n_relays + protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" internal_net = cluster_cfg.internal_net external_net = cluster_cfg.external_net gw_net = cluster_cfg.gw_net @@ -22,6 +23,7 @@ def create_nodes(cluster_cfg, workers): workers[i % len(workers)], f'ovn-scale-{i}', mgmt_ip + i, + protocol, DualStackSubnet.next(internal_net, i), DualStackSubnet.next(external_net, i), gw_net, diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index e5fdb992..11c109a2 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -182,6 +182,7 @@ def configure_tests(yaml, central_nodes, worker_nodes, global_cfg): def create_central_nodes(cluster_cfg, central): + protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" mgmt_ip = cluster_cfg.node_net.ip + 2 db_containers = ( ['ovn-central-1', 'ovn-central-2', 'ovn-central-3'] @@ -190,13 +191,13 @@ def create_central_nodes(cluster_cfg, central): ) central_nodes = [ - CentralNode(central, c, mgmt_ip + i) + CentralNode(central, c, mgmt_ip + i, protocol) for i, c in enumerate(db_containers) ] mgmt_ip += len(central_nodes) relay_nodes = [ - RelayNode(central, f'ovn-relay-{i + 1}', mgmt_ip + i) + RelayNode(central, f'ovn-relay-{i + 1}', mgmt_ip + i, protocol) for i in range(cluster_cfg.n_relays) ] return central_nodes, relay_nodes diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 513045c8..55725584 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -52,15 +52,16 @@ class Node(ovn_sandbox.Sandbox): - def __init__(self, phys_node, container, mgmt_ip): + def __init__(self, phys_node, container, mgmt_ip, protocol): super().__init__(phys_node, container) self.container = container self.mgmt_ip = netaddr.IPAddress(mgmt_ip) + self.protocol = protocol class CentralNode(Node): - def __init__(self, phys_node, container, mgmt_ip): - super().__init__(phys_node, container, mgmt_ip) + def __init__(self, phys_node, container, mgmt_ip, protocol): + super().__init__(phys_node, container, mgmt_ip, protocol) def start(self, cluster_cfg, update_election_timeout=False): log.info('Configuring central node') @@ -126,22 +127,20 @@ def enable_txns_db_logging(self): 'vlog/disable-rate-limit transaction' ) - def get_connection_string(self, cluster_cfg, port): - protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - return f'{protocol}:{self.mgmt_ip}:{port}' + def get_connection_string(self, port): + return f'{self.protocol}:{self.mgmt_ip}:{port}' class RelayNode(Node): - def __init__(self, phys_node, container, mgmt_ip): - super().__init__(phys_node, container, mgmt_ip) + def __init__(self, phys_node, container, mgmt_ip, protocol): + super().__init__(phys_node, container, mgmt_ip, protocol) def start(self): log.info(f'Configuring relay node {self.container}') self.enable_trim_on_compaction() - def get_connection_string(self, cluster_cfg, port): - protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - return f'{protocol}:{self.mgmt_ip}:{port}' + def get_connection_string(self, port): + return f'{self.protocol}:{self.mgmt_ip}:{port}' def enable_trim_on_compaction(self): log.info('Setting DB trim-on-compaction') @@ -158,12 +157,13 @@ def __init__( phys_node, container, mgmt_ip, + protocol, int_net, ext_net, gw_net, unique_id, ): - super().__init__(phys_node, container, mgmt_ip) + super().__init__(phys_node, container, mgmt_ip, protocol) self.int_net = int_net self.ext_net = ext_net self.gw_net = gw_net @@ -178,7 +178,7 @@ def __init__( def start(self, cluster_cfg): self.vsctl = ovn_utils.OvsVsctl( self, - self.get_connection_string(cluster_cfg, 6640), + self.get_connection_string(6640), cluster_cfg.db_inactivity_probe // 1000, ) @@ -433,9 +433,8 @@ def ping_ports(self, cluster, ports): if port.ip6: self.ping_port(cluster, port, dest=port.ext_gw6) - def get_connection_string(self, cluster_cfg, port): - protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - return f"{protocol}:{self.mgmt_ip}:{port}" + def get_connection_string(self, port): + return f"{self.protocol}:{self.mgmt_ip}:{port}" ACL_DEFAULT_DENY_PRIO = 1 @@ -793,29 +792,20 @@ def start(self): def get_nb_connection_string(self): return ','.join( - [ - db.get_connection_string(self.cluster_cfg, 6641) - for db in self.central_nodes - ] + [db.get_connection_string(6641) for db in self.central_nodes] ) def get_sb_connection_string(self): return ','.join( - [ - db.get_connection_string(self.cluster_cfg, 6642) - for db in self.central_nodes - ] + [db.get_connection_string(6642) for db in self.central_nodes] ) def get_relay_connection_string(self): if len(self.relay_nodes) > 0: return ','.join( - [ - db.get_connection_string(self.cluster_cfg, 6642) - for db in self.relay_nodes - ] + [db.get_connection_string(6642) for db in self.relay_nodes] ) - return self.get_sb_connection_string + return self.get_sb_connection_string() def create_cluster_router(self, rtr_name): self.router = self.nbctl.lr_add(rtr_name) From 1380bba13ab34d5fc3f93bf60a30737c95f8d945 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Fri, 29 Sep 2023 00:13:58 +0200 Subject: [PATCH 8/8] ovn-tester: Use Cluster instead of central + worker tuples. A cluster is a collection of central and worker nodes (+ relays). Signed-off-by: Dumitru Ceara --- .../cms/ovn_kubernetes/ovn_kubernetes.py | 56 +++++++++---------- .../tests/base_cluster_bringup.py | 4 +- .../ovn_kubernetes/tests/cluster_density.py | 4 +- .../cms/ovn_kubernetes/tests/density_heavy.py | 4 +- .../cms/ovn_kubernetes/tests/density_light.py | 4 +- ovn-tester/cms/ovn_kubernetes/tests/netpol.py | 4 +- .../ovn_kubernetes/tests/netpol_cross_ns.py | 4 +- .../cms/ovn_kubernetes/tests/netpol_large.py | 4 +- .../tests/netpol_multitenant.py | 4 +- .../cms/ovn_kubernetes/tests/netpol_small.py | 4 +- .../cms/ovn_kubernetes/tests/service_route.py | 4 +- ovn-tester/ovn_ext_cmd.py | 12 ++-- ovn-tester/ovn_tester.py | 28 ++++++---- ovn-tester/ovn_workload.py | 9 +-- 14 files changed, 76 insertions(+), 69 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 9c518aae..cf623fd1 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -1,5 +1,5 @@ from ovn_context import Context -from ovn_workload import WorkerNode, Cluster +from ovn_workload import WorkerNode from ovn_utils import DualStackSubnet @@ -8,38 +8,38 @@ class OVNKubernetes: @staticmethod - def create_nodes(cluster_cfg, workers): + def add_cluster_worker_nodes(cluster, workers): + cluster_cfg = cluster.cluster_cfg + # Allocate worker IPs after central and relay IPs. - mgmt_ip = cluster_cfg.node_net.ip + 2 - mgmt_ip += 3 if cluster_cfg.clustered_db else 1 - mgmt_ip += cluster_cfg.n_relays + mgmt_ip = ( + cluster_cfg.node_net.ip + + 2 + + len(cluster.central_nodes) + + len(cluster.relay_nodes) + ) protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" internal_net = cluster_cfg.internal_net external_net = cluster_cfg.external_net gw_net = cluster_cfg.gw_net - worker_nodes = [ - WorkerNode( - workers[i % len(workers)], - f'ovn-scale-{i}', - mgmt_ip + i, - protocol, - DualStackSubnet.next(internal_net, i), - DualStackSubnet.next(external_net, i), - gw_net, - i, - ) - for i in range(cluster_cfg.n_workers) - ] - return worker_nodes + cluster.add_workers( + [ + WorkerNode( + workers[i % len(workers)], + f'ovn-scale-{i}', + mgmt_ip + i, + protocol, + DualStackSubnet.next(internal_net, i), + DualStackSubnet.next(external_net, i), + gw_net, + i, + ) + for i in range(cluster_cfg.n_workers) + ] + ) @staticmethod - def prepare_test( - central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg - ): - ovn = Cluster( - central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg - ) - with Context(ovn, 'prepare_test'): - ovn.start() - return ovn + def prepare_test(cluster): + with Context(cluster, 'prepare_test'): + cluster.start() diff --git a/ovn-tester/cms/ovn_kubernetes/tests/base_cluster_bringup.py b/ovn-tester/cms/ovn_kubernetes/tests/base_cluster_bringup.py index c27d31de..7d6df2d3 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/base_cluster_bringup.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/base_cluster_bringup.py @@ -7,8 +7,8 @@ class BaseClusterBringup(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('base_cluster_bringup', dict()) self.config = ClusterBringupCfg( n_pods_per_node=test_config.get('n_pods_per_node', 0), diff --git a/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py b/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py index d77f879a..e6e312f1 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py @@ -17,8 +17,8 @@ class ClusterDensity(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('cluster_density', dict()) self.config = ClusterDensityCfg( n_runs=test_config.get('n_runs', 0), diff --git a/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py b/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py index 6084c315..0bbab209 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py @@ -23,8 +23,8 @@ class DensityHeavy(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('density_heavy', dict()) pods_vip_ratio = test_config.get( 'pods_vip_ratio', DENSITY_PODS_VIP_RATIO diff --git a/ovn-tester/cms/ovn_kubernetes/tests/density_light.py b/ovn-tester/cms/ovn_kubernetes/tests/density_light.py index a8400203..c2c160da 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/density_light.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/density_light.py @@ -10,8 +10,8 @@ class DensityLight(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('density_light', dict()) self.config = DensityCfg( n_pods=test_config.get('n_pods', 0), diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol.py index bcb9222f..31e2a118 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol.py @@ -8,8 +8,8 @@ class NetPol(ExtCmd): - def __init__(self, name, config, central_node, worker_nodes): - super().__init__(config, central_node, worker_nodes) + def __init__(self, name, config, cluster): + super().__init__(config, cluster) test_config = config.get(name, dict()) self.config = NpCfg( n_ns=test_config.get('n_ns', 0), diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py index 78c9a0ed..c1d7366c 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py @@ -7,8 +7,8 @@ class NetpolCrossNs(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('netpol_cross', dict()) self.config = NpCrossNsCfg( n_ns=test_config.get('n_ns', 0), diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_large.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_large.py index eef03585..af8fcdba 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_large.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_large.py @@ -2,8 +2,8 @@ class NetpolLarge(NetPol): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__('netpol_large', config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__('netpol_large', config, cluster) def run(self, ovn, global_cfg): self.init(ovn, global_cfg) diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py index 2a9a2eb8..474044c9 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py @@ -15,8 +15,8 @@ class NetpolMultitenant(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('netpol_multitenant', dict()) ranges = [ NsRange( diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_small.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_small.py index 9f3a06d4..8944424a 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_small.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_small.py @@ -2,8 +2,8 @@ class NetpolSmall(NetPol): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__('netpol_small', config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__('netpol_small', config, cluster) def run(self, ovn, global_cfg): self.init(ovn, global_cfg) diff --git a/ovn-tester/cms/ovn_kubernetes/tests/service_route.py b/ovn-tester/cms/ovn_kubernetes/tests/service_route.py index f6f7e28a..68d38db4 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/service_route.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/service_route.py @@ -16,8 +16,8 @@ class ServiceRoute(ExtCmd): - def __init__(self, config, central_node, worker_nodes, global_cfg): - super().__init__(config, central_node, worker_nodes) + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) test_config = config.get('service_route', dict()) self.config = ServiceRouteCfg( n_lb=test_config.get('n_lb', 16), diff --git a/ovn-tester/ovn_ext_cmd.py b/ovn-tester/ovn_ext_cmd.py index 6b7fca3d..b50c3423 100644 --- a/ovn-tester/ovn_ext_cmd.py +++ b/ovn-tester/ovn_ext_cmd.py @@ -4,7 +4,7 @@ class ExtCmdUnit: - def __init__(self, conf, central_node, worker_nodes): + def __init__(self, conf, cluster): self.iteration = conf.get('iteration') self.cmd = conf.get('cmd') self.test = conf.get('test') @@ -13,9 +13,11 @@ def __init__(self, conf, central_node, worker_nodes): self.pid_opt = conf.get('pid_opt', '') node = conf.get('node') - self.nodes = [n for n in worker_nodes if fnmatch(n.container, node)] + self.nodes = [ + n for n in cluster.worker_nodes if fnmatch(n.container, node) + ] self.nodes.extend( - [n for n in central_nodes if fnmatch(n.container, node)] + [n for n in cluster.central_nodes if fnmatch(n.container, node)] ) def is_valid(self): @@ -46,10 +48,10 @@ def _node_exec(self, node): class ExtCmd: - def __init__(self, config, central_node, worker_nodes): + def __init__(self, config, cluster): self.cmd_map = defaultdict(list) for ext_cmd in config.get('ext_cmd', list()): - cmd_unit = ExtCmdUnit(ext_cmd, central_node, worker_nodes) + cmd_unit = ExtCmdUnit(ext_cmd, cluster) if cmd_unit.is_valid(): self.cmd_map[(cmd_unit.iteration, cmd_unit.test)].append( cmd_unit diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index 11c109a2..d8133538 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -11,7 +11,13 @@ from collections import namedtuple from ovn_sandbox import PhysicalNode -from ovn_workload import BrExConfig, CentralNode, ClusterConfig, RelayNode +from ovn_workload import ( + BrExConfig, + CentralNode, + Cluster, + ClusterConfig, + RelayNode, +) from ovn_utils import DualStackSubnet from ovs.stream import Stream @@ -166,7 +172,7 @@ def load_cms(cms_name): return cls() -def configure_tests(yaml, central_nodes, worker_nodes, global_cfg): +def configure_tests(yaml, cluster, global_cfg): tests = [] for section, cfg in yaml.items(): if section in RESERVED: @@ -177,11 +183,11 @@ def configure_tests(yaml, central_nodes, worker_nodes, global_cfg): ) class_name = ''.join(s.title() for s in section.split('_')) cls = getattr(mod, class_name) - tests.append(cls(yaml, central_nodes, worker_nodes, global_cfg)) + tests.append(cls(yaml, cluster, global_cfg)) return tests -def create_central_nodes(cluster_cfg, central): +def create_cluster(cluster_cfg, central, brex_cfg): protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" mgmt_ip = cluster_cfg.node_net.ip + 2 db_containers = ( @@ -200,7 +206,7 @@ def create_central_nodes(cluster_cfg, central): RelayNode(central, f'ovn-relay-{i + 1}', mgmt_ip + i, protocol) for i in range(cluster_cfg.n_relays) ] - return central_nodes, relay_nodes + return Cluster(central_nodes, relay_nodes, cluster_cfg, brex_cfg) def set_ssl_keys(cluster_cfg): @@ -229,16 +235,14 @@ def set_ssl_keys(cluster_cfg): cms = load_cms(global_cfg.cms_name) central, workers = read_physical_deployment(sys.argv[1], global_cfg) - central_nodes, relay_nodes = create_central_nodes(cluster_cfg, central) - worker_nodes = cms.create_nodes(cluster_cfg, workers) - tests = configure_tests(config, central_nodes, worker_nodes, global_cfg) + cluster = create_cluster(cluster_cfg, central, brex_cfg) + cms.add_cluster_worker_nodes(cluster, workers) + tests = configure_tests(config, cluster, global_cfg) if cluster_cfg.enable_ssl: set_ssl_keys(cluster_cfg) - ovn = cms.prepare_test( - central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg - ) + cms.prepare_test(cluster) for test in tests: - test.run(ovn, global_cfg) + test.run(cluster, global_cfg) sys.exit(0) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 55725584..1a4ae3b2 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -738,13 +738,11 @@ def provision_vips_to_load_balancers(self, backend_lists, version): class Cluster: - def __init__( - self, central_nodes, relay_nodes, worker_nodes, cluster_cfg, brex_cfg - ): + def __init__(self, central_nodes, relay_nodes, cluster_cfg, brex_cfg): # In clustered mode use the first node for provisioning. self.central_nodes = central_nodes self.relay_nodes = relay_nodes - self.worker_nodes = worker_nodes + self.worker_nodes = [] self.cluster_cfg = cluster_cfg self.brex_cfg = brex_cfg self.nbctl = None @@ -757,6 +755,9 @@ def __init__( self.last_selected_worker = 0 self.n_ns = 0 + def add_workers(self, worker_nodes): + self.worker_nodes.extend(worker_nodes) + def start(self): for c in self.central_nodes: c.start(