From 4db21d0d7651fe196ec7fde5e7ad92d82cb9876f Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Wed, 28 Jun 2023 13:45:32 +0200 Subject: [PATCH] ovn-tester: introduce ovn-ic topology in bringup phase if we are running multiple az Signed-off-by: Lorenzo Bianconi --- ovn-fake-multinode-utils/translate_yaml.py | 2 + ovn-tester/ovn_tester.py | 30 ++++++++--- ovn-tester/ovn_utils.py | 58 +++++++++++++++++++++- ovn-tester/ovn_workload.py | 55 ++++++++++++++++++++ 4 files changed, 138 insertions(+), 7 deletions(-) diff --git a/ovn-fake-multinode-utils/translate_yaml.py b/ovn-fake-multinode-utils/translate_yaml.py index 63586eb0..9858cee3 100755 --- a/ovn-fake-multinode-utils/translate_yaml.py +++ b/ovn-fake-multinode-utils/translate_yaml.py @@ -104,6 +104,8 @@ class ClusterConfig: external_net6: str = "3::/64" gw_net: str = "2.0.0.0/16" gw_net6: str = "2::/64" + ts_net: str = "30.0.0.0/16" + ts_net6: str = "30::/64" cluster_net: str = "16.0.0.0/4" cluster_net6: str = "16::/32" n_workers: int = 2 diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index 120dcd1a..422cccc3 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -107,6 +107,14 @@ def read_config(config): if global_cfg.run_ipv6 else None, ), + ts_net=DualStackSubnet( + netaddr.IPNetwork(cluster_args['ts_net']) + if global_cfg.run_ipv4 + else None, + netaddr.IPNetwork(cluster_args['ts_net6']) + if global_cfg.run_ipv6 + else None, + ), n_workers=cluster_args['n_workers'], vips=cluster_args['vips'], vips6=cluster_args['vips6'], @@ -197,6 +205,7 @@ def create_nodes(cluster_config, central, workers): cluster_config.gw_net, i * (cluster_config.n_workers // cluster_config.n_az), ), + cluster_config.ts_net, ) for i in range(cluster_config.n_az) ] @@ -228,6 +237,7 @@ def create_nodes(cluster_config, central, workers): node_az_conf[i].getMgmtNet(), node_az_conf[i].getMgmtIp(), node_az_conf[i].getGwNet(), + node_az_conf[i].getTsNet(), i, ) for i in range(cluster_config.n_az) @@ -275,15 +285,20 @@ def prepare_test(central_nodes, worker_nodes, cluster_cfg, brex_cfg): def run_base_cluster_bringup(ovn, bringup_cfg, global_cfg): - for i in range(0, len(clusters)): + if clusters[0].cluster_cfg.n_az > 1: + clusters[0].create_transit_switch() + + for i in range(clusters[0].cluster_cfg.n_az): ovn = clusters[i] # create ovn topology with Context( - ovn, "base_cluster_bringup", len(ovn.worker_nodes) + ovn, f'base_cluster_bringup-az{i}', len(ovn.worker_nodes) ) as ctx: - ovn.create_cluster_router(f'lr-cluster{i}') - ovn.create_cluster_join_switch(f'ls-join{i}') - ovn.create_cluster_load_balancer(f'lb-cluster{i}', global_cfg) + ovn.create_cluster_router(f'lr-cluster{i + 1}') + ovn.create_cluster_join_switch(f'ls-join{i + 1}') + ovn.create_cluster_load_balancer(f'lb-cluster{i + 1}', global_cfg) + if ovn.cluster_cfg.n_az > 1: + ovn.connect_transit_switch() for i in ctx: worker = ovn.worker_nodes[i] worker.provision(ovn) @@ -292,7 +307,10 @@ def run_base_cluster_bringup(ovn, bringup_cfg, global_cfg): ) worker.provision_load_balancers(ovn, ports, global_cfg) worker.ping_ports(ovn, ports) - ovn.provision_lb_group(f'cluster-lb-group{i}') + ovn.provision_lb_group(f'cluster-lb-group{i + 1}') + + if clusters[0].cluster_cfg.n_az > 1: + clusters[0].check_ic_connectivity(clusters) if __name__ == '__main__': diff --git a/ovn-tester/ovn_utils.py b/ovn-tester/ovn_utils.py index a2989792..742467bb 100644 --- a/ovn-tester/ovn_utils.py +++ b/ovn-tester/ovn_utils.py @@ -2,11 +2,13 @@ import netaddr import select import ovn_exceptions +import time from collections import namedtuple from functools import partial import ovsdbapp.schema.open_vswitch.impl_idl as ovs_impl_idl import ovsdbapp.schema.ovn_northbound.impl_idl as nb_impl_idl import ovsdbapp.schema.ovn_southbound.impl_idl as sb_impl_idl +import ovsdbapp.schema.ovn_ic_northbound.impl_idl as nb_ic_impl_idl from ovsdbapp.backend import ovs_idl from ovsdbapp.backend.ovs_idl import connection from ovsdbapp.backend.ovs_idl import idlutils @@ -102,11 +104,12 @@ def external_host_provision(self, ip, gw, netns='ext-ns'): class NodeConf: - def __init__(self, mgmt_net, int_net, ext_net, gw_net): + def __init__(self, mgmt_net, int_net, ext_net, gw_net, ts_net): self.mgmt_net = mgmt_net self.int_net = int_net self.ext_net = ext_net self.gw_net = gw_net + self.ts_net = ts_net def getMgmtNet(self): return self.mgmt_net @@ -123,6 +126,9 @@ def getExtNet(self): def getGwNet(self): return self.gw_net + def getTsNet(self): + return self.ts_net + class DualStackSubnet: def __init__(self, n4=None, n6=None): @@ -445,6 +451,11 @@ def set_global(self, option, value): "NB_Global", self.idl._nb.uuid, ("options", {option: str(value)}) ).execute() + def set_global_name(self, value): + self.idl.db_set( + "NB_Global", self.idl._nb.uuid, ("name", str(value)) + ).execute() + def set_inactivity_probe(self, value): self.idl.db_set( "Connection", @@ -481,6 +492,17 @@ def ls_add(self, name, net_s): uuid=uuid, ) + def ls_get_uuid(self, name): + for _ in range(10): + uuid = self.idl.db_get( + "Logical_Switch", str(name), '_uuid' + ).execute() + if uuid != None: + return uuid + time.sleep(1) + + return None + def ls_port_add( self, lswitch, @@ -756,3 +778,37 @@ def chassis_bound(self, chassis=""): cmd = self.idl.db_find_rows("Chassis", ("name", "=", chassis)) cmd.execute() return len(cmd.result) == 1 + + +class NBIcIdl(nb_ic_impl_idl.OvnIcNbApiIdlImpl, Backend): + def __init__(self, connection): + super(NBIcIdl, self).__init__(connection) + + @property + def _connection(self): + return next(iter(self.db_list_rows('Connection').execute())) + + +class OvnIcNbctl: + def __init__(self, sb, connection_string, inactivity_probe): + log.info(f'OvnIcNbctl: {connection_string}, probe: {inactivity_probe}') + i = connection.OvsdbIdl.from_server( + connection_string, "OVN_IC_Northbound" + ) + c = connection.Connection(i, inactivity_probe) + self.idl = NBIcIdl(c) + + def uuid_transaction(self, func): + for _ in range(MAX_RETRY): + cmd = func(may_exist=True) + cmd.execute() + try: + return cmd.result.uuid + except AttributeError: + continue + + raise UUIDTransactionError("Failed to get UUID from transaction") + + def ts_add(self): + log.info('Creating transit switch') + self.uuid_transaction(partial(self.idl.ts_add, 'ts')) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 1f58a201..c91636bf 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -10,6 +10,7 @@ from collections import defaultdict from randmac import RandMac from datetime import datetime +from ovn_utils import LSwitch log = logging.getLogger(__name__) @@ -31,6 +32,7 @@ 'internal_net', 'external_net', 'gw_net', + 'ts_net', 'cluster_net', 'n_workers', 'n_relays', @@ -68,6 +70,7 @@ def __init__( mgmt_net, mgmt_ip, gw_net, + ts_net, idx, ): super(CentralNode, self).__init__( @@ -77,6 +80,7 @@ def __init__( self.relay_containers = relay_containers self.id = idx self.gw_net = gw_net + self.ts_net = ts_net def start(self, cluster_cfg): log.info('Configuring central node') @@ -770,6 +774,7 @@ def __init__(self, central_node, worker_nodes, cluster_cfg, brex_cfg): self.brex_cfg = brex_cfg self.nbctl = None self.sbctl = None + self.icnbctl = None self.net = cluster_cfg.cluster_net self.router = None self.load_balancer = None @@ -777,6 +782,7 @@ def __init__(self, central_node, worker_nodes, cluster_cfg, brex_cfg): self.join_switch = None self.last_selected_worker = 0 self.n_ns = 0 + self.ts_switch = None def start(self): self.central_node.start(self.cluster_cfg) @@ -794,6 +800,14 @@ def start(self): self.sbctl = ovn_utils.OvnSbctl( self.central_node, sb_conn, inactivity_probe ) + # ovn-ic configuration + if self.cluster_cfg.n_az > 1: + self.icnbctl = ovn_utils.OvnIcNbctl( + None, f'tcp:{self.central_node.mgmt_ip}:6645', inactivity_probe + ) + self.nbctl.set_global('ic-route-learn', 'true') + self.nbctl.set_global('ic-route-adv', 'true') + for w in self.worker_nodes: w.start(self.cluster_cfg) w.configure(self.brex_cfg.physical_net) @@ -804,9 +818,50 @@ def start(self): self.nbctl.set_global( 'northd_probe_interval', self.cluster_cfg.northd_probe_interval ) + self.nbctl.set_global_name(f'az{self.central_node.id + 1}') self.nbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe) self.sbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe) + def create_transit_switch(self): + self.icnbctl.ts_add() + + def connect_transit_switch(self): + uuid = self.nbctl.ls_get_uuid('ts') + self.ts_switch = LSwitch( + name='ts', + cidr=self.central_node.ts_net.n4, + cidr6=self.central_node.ts_net.n6, + uuid=uuid, + ) + rp = self.nbctl.lr_port_add( + self.router, + f'lr-cluster{self.central_node.id + 1}-to-ts', + RandMac(), + self.central_node.ts_net.forward(self.central_node.id + 1), + ) + self.nbctl.ls_port_add( + self.ts_switch, f'ts-to-lr-cluster{self.central_node.id + 1}', rp + ) + self.nbctl.lr_port_set_gw_chassis(rp, self.worker_nodes[0].container) + self.worker_nodes[0].vsctl.set_global_external_id( + 'ovn-is-interconn', 'true' + ) + + def check_ic_connectivity(self, clusters): + for i in range(self.cluster_cfg.n_az): + if self == clusters[i]: + continue + for w in clusters[i].worker_nodes: + port = w.lports[0] + if port.ip: + self.worker_nodes[0].run_ping( + self, self.worker_nodes[0].lports[0].name, port.ip + ) + if port.ip6: + self.worker_nodes[0].run_ping( + self, self.worker_nodes[0].lports[0].name, port.ip6 + ) + def create_cluster_router(self, rtr_name): self.router = self.nbctl.lr_add(rtr_name) self.nbctl.lr_set_options(