ovn-tester: introduce ovn-ic topology in bringup phase if we are runn…

…ing multiple az

Signed-off-by: Lorenzo Bianconi <[email protected]>

ovn-fake-multinode-utils/translate_yaml.py

@@ -104,6 +104,8 @@ class ClusterConfig:
     external_net6: str = "3::/64"
     gw_net: str = "2.0.0.0/16"
     gw_net6: str = "2::/64"
+    ts_net: str = "30.0.0.0/16"
+    ts_net6: str = "30::/64"
     cluster_net: str = "16.0.0.0/4"
     cluster_net6: str = "16::/32"
     n_workers: int = 2

ovn-tester/ovn_tester.py

@@ -107,6 +107,14 @@ def read_config(config):
             if global_cfg.run_ipv6
             else None,
         ),
+        ts_net=DualStackSubnet(
+            netaddr.IPNetwork(cluster_args['ts_net'])
+            if global_cfg.run_ipv4
+            else None,
+            netaddr.IPNetwork(cluster_args['ts_net6'])
+            if global_cfg.run_ipv6
+            else None,
+        ),
         n_workers=cluster_args['n_workers'],
         vips=cluster_args['vips'],
         vips6=cluster_args['vips6'],
@@ -197,6 +205,7 @@ def create_nodes(cluster_config, central, workers):
                 cluster_config.gw_net,
                 i * (cluster_config.n_workers // cluster_config.n_az),
             ),
+            cluster_config.ts_net,
         )
         for i in range(cluster_config.n_az)
     ]
@@ -228,6 +237,7 @@ def create_nodes(cluster_config, central, workers):
             node_az_conf[i].getMgmtNet(),
             node_az_conf[i].getMgmtIp(),
             node_az_conf[i].getGwNet(),
+            node_az_conf[i].getTsNet(),
             i,
         )
         for i in range(cluster_config.n_az)
@@ -275,15 +285,20 @@ def prepare_test(central_nodes, worker_nodes, cluster_cfg, brex_cfg):
 
 
 def run_base_cluster_bringup(ovn, bringup_cfg, global_cfg):
-    for i in range(0, len(clusters)):
+    if clusters[0].cluster_cfg.n_az > 1:
+        clusters[0].create_transit_switch()
+
+    for i in range(clusters[0].cluster_cfg.n_az):
         ovn = clusters[i]
         # create ovn topology
         with Context(
-            ovn, "base_cluster_bringup", len(ovn.worker_nodes)
+            ovn, f'base_cluster_bringup-az{i}', len(ovn.worker_nodes)
         ) as ctx:
-            ovn.create_cluster_router(f'lr-cluster{i}')
-            ovn.create_cluster_join_switch(f'ls-join{i}')
-            ovn.create_cluster_load_balancer(f'lb-cluster{i}', global_cfg)
+            ovn.create_cluster_router(f'lr-cluster{i + 1}')
+            ovn.create_cluster_join_switch(f'ls-join{i + 1}')
+            ovn.create_cluster_load_balancer(f'lb-cluster{i + 1}', global_cfg)
+            if ovn.cluster_cfg.n_az > 1:
+                ovn.connect_transit_switch()
             for i in ctx:
                 worker = ovn.worker_nodes[i]
                 worker.provision(ovn)
@@ -292,7 +307,10 @@ def run_base_cluster_bringup(ovn, bringup_cfg, global_cfg):
                 )
                 worker.provision_load_balancers(ovn, ports, global_cfg)
                 worker.ping_ports(ovn, ports)
-            ovn.provision_lb_group(f'cluster-lb-group{i}')
+            ovn.provision_lb_group(f'cluster-lb-group{i + 1}')
+
+    if clusters[0].cluster_cfg.n_az > 1:
+        clusters[0].check_ic_connectivity(clusters)
 
 
 if __name__ == '__main__':

ovn-tester/ovn_utils.py

@@ -2,11 +2,13 @@
 import netaddr
 import select
 import ovn_exceptions
+import time
 from collections import namedtuple
 from functools import partial
 import ovsdbapp.schema.open_vswitch.impl_idl as ovs_impl_idl
 import ovsdbapp.schema.ovn_northbound.impl_idl as nb_impl_idl
 import ovsdbapp.schema.ovn_southbound.impl_idl as sb_impl_idl
+import ovsdbapp.schema.ovn_ic_northbound.impl_idl as nb_ic_impl_idl
 from ovsdbapp.backend import ovs_idl
 from ovsdbapp.backend.ovs_idl import connection
 from ovsdbapp.backend.ovs_idl import idlutils
@@ -102,11 +104,12 @@ def external_host_provision(self, ip, gw, netns='ext-ns'):
 
 
 class NodeConf:
-    def __init__(self, mgmt_net, int_net, ext_net, gw_net):
+    def __init__(self, mgmt_net, int_net, ext_net, gw_net, ts_net):
         self.mgmt_net = mgmt_net
         self.int_net = int_net
         self.ext_net = ext_net
         self.gw_net = gw_net
+        self.ts_net = ts_net
 
     def getMgmtNet(self):
         return self.mgmt_net
@@ -123,6 +126,9 @@ def getExtNet(self):
     def getGwNet(self):
         return self.gw_net
 
+    def getTsNet(self):
+        return self.ts_net
+
 
 class DualStackSubnet:
     def __init__(self, n4=None, n6=None):
@@ -445,6 +451,11 @@ def set_global(self, option, value):
             "NB_Global", self.idl._nb.uuid, ("options", {option: str(value)})
         ).execute()
 
+    def set_global_name(self, value):
+        self.idl.db_set(
+            "NB_Global", self.idl._nb.uuid, ("name", str(value))
+        ).execute()
+
     def set_inactivity_probe(self, value):
         self.idl.db_set(
             "Connection",
@@ -481,6 +492,17 @@ def ls_add(self, name, net_s):
             uuid=uuid,
         )
 
+    def ls_get_uuid(self, name):
+        for _ in range(10):
+            uuid = self.idl.db_get(
+                "Logical_Switch", str(name), '_uuid'
+            ).execute()
+            if uuid != None:
+                return uuid
+            time.sleep(1)
+
+        return None
+
     def ls_port_add(
         self,
         lswitch,
@@ -756,3 +778,37 @@ def chassis_bound(self, chassis=""):
         cmd = self.idl.db_find_rows("Chassis", ("name", "=", chassis))
         cmd.execute()
         return len(cmd.result) == 1
+
+
+class NBIcIdl(nb_ic_impl_idl.OvnIcNbApiIdlImpl, Backend):
+    def __init__(self, connection):
+        super(NBIcIdl, self).__init__(connection)
+
+    @property
+    def _connection(self):
+        return next(iter(self.db_list_rows('Connection').execute()))
+
+
+class OvnIcNbctl:
+    def __init__(self, sb, connection_string, inactivity_probe):
+        log.info(f'OvnIcNbctl: {connection_string}, probe: {inactivity_probe}')
+        i = connection.OvsdbIdl.from_server(
+            connection_string, "OVN_IC_Northbound"
+        )
+        c = connection.Connection(i, inactivity_probe)
+        self.idl = NBIcIdl(c)
+
+    def uuid_transaction(self, func):
+        for _ in range(MAX_RETRY):
+            cmd = func(may_exist=True)
+            cmd.execute()
+            try:
+                return cmd.result.uuid
+            except AttributeError:
+                continue
+
+        raise UUIDTransactionError("Failed to get UUID from transaction")
+
+    def ts_add(self):
+        log.info('Creating transit switch')
+        self.uuid_transaction(partial(self.idl.ts_add, 'ts'))

ovn-tester/ovn_workload.py

@@ -10,6 +10,7 @@
 from collections import defaultdict
 from randmac import RandMac
 from datetime import datetime
+from ovn_utils import LSwitch
 
 log = logging.getLogger(__name__)
 
@@ -31,6 +32,7 @@
         'internal_net',
         'external_net',
         'gw_net',
+        'ts_net',
         'cluster_net',
         'n_workers',
         'n_relays',
@@ -68,6 +70,7 @@ def __init__(
         mgmt_net,
         mgmt_ip,
         gw_net,
+        ts_net,
         idx,
     ):
         super(CentralNode, self).__init__(
@@ -77,6 +80,7 @@ def __init__(
         self.relay_containers = relay_containers
         self.id = idx
         self.gw_net = gw_net
+        self.ts_net = ts_net
 
     def start(self, cluster_cfg):
         log.info('Configuring central node')
@@ -770,13 +774,15 @@ def __init__(self, central_node, worker_nodes, cluster_cfg, brex_cfg):
         self.brex_cfg = brex_cfg
         self.nbctl = None
         self.sbctl = None
+        self.icnbctl = None
         self.net = cluster_cfg.cluster_net
         self.router = None
         self.load_balancer = None
         self.load_balancer6 = None
         self.join_switch = None
         self.last_selected_worker = 0
         self.n_ns = 0
+        self.ts_switch = None
 
     def start(self):
         self.central_node.start(self.cluster_cfg)
@@ -794,6 +800,14 @@ def start(self):
         self.sbctl = ovn_utils.OvnSbctl(
             self.central_node, sb_conn, inactivity_probe
         )
+        # ovn-ic configuration
+        if self.cluster_cfg.n_az > 1:
+            self.icnbctl = ovn_utils.OvnIcNbctl(
+                None, f'tcp:{self.central_node.mgmt_ip}:6645', inactivity_probe
+            )
+            self.nbctl.set_global('ic-route-learn', 'true')
+            self.nbctl.set_global('ic-route-adv', 'true')
+
         for w in self.worker_nodes:
             w.start(self.cluster_cfg)
             w.configure(self.brex_cfg.physical_net)
@@ -804,9 +818,50 @@ def start(self):
         self.nbctl.set_global(
             'northd_probe_interval', self.cluster_cfg.northd_probe_interval
         )
+        self.nbctl.set_global_name(f'az{self.central_node.id + 1}')
         self.nbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe)
         self.sbctl.set_inactivity_probe(self.cluster_cfg.db_inactivity_probe)
 
+    def create_transit_switch(self):
+        self.icnbctl.ts_add()
+
+    def connect_transit_switch(self):
+        uuid = self.nbctl.ls_get_uuid('ts')
+        self.ts_switch = LSwitch(
+            name='ts',
+            cidr=self.central_node.ts_net.n4,
+            cidr6=self.central_node.ts_net.n6,
+            uuid=uuid,
+        )
+        rp = self.nbctl.lr_port_add(
+            self.router,
+            f'lr-cluster{self.central_node.id + 1}-to-ts',
+            RandMac(),
+            self.central_node.ts_net.forward(self.central_node.id + 1),
+        )
+        self.nbctl.ls_port_add(
+            self.ts_switch, f'ts-to-lr-cluster{self.central_node.id + 1}', rp
+        )
+        self.nbctl.lr_port_set_gw_chassis(rp, self.worker_nodes[0].container)
+        self.worker_nodes[0].vsctl.set_global_external_id(
+            'ovn-is-interconn', 'true'
+        )
+
+    def check_ic_connectivity(self, clusters):
+        for i in range(self.cluster_cfg.n_az):
+            if self == clusters[i]:
+                continue
+            for w in clusters[i].worker_nodes:
+                port = w.lports[0]
+                if port.ip:
+                    self.worker_nodes[0].run_ping(
+                        self, self.worker_nodes[0].lports[0].name, port.ip
+                    )
+                if port.ip6:
+                    self.worker_nodes[0].run_ping(
+                        self, self.worker_nodes[0].lports[0].name, port.ip6
+                    )
+
     def create_cluster_router(self, rtr_name):
         self.router = self.nbctl.lr_add(rtr_name)
         self.nbctl.lr_set_options(