diff --git a/.cirrus.yml b/.cirrus.yml index 536b6408..be957119 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -69,12 +69,17 @@ low_scale_task: upload_caches: - runtime - test_script: + test_ovn_kubernetes_script: - 'sed -i "s/^ log_cmds\: False/ log_cmds\: True/" test-scenarios/ovn-low-scale*.yml' - ./do.sh run test-scenarios/ovn-low-scale.yml low-scale - ./do.sh run test-scenarios/ovn-low-scale-ic.yml low-scale-ic + test_openstack_script: + - 'sed -i "s/^ log_cmds\: false/ log_cmds\: true/" + test-scenarios/openstack-low-scale.yml' + - ./do.sh run test-scenarios/openstack-low-scale.yml openstack-low-scale + check_logs_script: - ./utils/logs-checker.sh diff --git a/ovn-tester/cms/openstack/__init__.py b/ovn-tester/cms/openstack/__init__.py new file mode 100644 index 00000000..c8e6f9d0 --- /dev/null +++ b/ovn-tester/cms/openstack/__init__.py @@ -0,0 +1,11 @@ +from .openstack import ( + OpenStackCloud, + OVN_HEATER_CMS_PLUGIN, + ExternalNetworkSpec, +) + +__all__ = [ + OpenStackCloud, + OVN_HEATER_CMS_PLUGIN, + ExternalNetworkSpec, +] diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py new file mode 100644 index 00000000..84a8bb5e --- /dev/null +++ b/ovn-tester/cms/openstack/openstack.py @@ -0,0 +1,731 @@ +import logging +import random +import uuid + +from dataclasses import dataclass +from itertools import cycle +from typing import List, Optional, Dict + +import netaddr + +from randmac import RandMac + +from ovn_sandbox import PhysicalNode +from ovn_utils import ( + DualStackSubnet, + LSwitch, + LSPort, + PortGroup, + LRouter, + LRPort, + DualStackIP, +) +from ovn_workload import ChassisNode, Cluster + +log = logging.getLogger(__name__) + +OVN_HEATER_CMS_PLUGIN = 'OpenStackCloud' + + +@dataclass +class NeutronNetwork: + """Group of OVN objects that logically belong to same Neutron network.""" + + network: LSwitch + ports: Dict[str, LSPort] + name: str + mtu: int + security_group: Optional[PortGroup] = None + + def __post_init__(self): + self._ip_pool = self.network.cidr.iter_hosts() + self.gateway = self.next_host_ip() + + def next_host_ip(self) -> netaddr.IPAddress: + """Return next available host IP in this network's range.""" + return next(self._ip_pool) + + +@dataclass +class ExternalNetworkSpec: + """Information required to connect external network to Project's router. + :param neutron_net: Object of already provisioned NeutronNetwork + :param num_gw_nodes: Number of Chassis to use as gateways for this + network. Note that the number can't be greater than 5 + and can't be greater than total number of Chassis + available in the system. + """ + + neutron_net: NeutronNetwork + num_gw_nodes: int + + +class Project: + """Represent network components of an OpenStack Project aka. Tenant.""" + + def __init__( + self, + int_net: NeutronNetwork = None, + ext_net: NeutronNetwork = None, + router: Optional[LRouter] = None, + ): + self.int_net = int_net + self.ext_net = ext_net + self.router = router + self.vm_ports: List[LSPort] = [] + self._id = str(uuid.uuid4()) + + @property + def uuid(self) -> str: + """Return arbitrary UUID assigned to this Openstack Project.""" + return self._id + + +class OpenStackCloud(Cluster): + """Representation of Openstack cloud/deployment.""" + + MAX_GW_PER_ROUTER = 5 + + def __init__(self, cluster_cfg, central, brex_cfg, az): + super().__init__(cluster_cfg, central, brex_cfg, az) + self.router = None + self._projects: List[Project] = [] + + self._int_net_base = DualStackSubnet(cluster_cfg.node_net) + self._int_net_offset = 0 + + self._ext_net_pool_index = 0 + + def add_workers(self, workers: List[PhysicalNode]): + """Expand parent method to update cycled list.""" + super().add_workers(workers) + self._compute_nodes = cycle( + [ + node + for node in self.worker_nodes + if isinstance(node, ComputeNode) + ] + ) + + def add_cluster_worker_nodes(self, workers: List[PhysicalNode]): + """Add OpenStack flavored worker nodes as per configuration.""" + # Allocate worker IPs after central and relay IPs. + mgmt_ip = ( + self.cluster_cfg.node_net.ip + + 2 + + len(self.central_nodes) + + len(self.relay_nodes) + ) + + protocol = "ssl" if self.cluster_cfg.enable_ssl else "tcp" + # XXX: To facilitate network nodes we'd have to make bigger change + # to introduce `n_compute_nodes` and `n_network_nodes` into the + # ClusterConfig. I tested it and sems like additional changes + # would be required in `ovn-fake-multinode` repo to handle worker + # creation in similar way as it's done for `n_workers` and + # `n_relays` right now. + network_nodes = [] + compute_nodes = [ + ComputeNode( + workers[i % len(workers)], + f"ovn-scale-{i}", + mgmt_ip + i, + protocol, + True, + ) + for i in range(self.cluster_cfg.n_workers) + ] + self.add_workers(network_nodes + compute_nodes) + + @property + def projects(self) -> List[Project]: + """Return list of Projects that exist in Openstack deployment.""" + return self._projects + + def next_external_ip(self) -> DualStackIP: + """Return next available IP address from configured 'external_net'.""" + self._ext_net_pool_index += 1 + return self.cluster_cfg.external_net.forward(self._ext_net_pool_index) + + def next_int_net(self): + """Return next subnet for project/tenant internal network. + + This method takes subnet configured as 'node_net' in cluster config as + base and each call returns next subnet in line. + """ + network = DualStackSubnet.next( + self._int_net_base, self._int_net_offset + ) + self._int_net_offset += 1 + + return network + + def select_worker_for_port(self) -> 'ComputeNode': + """Cyclically return compute nodes available in cluster.""" + return next(self._compute_nodes) + + def new_project(self, ext_net: Optional[ExternalNetworkSpec]) -> Project: + """Create new Project/Tenant in the Openstack cloud. + + Following things are provisioned for the Project: + * Project router + * Internal network + + If 'ext_net' is provided, this external network will be connected + to the project's router and default gateway route will be configured + through this network. + + :param ext_net: External network to be connected to the Project. + :return: New Project object that is automatically also added + to 'self.projects' list. + """ + project = Project() + gateway_port = None + + project.router = self._create_project_router( + f"provider-router-{project.uuid}" + ) + if ext_net is not None: + gateway_port = self.connect_external_network_to_project( + project, ext_net + ) + + self.add_internal_network_to_project(project, gateway_port) + self._projects.append(project) + return project + + def new_external_network( + self, provider_network: str = "physnet1" + ) -> NeutronNetwork: + """Provision external network that can be added as gw to Projects. + + :param provider_network: Name of the provider network used when + creating provider port. + :return: NeutronNetwork object representing external network. + """ + ext_net_uuid = uuid.uuid4() + ext_net_name = f"ext_net_{ext_net_uuid}" + mtu = 1500 + ext_net = self._create_project_net(f"ext_net_{ext_net_uuid}", mtu) + ext_net_port = self._add_metadata_port(ext_net, str(ext_net_uuid)) + provider_port = self._add_provider_network_port( + ext_net, provider_network + ) + return NeutronNetwork( + network=ext_net, + ports={ + ext_net_port.uuid: ext_net_port, + provider_port.uuid: provider_port, + }, + name=ext_net_name, + mtu=mtu, + ) + + def connect_external_network_to_project( + self, + project: Project, + external_network: ExternalNetworkSpec, + ) -> LRPort: + """Connect external net to Project that adds external connectivity. + This method takes existing Neutron network, connects it to the + Project's router and configures default route on the router to go + through this network. + + Gateway Chassis will be picked at random from available Chassis + nodes based on the number of gateways specified in 'external_network' + specification. + + :param project: Project to which external network will be added + :param external_network: External network that will be connected. + :return: Logical Router Port that connects to the external network. + """ + + ls_port, lr_port = self._add_router_port_external_gw( + external_network.neutron_net, project.router + ) + self.external_port = lr_port + gw_net = DualStackSubnet(netaddr.IPNetwork("0.0.0.0/0")) + + # XXX: ovsdbapp does not allow setting external IDs to static route + # XXX: Setting 'policy' to "" throws "constraint violation" error in + # logs because ovsdbapp does not allow not specifying policy. + # However, the route itself is created successfully with no + # policy, the same way Neutron does it. + self.nbctl.route_add(project.router, gw_net, lr_port.ip, "") + + gw_nodes = self._get_gateway_chassis(external_network.num_gw_nodes) + for index, chassis in enumerate(gw_nodes): + self.nbctl.lr_port_set_gw_chassis( + lr_port, chassis.container, index + 1 + ) + + project.ext_net = external_network + + return lr_port + + def add_internal_network_to_project( + self, project: Project, snat_port: Optional[LRPort] = None + ) -> None: + """Provision internal net to the Project for guest communication. + + If 'snat_port' is provided. A SNAT rule will be configured for traffic + exiting this network. + + :param project: Project to which internal network will be added. + :param snat_port: Gateway port that should SNAT outgoing traffic. + :return: None + """ + int_net_name = f"int_net_{project.uuid}" + mtu = 1442 + int_net = self._create_project_net(int_net_name, mtu) + + int_net_port = self._add_metadata_port(int_net, project.uuid) + security_group = self._create_default_security_group() + neutron_int_network = NeutronNetwork( + network=int_net, + ports={int_net_port.uuid: int_net_port}, + name=int_net_name, + mtu=mtu, + security_group=security_group, + ) + + self._add_network_subnet(network=neutron_int_network) + self._assign_port_ips(neutron_int_network) + project.int_net = neutron_int_network + + self._add_router_port_internal( + neutron_int_network, project.router, project + ) + + snated_network = DualStackSubnet(int_net.cidr) + if snat_port is not None: + self.nbctl.nat_add(project.router, snat_port.ip, snated_network) + + def add_vm_to_project(self, project: Project, vm_name: str): + compute = self.select_worker_for_port() + vm_port = self._add_vm_port( + project.int_net, project.uuid, compute, vm_name + ) + compute.bind_port(vm_port, mtu_request=project.int_net.mtu) + + project.vm_ports.append(vm_port) + + def _get_gateway_chassis(self, count: int = 1) -> List[ChassisNode]: + """Return list of Gateway Chassis with size defined by 'count'. + + Chassis are picked at random from all available 'worker_nodes' to + attempt to evenly distribute gateway loads. + + Parameter 'count' can't be larger than number of all available gateways + or larger than 5 which is hardcoded maximum of gateways per router in + Neutron. + """ + warn = "" + worker_count = len(self.worker_nodes) + if count > worker_count: + count = worker_count + warn += ( + f"{count} Gateway chassis requested but only " + f"{worker_count} available.\n" + ) + + if count > self.MAX_GW_PER_ROUTER: + count = self.MAX_GW_PER_ROUTER + warn += ( + f"Maximum number of gateways per router " + f"is {self.MAX_GW_PER_ROUTER}\n" + ) + + if warn: + warn += f"Using only {count} Gateways per router." + log.warning(warn) + + return random.sample(self.worker_nodes, count) + + def _create_project_net(self, net_name: str, mtu: int = 1500) -> LSwitch: + """Create Logical Switch that represents neutron network. + + This method creates Logical Switch with same parameters as Neutron + would. + """ + switch_ext_ids = { + "neutron:availability_zone_hints": "", + "neutron:mtu": str(mtu), + "neutron:network_name": net_name, + "neutron:revision_number": "1", + } + switch_config = { + "mcast_flood_unregistered": "false", + "mcast_snoop": "false", + "vlan-passthru": "false", + } + + neutron_net_name = f"neutron-{uuid.uuid4()}" + neutron_network = self.nbctl.ls_add( + neutron_net_name, + self.next_int_net(), + ext_ids=switch_ext_ids, + other_config=switch_config, + ) + + return neutron_network + + def _add_network_subnet(self, network: NeutronNetwork) -> None: + """Create DHCP subnet for a network. + + Adding subnet to a network in Neutron results in DHCP_Options being + created in OVN. + """ + external_ids = { + "neutron:revision_number": "0", + "subnet_id": str(uuid.uuid4()), + } + static_routes = ( + f"{{169.254.169.254/32,{network.gateway}, " + f"0.0.0.0/0,{network.gateway}}}" + ) + options = { + "classless_static_route": static_routes, + "dns_server": "{1.1.1.1}", + "lease_time": "43200", + "mtu": "1442", + "router": str(network.gateway), + "server_id": str(network.gateway), + "server_mac": str(RandMac()), + } + dhcp_options = self.nbctl.create_dhcp_options( + str(network.network.cidr), ext_ids=external_ids + ) + + self.nbctl.dhcp_options_set_options(dhcp_options.uuid, options) + + def _assign_port_ips(self, network: NeutronNetwork) -> None: + """Assign IPs to each LS port associated with the Network.""" + for uuid_, port in network.ports.items(): + if port.ip is None or port.ip == "unknown": + updated_port = self.nbctl.ls_port_set_ipv4_address( + port, str(network.next_host_ip()) + ) + network.ports[uuid_] = updated_port + # XXX: Unable to update external_ids + + def _add_vm_port( + self, + neutron_net: NeutronNetwork, + project_id: str, + chassis: 'ComputeNode', + vm_name: str, + ) -> LSPort: + """Create port that will represent interface of a VM. + + :param neutron_net: Network in which the port will be created. + :param project_id: ID of a project to which the VM belongs. + :param chassis: Chassis on which the port will be provisioned. + :param vm_name: VM name that's used to derive port name. Beware that + due to the port naming convention and limits on + interface name length, this name can't be longer than + 12 characters. + :return: LSPort representing VM's network interface + """ + port_name = f"lp-{vm_name}" + if len(port_name) > 15: + raise RuntimeError( + f"Maximum port name length is 15. Port {port_name} is too " + f"long. Consider using shorter VM name." + ) + port_addr = neutron_net.next_host_ip() + net_addr: netaddr.IPNetwork = neutron_net.network.cidr + port_ext_ids = { + "neutron:cidrs": f"{port_addr}/{net_addr.prefixlen}", + "neutron:device_id": str(uuid.uuid4()), + "neutron:device_owner": "compute:nova", + "neutron:network_name": neutron_net.name, + "neutron:port_name": "", + "neutron:project_id": project_id, + "neutron:revision_number": "1", + "neutron:security_group_ids": neutron_net.security_group.name, + "neutron:subnet_pool_addr_scope4": "", + "neutron:subnet_pool_addr_scope6": "", + } + port_options = f"requested-chassis={chassis.container}" + ls_port = self.nbctl.ls_port_add( + lswitch=neutron_net.network, + name=port_name, + mac=str(RandMac()), + ip=DualStackIP(port_addr, net_addr.prefixlen, None, None), + ext_ids=port_ext_ids, + gw=DualStackIP(neutron_net.gateway, None, None, None), + metadata=chassis, + ) + self.nbctl.ls_port_set_set_options(ls_port, port_options) + self.nbctl.ls_port_enable(ls_port) + self.nbctl.port_group_add_ports( + pg=neutron_net.security_group, lports=[ls_port] + ) + + return ls_port + + def _add_metadata_port(self, network: LSwitch, project_id: str) -> LSPort: + """Create metadata port in LSwitch with Neutron's external IDs.""" + port_ext_ids = { + "neutron:cidrs": "", + "neutron:device_id": f"ovnmeta-{network.uuid}", + "neutron:device_owner": "network:distributed", + "neutron:network_name": network.name, + "neutron:port_name": "", + "neutron:project_id": project_id, + "neutron:revision_number": "1", + "neutron:security_group_ids": "", + "neutron:subnet_pool_addr_scope4": "", + "neutron:subnet_pool_addr_scope6": "", + } + + port = self.nbctl.ls_port_add( + lswitch=network, + name=str(uuid.uuid4()), + mac=str(RandMac()), + ext_ids=port_ext_ids, + ) + self.nbctl.ls_port_set_set_type(port, "localport") + self.nbctl.ls_port_enable(port) + + return port + + def _add_provider_network_port( + self, network: LSwitch, network_name: str + ) -> LSPort: + """Add port to Logical Switch that represents connection to ext. net + :param network: Network (LSwitch) in which the port will be created. + :param network_name: Name of the provider network (used when setting + provider port options) + :return: LSPort representing provider port + """ + options = ( + "mcast_flood=false mcast_flood_reports=true " + f"network_name={network_name}" + ) + port = self.nbctl.ls_port_add( + lswitch=network, + name=f"provnet-{uuid.uuid4()}", + localnet=True, + ) + self.nbctl.ls_port_set_set_type(port, "localnet") + self.nbctl.ls_port_set_set_options(port, options) + + return port + + def _add_router_port_internal( + self, neutron_net: NeutronNetwork, router: LRouter, project: Project + ) -> (LSPort, LRPort): + """Add a pair of ports that connect router to the internal network. + + The Router Port is automatically assigned network's gateway IP. + + :param neutron_net: Neutron network that represents internal network + :param router: Router to which the network is connected + :param project: Project to which 'neutron_net' belongs + :return: The pair of Logical Switch Port and Logical Router Port + """ + port_ip = DualStackIP( + neutron_net.gateway, neutron_net.network.cidr.prefixlen, None, None + ) + return self._add_router_port( + neutron_net.network, router, port_ip, project.uuid, False + ) + + def _add_router_port_external_gw( + self, neutron_net: NeutronNetwork, router: LRouter + ) -> (LSPort, LRPort): + """Add a pair of ports that connect router to the external network. + + The Router Port is assigned IP form external network's range. + + :param neutron_net: Neutron network that represents external network + :param router: Router to which the network is connected + :return: The pair of Logical Switch Port and Logical Router Port + """ + port_ip = self.next_external_ip() + return self._add_router_port( + neutron_net.network, router, port_ip, "", True, neutron_net.mtu + ) + + def _add_router_port( + self, + network: LSwitch, + router: LRouter, + port_ip: DualStackIP, + project_id: str = "", + is_gw: bool = False, + mtu: Optional[int] = None, + ) -> (LSPort, LRPort): + """Add a pair of ports that connect Logical Router and Logical Switch. + + :param network: Logical Switch to which a Logical Switch Port is added + :param router: Logical Router to which a Logical Router Port is added + :param port_ip: IP assigned to the router port + :param project_id: Optional Openstack Project ID that's associated with + the network. + :param is_gw: Set to True if Router port is connected to external + network and should act as gateway. + :return: The pair of Logical Switch Port and Logical Router Port + """ + subnet_id = str(uuid.uuid4()) + port_name = uuid.uuid4() + router_port_name = f"lrp-{port_name}" + device_owner = "network:router_gateway" if is_gw else "" + # XXX: neutron adds "neutron-" prefix to router name (which is UUID), + # but for the ports' external IDs we want to extract just the + # UUID. + router_id = router.name.lstrip("neutron-") + + lsp_external_ids = { + "neutron:cidrs": str(self.cluster_cfg.external_net.n4), + "neutron:device_id": router_id, + "neutron:device_owner": device_owner, + "neutron:network_name": network.name, + "neutron:port_name": "", + "neutron:project_id": project_id, + "neutron:revision_number": "1", + "neutron:security_group_ids": "", + "neutron:subnet_pool_addr_scope4": "", + "neutron:subnet_pool_addr_scope6": "", + } + lrp_external_ids = { + "neutron:network_name": network.name, + "neutron:revision_number": "1", + "neutron:router_name": router_id, + "neutron:subnet_ids": subnet_id, + } + + lr_port = self.nbctl.lr_port_add( + router, + router_port_name, + str(RandMac()), + port_ip, + lrp_external_ids, + {"gateway_mtu": str(mtu)} if mtu else None, + ) + + ls_port = self.nbctl.ls_port_add( + lswitch=network, + name=str(port_name), + router_port=lr_port, + ext_ids=lsp_external_ids, + ) + self.nbctl.ls_port_enable(ls_port) + + if is_gw: + lsp_options = ( + "exclude-lb-vips-from-garp=true, " + "nat-addresses=router, " + f"router-port={router_port_name}" + ) + self.nbctl.ls_port_set_set_options(ls_port, lsp_options) + + return ls_port, lr_port + + def _create_default_security_group(self) -> PortGroup: + """Create default security group for the project. + + This method provisions Port Group and default ACL rules. + """ + neutron_security_group = str(uuid.uuid4()) + pg_name = f"pg_{neutron_security_group}".replace("-", "_") + pg_ext_ids = {"neutron:security_group_id": neutron_security_group} + port_group = self.nbctl.port_group_create(pg_name, ext_ids=pg_ext_ids) + + in_rules = [ + f"inport == @{pg_name} && ip4", + f"inport == @{pg_name} && ip6", + ] + out_rules = [ + f"outport == @{pg_name} && ip4 && ip4.src == $pg_{pg_name}_ip4", + f"outport == @{pg_name} && ip6 && ip6.src == $pg_{pg_name}_ip6", + ] + + for rule in in_rules: + ext_ids = {"neutron:security_group_rule_id": str(uuid.uuid4())} + self.nbctl.acl_add( + name=port_group.name, + direction="from-lport", + priority=1002, + entity="port-group", + match=rule, + verdict="allow-related", + ext_ids=ext_ids, + ) + + for rule in out_rules: + ext_ids = {"neutron:security_group_rule_id": str(uuid.uuid4())} + self.nbctl.acl_add( + name=port_group.name, + direction="to-lport", + priority=1002, + entity="port-group", + match=rule, + verdict="allow-related", + ext_ids=ext_ids, + ) + + return port_group + + def _create_project_router(self, name: str) -> LRouter: + """Create Logical Router in the same way as Neutron would.""" + options = { + "always_learn_from_arp_request": "false", + "dynamic_neigh_routers": "true", + } + external_ids = { + "neutron:availability_zone_hints": "", + "neutron:gw_port_id": "", + "neutron:revision_number": "1", + "neutron:router_name": name, + } + router = self.nbctl.lr_add(f"neutron-{uuid.uuid4()}", external_ids) + self.nbctl.lr_set_options(router, options) + + return router + + +class NetworkNode(ChassisNode): + """Represent a network node, a node with OVS/OVN but no VMs.""" + + def __init__( + self, + phys_node, + container, + mgmt_ip, + protocol, + is_gateway: bool = True, + ): + super().__init__(phys_node, container, mgmt_ip, protocol) + self.is_gateway = is_gateway + + def configure(self, physical_net): + pass + + def provision(self, cluster: OpenStackCloud) -> None: + """Connect Chassis to OVN Central.""" + self.connect(cluster.get_relay_connection_string()) + + +class ComputeNode(ChassisNode): + """Represent a compute node, a node with OVS/OVN as well as VMs.""" + + def __init__( + self, + phys_node, + container, + mgmt_ip, + protocol, + is_gateway: bool = True, + ): + super().__init__(phys_node, container, mgmt_ip, protocol) + self.is_gateway = is_gateway + + def configure(self, physical_net): + pass + + def provision(self, cluster: OpenStackCloud) -> None: + """Connect Chassis to OVN Central.""" + self.connect(cluster.get_relay_connection_string()) diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py new file mode 100644 index 00000000..e595094d --- /dev/null +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -0,0 +1,63 @@ +import logging + +from dataclasses import dataclass +from typing import List + +from ovn_ext_cmd import ExtCmd +from ovn_context import Context +from ovn_workload import ChassisNode + +from cms.openstack import OpenStackCloud, ExternalNetworkSpec + +log = logging.getLogger(__name__) + + +@dataclass +class BaseOpenstackConfig: + n_projects: int = 1 + n_chassis_per_gw_lrp: int = 1 + n_vms_per_project: int = 3 + + +class BaseOpenstack(ExtCmd): + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) + test_config = config.get("base_openstack") + self.config = BaseOpenstackConfig(**test_config) + + def run(self, clouds: List[OpenStackCloud], global_cfg): + # create ovn topology + with Context(clouds, "base_openstack_bringup", len(clouds)) as ctx: + for i in ctx: + ovn = clouds[i] + worker_count = len(ovn.worker_nodes) + for i in range(worker_count): + worker_node: ChassisNode = ovn.worker_nodes[i] + log.info( + f"Provisioning {worker_node.__class__.__name__} " + f"({i+1}/{worker_count})" + ) + worker_node.provision(ovn) + + with Context(clouds, "base_openstack_provision", len(clouds)) as ctx: + for i in ctx: + ovn = clouds[i] + ext_net = ExternalNetworkSpec( + neutron_net=ovn.new_external_network(), + num_gw_nodes=self.config.n_chassis_per_gw_lrp, + ) + + for _ in range(self.config.n_projects): + _ = ovn.new_project(ext_net=ext_net) + + for project in ovn.projects: + for index in range(self.config.n_vms_per_project): + ovn.add_vm_to_project( + project, f"{project.uuid[:6]}-{index}" + ) + + with Context(clouds, "base_openstack", len(clouds)) as ctx: + for i in ctx: + ovn = clouds[i] + for project in ovn.projects: + ovn.mesh_ping_ports(project.vm_ports) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 7dfaeed4..c52d2b2b 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -1,22 +1,371 @@ -from ovn_context import Context -from ovn_workload import WorkerNode +import logging +from collections import namedtuple + +import netaddr + +from randmac import RandMac + +import ovn_load_balancer as lb +import ovn_utils +import ovn_stats + from ovn_utils import DualStackSubnet +from ovn_workload import ChassisNode, Cluster + +log = logging.getLogger(__name__) +ClusterBringupCfg = namedtuple('ClusterBringupCfg', ['n_pods_per_node']) +OVN_HEATER_CMS_PLUGIN = 'OVNKubernetesCluster' +ACL_DEFAULT_DENY_PRIO = 1 +ACL_DEFAULT_ALLOW_ARP_PRIO = 2 +ACL_NETPOL_ALLOW_PRIO = 3 +DEFAULT_NS_VIP_SUBNET = netaddr.IPNetwork('30.0.0.0/16') +DEFAULT_NS_VIP_SUBNET6 = netaddr.IPNetwork('30::/32') +DEFAULT_VIP_PORT = 80 +DEFAULT_BACKEND_PORT = 8080 + + +class Namespace: + def __init__(self, clusters, name, global_cfg): + self.clusters = clusters + self.nbctl = [cluster.nbctl for cluster in clusters] + self.ports = [[] for _ in range(len(clusters))] + self.enforcing = False + self.pg_def_deny_igr = [ + nbctl.port_group_create(f'pg_deny_igr_{name}') + for nbctl in self.nbctl + ] + self.pg_def_deny_egr = [ + nbctl.port_group_create(f'pg_deny_egr_{name}') + for nbctl in self.nbctl + ] + self.pg = [ + nbctl.port_group_create(f'pg_{name}') for nbctl in self.nbctl + ] + self.addr_set4 = [ + ( + nbctl.address_set_create(f'as_{name}') + if global_cfg.run_ipv4 + else None + ) + for nbctl in self.nbctl + ] + self.addr_set6 = [ + ( + nbctl.address_set_create(f'as6_{name}') + if global_cfg.run_ipv6 + else None + ) + for nbctl in self.nbctl + ] + self.sub_as = [[] for _ in range(len(clusters))] + self.sub_pg = [[] for _ in range(len(clusters))] + self.load_balancer = None + for cluster in self.clusters: + cluster.n_ns += 1 + self.name = name + + @ovn_stats.timeit + def add_ports(self, ports, az=0): + self.ports[az].extend(ports) + # Always add port IPs to the address set but not to the PGs. + # Simulate what OpenShift does, which is: create the port groups + # when the first network policy is applied. + if self.addr_set4: + for i, nbctl in enumerate(self.nbctl): + nbctl.address_set_add_addrs( + self.addr_set4[i], [str(p.ip) for p in ports] + ) + if self.addr_set6: + for i, nbctl in enumerate(self.nbctl): + nbctl.address_set_add_addrs( + self.addr_set6[i], [str(p.ip6) for p in ports] + ) + if self.enforcing: + self.nbctl[az].port_group_add_ports( + self.pg_def_deny_igr[az], ports + ) + self.nbctl[az].port_group_add_ports( + self.pg_def_deny_egr[az], ports + ) + self.nbctl[az].port_group_add_ports(self.pg[az], ports) + + def unprovision(self): + # ACLs are garbage collected by OVSDB as soon as all the records + # referencing them are removed. + for i, cluster in enumerate(self.clusters): + cluster.unprovision_ports(self.ports[i]) + for i, nbctl in enumerate(self.nbctl): + nbctl.port_group_del(self.pg_def_deny_igr[i]) + nbctl.port_group_del(self.pg_def_deny_egr[i]) + nbctl.port_group_del(self.pg[i]) + if self.addr_set4: + nbctl.address_set_del(self.addr_set4[i]) + if self.addr_set6: + nbctl.address_set_del(self.addr_set6[i]) + nbctl.port_group_del(self.sub_pg[i]) + nbctl.address_set_del(self.sub_as[i]) + + def unprovision_ports(self, ports, az=0): + '''Unprovision a subset of ports in the namespace without having to + unprovision the entire namespace or any of its network policies.''' + for port in ports: + self.ports[az].remove(port) -OVN_HEATER_CMS_PLUGIN = 'OVNKubernetes' + self.clusters[az].unprovision_ports(ports) + def enforce(self): + if self.enforcing: + return + self.enforcing = True + for i, nbctl in enumerate(self.nbctl): + nbctl.port_group_add_ports(self.pg_def_deny_igr[i], self.ports[i]) + nbctl.port_group_add_ports(self.pg_def_deny_egr[i], self.ports[i]) + nbctl.port_group_add_ports(self.pg[i], self.ports[i]) -class OVNKubernetes: - @staticmethod - def add_cluster_worker_nodes(cluster, workers, az): - cluster_cfg = cluster.cluster_cfg + def create_sub_ns(self, ports, global_cfg, az=0): + n_sub_pgs = len(self.sub_pg[az]) + suffix = f'{self.name}_{n_sub_pgs}' + pg = self.nbctl[az].port_group_create(f'sub_pg_{suffix}') + self.nbctl[az].port_group_add_ports(pg, ports) + self.sub_pg[az].append(pg) + for i, nbctl in enumerate(self.nbctl): + if global_cfg.run_ipv4: + addr_set = nbctl.address_set_create(f'sub_as_{suffix}') + nbctl.address_set_add_addrs( + addr_set, [str(p.ip) for p in ports] + ) + self.sub_as[i].append(addr_set) + if global_cfg.run_ipv6: + addr_set = nbctl.address_set_create(f'sub_as_{suffix}6') + nbctl.address_set_add_addrs( + addr_set, [str(p.ip6) for p in ports] + ) + self.sub_as[i].append(addr_set) + return n_sub_pgs + + @ovn_stats.timeit + def default_deny(self, family, az=0): + self.enforce() + + addr_set = f'self.addr_set{family}.name' + self.nbctl[az].acl_add( + self.pg_def_deny_igr[az].name, + 'to-lport', + ACL_DEFAULT_DENY_PRIO, + 'port-group', + f'ip4.src == \\${addr_set} && ' + f'outport == @{self.pg_def_deny_igr[az].name}', + 'drop', + ) + self.nbctl[az].acl_add( + self.pg_def_deny_egr[az].name, + 'to-lport', + ACL_DEFAULT_DENY_PRIO, + 'port-group', + f'ip4.dst == \\${addr_set} && ' + f'inport == @{self.pg_def_deny_egr[az].name}', + 'drop', + ) + self.nbctl[az].acl_add( + self.pg_def_deny_igr[az].name, + 'to-lport', + ACL_DEFAULT_ALLOW_ARP_PRIO, + 'port-group', + f'outport == @{self.pg_def_deny_igr[az].name} && arp', + 'allow', + ) + self.nbctl[az].acl_add( + self.pg_def_deny_egr[az].name, + 'to-lport', + ACL_DEFAULT_ALLOW_ARP_PRIO, + 'port-group', + f'inport == @{self.pg_def_deny_egr[az].name} && arp', + 'allow', + ) + + @ovn_stats.timeit + def allow_within_namespace(self, family, az=0): + self.enforce() + + addr_set = f'self.addr_set{family}.name' + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.src == \\${addr_set} && ' f'outport == @{self.pg[az].name}', + 'allow-related', + ) + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.dst == \\${addr_set} && ' f'inport == @{self.pg[az].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def allow_cross_namespace(self, ns, family): + self.enforce() + + for az, nbctl in enumerate(self.nbctl): + if len(self.ports[az]) == 0: + continue + addr_set = f'self.addr_set{family}.name' + nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.src == \\${addr_set} && ' + f'outport == @{ns.pg[az].name}', + 'allow-related', + ) + ns_addr_set = f'ns.addr_set{family}.name' + nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.dst == \\${ns_addr_set} && ' + f'inport == @{self.pg[az].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def allow_sub_namespace(self, src, dst, family, az=0): + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip{family}.src == \\${self.sub_as[az][src].name} && ' + f'outport == @{self.sub_pg[az][dst].name}', + 'allow-related', + ) + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip{family}.dst == \\${self.sub_as[az][dst].name} && ' + f'inport == @{self.sub_pg[az][src].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def allow_from_external( + self, external_ips, include_ext_gw=False, family=4, az=0 + ): + self.enforce() + # If requested, include the ext-gw of the first port in the namespace + # so we can check that this rule is enforced. + if include_ext_gw: + assert len(self.ports) > 0 + if family == 4 and self.ports[az][0].ext_gw: + external_ips.append(self.ports[az][0].ext_gw) + elif family == 6 and self.ports[az][0].ext_gw6: + external_ips.append(self.ports[az][0].ext_gw6) + ips = [str(ip) for ip in external_ips] + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip.{family} == {{{",".join(ips)}}} && ' + f'outport == @{self.pg[az].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def check_enforcing_internal(self, az=0): + # "Random" check that first pod can reach last pod in the namespace. + if len(self.ports[az]) > 1: + src = self.ports[az][0] + dst = self.ports[az][-1] + worker = src.metadata + if src.ip: + worker.ping_port(self.clusters[az], src, dst.ip) + if src.ip6: + worker.ping_port(self.clusters[az], src, dst.ip6) + + @ovn_stats.timeit + def check_enforcing_external(self, az=0): + if len(self.ports[az]) > 0: + dst = self.ports[az][0] + worker = dst.metadata + worker.ping_external(self.clusters[az], dst) + + @ovn_stats.timeit + def check_enforcing_cross_ns(self, ns, az=0): + if len(self.ports[az]) > 0 and len(ns.ports[az]) > 0: + dst = ns.ports[az][0] + src = self.ports[az][0] + worker = src.metadata + if src.ip and dst.ip: + worker.ping_port(self.clusters[az], src, dst.ip) + if src.ip6 and dst.ip6: + worker.ping_port(self.clusters[az], src, dst.ip6) + + def create_load_balancer(self, az=0): + self.load_balancer = lb.OvnLoadBalancer( + f'lb_{self.name}', self.nbctl[az] + ) + + @ovn_stats.timeit + def provision_vips_to_load_balancers(self, backend_lists, version, az=0): + vip_ns_subnet = DEFAULT_NS_VIP_SUBNET + if version == 6: + vip_ns_subnet = DEFAULT_NS_VIP_SUBNET6 + vip_net = vip_ns_subnet.next(self.clusters[az].n_ns) + n_vips = len(self.load_balancer.vips.keys()) + vip_ip = vip_net.ip.__add__(n_vips + 1) + + if version == 6: + vips = { + f'[{vip_ip + i}]:{DEFAULT_VIP_PORT}': [ + f'[{p.ip6}]:{DEFAULT_BACKEND_PORT}' for p in ports + ] + for i, ports in enumerate(backend_lists) + } + self.load_balancer.add_vips(vips) + else: + vips = { + f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ + f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports + ] + for i, ports in enumerate(backend_lists) + } + self.load_balancer.add_vips(vips) + + +class OVNKubernetesCluster(Cluster): + def __init__(self, cluster_cfg, central, brex_cfg, az): + super().__init__(cluster_cfg, central, brex_cfg, az) + self.net = cluster_cfg.cluster_net + self.gw_net = ovn_utils.DualStackSubnet.next( + cluster_cfg.gw_net, + az * (cluster_cfg.n_workers // cluster_cfg.n_az), + ) + self.router = None + self.load_balancer = None + self.load_balancer6 = None + self.join_switch = None + self.last_selected_worker = 0 + self.n_ns = 0 + self.ts_switch = None + + def add_cluster_worker_nodes(self, workers): + cluster_cfg = self.cluster_cfg # Allocate worker IPs after central and relay IPs. mgmt_ip = ( cluster_cfg.node_net.ip + 2 + cluster_cfg.n_az - * (len(cluster.central_nodes) + len(cluster.relay_nodes)) + * (len(self.central_nodes) + len(self.relay_nodes)) ) protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" @@ -24,7 +373,7 @@ def add_cluster_worker_nodes(cluster, workers, az): external_net = cluster_cfg.external_net # Number of workers for each az n_az_workers = cluster_cfg.n_workers // cluster_cfg.n_az - cluster.add_workers( + self.add_workers( [ WorkerNode( workers[i % len(workers)], @@ -33,15 +382,264 @@ def add_cluster_worker_nodes(cluster, workers, az): protocol, DualStackSubnet.next(internal_net, i), DualStackSubnet.next(external_net, i), - cluster.gw_net, + self.gw_net, i, ) - for i in range(az * n_az_workers, (az + 1) * n_az_workers) + for i in range( + self.az * n_az_workers, (self.az + 1) * n_az_workers + ) ] ) - @staticmethod - def prepare_test(clusters): - with Context(clusters, 'prepare_test clusters'): - for c in clusters: - c.start() + def create_cluster_router(self, rtr_name): + self.router = self.nbctl.lr_add(rtr_name) + self.nbctl.lr_set_options( + self.router, + { + 'always_learn_from_arp_request': 'false', + }, + ) + + def create_cluster_load_balancer(self, lb_name, global_cfg): + if global_cfg.run_ipv4: + self.load_balancer = lb.OvnLoadBalancer( + lb_name, self.nbctl, self.cluster_cfg.vips + ) + self.load_balancer.add_vips(self.cluster_cfg.static_vips) + + if global_cfg.run_ipv6: + self.load_balancer6 = lb.OvnLoadBalancer( + f'{lb_name}6', self.nbctl, self.cluster_cfg.vips6 + ) + self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) + + def create_cluster_join_switch(self, sw_name): + self.join_switch = self.nbctl.ls_add(sw_name, net_s=self.gw_net) + + self.join_rp = self.nbctl.lr_port_add( + self.router, + f'rtr-to-{sw_name}', + RandMac(), + self.gw_net.reverse(), + ) + self.join_ls_rp = self.nbctl.ls_port_add( + self.join_switch, f'{sw_name}-to-rtr', self.join_rp + ) + + @ovn_stats.timeit + def provision_vips_to_load_balancers(self, backend_lists): + n_vips = len(self.load_balancer.vips.keys()) + vip_ip = self.cluster_cfg.vip_subnet.ip.__add__(n_vips + 1) + + vips = { + f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ + f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports + ] + for i, ports in enumerate(backend_lists) + } + self.load_balancer.add_vips(vips) + + def unprovision_vips(self): + if self.load_balancer: + self.load_balancer.clear_vips() + self.load_balancer.add_vips(self.cluster_cfg.static_vips) + if self.load_balancer6: + self.load_balancer6.clear_vips() + self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) + + def provision_lb_group(self, name='cluster-lb-group'): + self.lb_group = lb.OvnLoadBalancerGroup(name, self.nbctl) + for w in self.worker_nodes: + self.nbctl.ls_add_lbg(w.switch, self.lb_group.lbg) + self.nbctl.lr_add_lbg(w.gw_router, self.lb_group.lbg) + + def provision_lb(self, lb): + log.info(f'Creating load balancer {lb.name}') + self.lb_group.add_lb(lb) + + +class WorkerNode(ChassisNode): + def __init__( + self, + phys_node, + container, + mgmt_ip, + protocol, + int_net, + ext_net, + gw_net, + unique_id, + ): + super().__init__(phys_node, container, mgmt_ip, protocol) + self.int_net = int_net + self.ext_net = ext_net + self.gw_net = gw_net + self.id = unique_id + + def configure(self, physical_net): + self.configure_localnet(physical_net) + phys_ctl = ovn_utils.PhysCtl(self) + phys_ctl.external_host_provision( + ip=self.ext_net.reverse(2), gw=self.ext_net.reverse() + ) + + @ovn_stats.timeit + def provision(self, cluster): + self.connect(cluster.get_relay_connection_string()) + self.wait(cluster.sbctl, cluster.cluster_cfg.node_timeout_s) + + # Create a node switch and connect it to the cluster router. + self.switch = cluster.nbctl.ls_add( + f'lswitch-{self.container}', net_s=self.int_net + ) + lrp_name = f'rtr-to-node-{self.container}' + ls_rp_name = f'node-to-rtr-{self.container}' + self.rp = cluster.nbctl.lr_port_add( + cluster.router, lrp_name, RandMac(), self.int_net.reverse() + ) + self.ls_rp = cluster.nbctl.ls_port_add( + self.switch, ls_rp_name, self.rp + ) + + # Make the lrp as distributed gateway router port. + cluster.nbctl.lr_port_set_gw_chassis(self.rp, self.container) + + # Create a gw router and connect it to the cluster join switch. + self.gw_router = cluster.nbctl.lr_add(f'gwrouter-{self.container}') + cluster.nbctl.lr_set_options( + self.gw_router, + { + 'always_learn_from_arp_request': 'false', + 'dynamic_neigh_routers': 'true', + 'chassis': self.container, + 'lb_force_snat_ip': 'router_ip', + 'snat-ct-zone': 0, + }, + ) + join_grp_name = f'gw-to-join-{self.container}' + join_ls_grp_name = f'join-to-gw-{self.container}' + + gr_gw = self.gw_net.reverse(self.id + 2) + self.gw_rp = cluster.nbctl.lr_port_add( + self.gw_router, join_grp_name, RandMac(), gr_gw + ) + self.join_gw_rp = cluster.nbctl.ls_port_add( + cluster.join_switch, join_ls_grp_name, self.gw_rp + ) + + # Create an external switch connecting the gateway router to the + # physnet. + self.ext_switch = cluster.nbctl.ls_add( + f'ext-{self.container}', net_s=self.ext_net + ) + ext_lrp_name = f'gw-to-ext-{self.container}' + ext_ls_rp_name = f'ext-to-gw-{self.container}' + self.ext_rp = cluster.nbctl.lr_port_add( + self.gw_router, ext_lrp_name, RandMac(), self.ext_net.reverse() + ) + self.ext_gw_rp = cluster.nbctl.ls_port_add( + self.ext_switch, ext_ls_rp_name, self.ext_rp + ) + + # Configure physnet. + self.physnet_port = cluster.nbctl.ls_port_add( + self.ext_switch, + f'provnet-{self.container}', + localnet=True, + ) + cluster.nbctl.ls_port_set_set_type(self.physnet_port, 'localnet') + cluster.nbctl.ls_port_set_set_options( + self.physnet_port, f'network_name={cluster.brex_cfg.physical_net}' + ) + + # Route for traffic entering the cluster. + cluster.nbctl.route_add( + self.gw_router, cluster.net, self.gw_net.reverse() + ) + + # Default route to get out of cluster via physnet. + cluster.nbctl.route_add( + self.gw_router, + ovn_utils.DualStackSubnet( + netaddr.IPNetwork("0.0.0.0/0"), netaddr.IPNetwork("::/0") + ), + self.ext_net.reverse(2), + ) + + # Route for traffic that needs to exit the cluster + # (via gw router). + cluster.nbctl.route_add( + cluster.router, self.int_net, gr_gw, policy="src-ip" + ) + + # SNAT traffic leaving the cluster. + cluster.nbctl.nat_add(self.gw_router, gr_gw, cluster.net) + + @ovn_stats.timeit + def provision_port(self, cluster, passive=False): + name = f'lp-{self.id}-{self.next_lport_index}' + + log.info(f'Creating lport {name}') + lport = cluster.nbctl.ls_port_add( + self.switch, + name, + mac=str(RandMac()), + ip=self.int_net.forward(self.next_lport_index + 1), + gw=self.int_net.reverse(), + ext_gw=self.ext_net.reverse(2), + metadata=self, + passive=passive, + security=True, + ) + + self.lports.append(lport) + self.next_lport_index += 1 + return lport + + @ovn_stats.timeit + def provision_load_balancers(self, cluster, ports, global_cfg): + # Add one port IP as a backend to the cluster load balancer. + if global_cfg.run_ipv4: + port_ips = ( + f'{port.ip}:{DEFAULT_BACKEND_PORT}' + for port in ports + if port.ip is not None + ) + cluster_vips = cluster.cluster_cfg.vips.keys() + cluster.load_balancer.add_backends_to_vip(port_ips, cluster_vips) + cluster.load_balancer.add_to_switches([self.switch.name]) + cluster.load_balancer.add_to_routers([self.gw_router.name]) + + if global_cfg.run_ipv6: + port_ips6 = ( + f'[{port.ip6}]:{DEFAULT_BACKEND_PORT}' + for port in ports + if port.ip6 is not None + ) + cluster_vips6 = cluster.cluster_cfg.vips6.keys() + cluster.load_balancer6.add_backends_to_vip( + port_ips6, cluster_vips6 + ) + cluster.load_balancer6.add_to_switches([self.switch.name]) + cluster.load_balancer6.add_to_routers([self.gw_router.name]) + + # GW Load balancer has no VIPs/backends configured on it, since + # this load balancer is used for hostnetwork services. We're not + # using those right now so the load blaancer is empty. + if global_cfg.run_ipv4: + self.gw_load_balancer = lb.OvnLoadBalancer( + f'lb-{self.gw_router.name}', cluster.nbctl + ) + self.gw_load_balancer.add_to_routers([self.gw_router.name]) + if global_cfg.run_ipv6: + self.gw_load_balancer6 = lb.OvnLoadBalancer( + f'lb-{self.gw_router.name}6', cluster.nbctl + ) + self.gw_load_balancer6.add_to_routers([self.gw_router.name]) + + @ovn_stats.timeit + def ping_external(self, cluster, port): + if port.ip: + self.run_ping(cluster, 'ext-ns', port.ip) + if port.ip6: + self.run_ping(cluster, 'ext-ns', port.ip6) diff --git a/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py b/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py index b397cd04..d3ab8017 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd import ovn_exceptions diff --git a/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py b/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py index e34ca600..1c3d87f1 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd import ovn_load_balancer as lb import ovn_exceptions diff --git a/ovn-tester/cms/ovn_kubernetes/tests/density_light.py b/ovn-tester/cms/ovn_kubernetes/tests/density_light.py index 3732e3f3..a0bf135d 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/density_light.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/density_light.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol.py index 7601d862..9feda706 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd from itertools import chain import ovn_exceptions diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py index 4ddcf0a6..7677cf53 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd NpCrossNsCfg = namedtuple('NpCrossNsCfg', ['n_ns', 'pods_ns_ratio']) diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py index 6ef719f3..9dc43061 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py @@ -1,7 +1,7 @@ from collections import namedtuple import netaddr from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd diff --git a/ovn-tester/cms/ovn_kubernetes/tests/service_route.py b/ovn-tester/cms/ovn_kubernetes/tests/service_route.py index 5f333a80..0b3a297b 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/service_route.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/service_route.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd import netaddr diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index e89addae..9c341096 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -10,13 +10,11 @@ import time from collections import namedtuple +from ovn_context import Context from ovn_sandbox import PhysicalNode from ovn_workload import ( BrExConfig, - CentralNode, - Cluster, ClusterConfig, - RelayNode, ) from ovn_utils import DualStackSubnet from ovs.stream import Stream @@ -178,7 +176,7 @@ def load_cms(cms_name): mod = importlib.import_module(f'cms.{cms_name}') class_name = getattr(mod, 'OVN_HEATER_CMS_PLUGIN') cls = getattr(mod, class_name) - return cls() + return cls def configure_tests(yaml, clusters, global_cfg): @@ -196,40 +194,6 @@ def configure_tests(yaml, clusters, global_cfg): return tests -def create_cluster(cluster_cfg, central, workers, brex_cfg, cms, az): - protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - db_containers = ( - [ - f'ovn-central-az{az+1}-1', - f'ovn-central-az{az+1}-2', - f'ovn-central-az{az+1}-3', - ] - if cluster_cfg.clustered_db - else [f'ovn-central-az{az+1}-1'] - ) - - mgmt_ip = cluster_cfg.node_net.ip + 2 + az * len(db_containers) - central_nodes = [ - CentralNode(central, c, mgmt_ip + i, protocol) - for i, c in enumerate(db_containers) - ] - - mgmt_ip = ( - cluster_cfg.node_net.ip - + 2 - + cluster_cfg.n_az * len(central_nodes) - + az * cluster_cfg.n_relays - ) - relay_nodes = [ - RelayNode(central, f'ovn-relay-az{az+1}-{i+1}', mgmt_ip + i, protocol) - for i in range(cluster_cfg.n_relays) - ] - - cluster = Cluster(central_nodes, relay_nodes, cluster_cfg, brex_cfg, az) - cms.add_cluster_worker_nodes(cluster, workers, az) - return cluster - - def set_ssl_keys(cluster_cfg): Stream.ssl_set_private_key_file(cluster_cfg.ssl_private_key) Stream.ssl_set_certificate_file(cluster_cfg.ssl_cert) @@ -253,19 +217,23 @@ def set_ssl_keys(cluster_cfg): ): raise ovn_exceptions.OvnInvalidConfigException() - cms = load_cms(global_cfg.cms_name) + cms_cls = load_cms(global_cfg.cms_name) central, workers = read_physical_deployment(sys.argv[1], global_cfg) clusters = [ - create_cluster(cluster_cfg, central, workers, brex_cfg, cms, i) + cms_cls(cluster_cfg, central, brex_cfg, i) for i in range(cluster_cfg.n_az) ] + for c in clusters: + c.add_cluster_worker_nodes(workers) tests = configure_tests(config, clusters, global_cfg) if cluster_cfg.enable_ssl: set_ssl_keys(cluster_cfg) - cms.prepare_test(clusters) + with Context(clusters, 'prepare_test clusters'): + for c in clusters: + c.prepare_test() for test in tests: test.run(clusters, global_cfg) sys.exit(0) diff --git a/ovn-tester/ovn_utils.py b/ovn-tester/ovn_utils.py index 10d02dbf..313afdae 100644 --- a/ovn-tester/ovn_utils.py +++ b/ovn-tester/ovn_utils.py @@ -5,6 +5,7 @@ import time from collections import namedtuple from functools import partial +from typing import Dict, List, Optional import ovsdbapp.schema.open_vswitch.impl_idl as ovs_impl_idl import ovsdbapp.schema.ovn_northbound.impl_idl as nb_impl_idl import ovsdbapp.schema.ovn_southbound.impl_idl as sb_impl_idl @@ -45,6 +46,7 @@ AddressSet = namedtuple('AddressSet', ['name']) LoadBalancer = namedtuple('LoadBalancer', ['name', 'uuid']) LoadBalancerGroup = namedtuple('LoadBalancerGroup', ['name', 'uuid']) +DhcpOptions = namedtuple("DhcpOptions", ["uuid", "cidr"]) DEFAULT_CTL_TIMEOUT = 60 @@ -207,7 +209,14 @@ def set_global_external_id(self, key, value): ("external_ids", {key: str(value)}), ).execute(check_error=True) - def add_port(self, port, bridge, internal=True, ifaceid=None): + def add_port( + self, + port, + bridge, + internal=True, + ifaceid=None, + mtu_request: Optional[int] = None, + ): name = port.name with self.idl.transaction(check_error=True) as txn: txn.add(self.idl.add_port(bridge, name)) @@ -219,6 +228,12 @@ def add_port(self, port, bridge, internal=True, ifaceid=None): txn.add( self.idl.iface_set_external_id(name, "iface-id", ifaceid) ) + if mtu_request: + txn.add( + self.idl.db_set( + "Interface", name, ("mtu_request", mtu_request) + ) + ) def del_port(self, port): self.idl.del_port(port.name).execute(check_error=True) @@ -436,28 +451,65 @@ def set_inactivity_probe(self, value): ("inactivity_probe", value), ).execute() - def lr_add(self, name): + def lr_add(self, name, ext_ids: Optional[Dict] = None): + ext_ids = {} if ext_ids is None else ext_ids + log.info(f'Creating lrouter {name}') - uuid = self.uuid_transaction(partial(self.idl.lr_add, name)) + uuid = self.uuid_transaction( + partial(self.idl.lr_add, name, external_ids=ext_ids) + ) return LRouter(name=name, uuid=uuid) - def lr_port_add(self, router, name, mac, dual_ip=None): + def lr_port_add( + self, + router, + name, + mac, + dual_ip=None, + ext_ids: Optional[Dict] = None, + options: Optional[Dict] = None, + ): + ext_ids = {} if ext_ids is None else ext_ids + options = {} if options is None else options networks = [] if dual_ip.ip4 and dual_ip.plen4: networks.append(f'{dual_ip.ip4}/{dual_ip.plen4}') if dual_ip.ip6 and dual_ip.plen6: networks.append(f'{dual_ip.ip6}/{dual_ip.plen6}') - self.idl.lrp_add(router.uuid, name, str(mac), networks).execute() + self.idl.lrp_add( + router.uuid, + name, + str(mac), + networks, + external_ids=ext_ids, + options=options, + ).execute() return LRPort(name=name, mac=mac, ip=dual_ip) def lr_port_set_gw_chassis(self, rp, chassis, priority=10): log.info(f'Setting gw chassis {chassis} for router port {rp.name}') self.idl.lrp_set_gateway_chassis(rp.name, chassis, priority).execute() - def ls_add(self, name, net_s): + def ls_add( + self, + name: str, + net_s: DualStackSubnet, + ext_ids: Optional[Dict] = None, + other_config: Optional[Dict] = None, + ) -> LSwitch: + ext_ids = {} if ext_ids is None else ext_ids + other_config = {} if other_config is None else other_config + log.info(f'Creating lswitch {name}') - uuid = self.uuid_transaction(partial(self.idl.ls_add, name)) + uuid = self.uuid_transaction( + partial( + self.idl.ls_add, + name, + external_ids=ext_ids, + other_config=other_config, + ) + ) return LSwitch( name=name, cidr=net_s.n4, @@ -478,17 +530,18 @@ def ls_get_uuid(self, name, timeout): def ls_port_add( self, - lswitch, - name, - router_port=None, - mac=None, - ip=None, - gw=None, - ext_gw=None, - metadata=None, - passive=False, - security=False, - localnet=False, + lswitch: LSwitch, + name: str, + router_port: Optional[LRPort] = None, + mac: Optional[str] = None, + ip: Optional[DualStackIP] = None, + gw: Optional[DualStackIP] = None, + ext_gw: Optional[DualStackIP] = None, + metadata=None, # typehint: ovn_workload.ChassisNode + passive: bool = False, + security: bool = False, + localnet: bool = False, + ext_ids: Optional[Dict] = None, ): columns = dict() if router_port: @@ -512,6 +565,9 @@ def ls_port_add( if security: columns["port_security"] = addresses + if ext_ids is not None: + columns["external_ids"] = ext_ids + uuid = self.uuid_transaction( partial(self.idl.lsp_add, lswitch.uuid, name, **columns) ) @@ -545,7 +601,15 @@ def ls_port_add( def ls_port_del(self, port): self.idl.lsp_del(port.name).execute() - def ls_port_set_set_options(self, port, options): + def ls_port_set_set_options(self, port: LSPort, options: str): + """Set 'options' column for Logical Switch Port. + + :param port: Logical Switch Port to modify + :param options: Space-separated key-value pairs that are set as + options. Keys and values are separated by '='. + i.e.: 'opt1=val1 opt2=val2' + :return: None + """ opts = dict( (k, v) for k, v in (element.split("=") for element in options.split()) @@ -555,14 +619,32 @@ def ls_port_set_set_options(self, port, options): def ls_port_set_set_type(self, port, lsp_type): self.idl.lsp_set_type(port.name, lsp_type).execute() - def port_group_create(self, name): - self.idl.pg_add(name).execute() + def ls_port_enable(self, port: LSPort) -> None: + """Set Logical Switch Port's state to 'enabled'.""" + self.idl.lsp_set_enabled(port.name, True).execute() + + def ls_port_set_ipv4_address(self, port: LSPort, addr: str) -> LSPort: + """Set Logical Switch Port's IPv4 address. + + :param port: LSPort to modify + :param addr: IPv4 address to set + :return: Modified LSPort object with updated 'ip' attribute + """ + addresses = [f"{port.mac} {addr}"] + log.info(f"Setting addresses for port {port.uuid}: {addresses}") + self.idl.lsp_set_addresses(port.uuid, addresses).execute() + + return port._replace(ip=addr) + + def port_group_create(self, name, ext_ids: Optional[Dict] = None): + ext_ids = {} if ext_ids is None else ext_ids + self.idl.pg_add(name, external_ids=ext_ids).execute() return PortGroup(name=name) def port_group_add(self, pg, lport): self.idl.pg_add_ports(pg.name, lport.uuid).execute() - def port_group_add_ports(self, pg, lports): + def port_group_add_ports(self, pg: PortGroup, lports: List[LSPort]): MAX_PORTS_IN_BATCH = 500 for i in range(0, len(lports), MAX_PORTS_IN_BATCH): lports_slice = lports[i : i + MAX_PORTS_IN_BATCH] @@ -601,14 +683,16 @@ def acl_add( entity="switch", match="", verdict="allow", + ext_ids: Optional[Dict] = None, ): + ext_ids = {} if ext_ids is None else ext_ids if entity == "switch": self.idl.acl_add( - name, direction, priority, match, verdict + name, direction, priority, match, verdict, **ext_ids ).execute() else: # "port-group" self.idl.pg_acl_add( - name, direction, priority, match, verdict + name, direction, priority, match, verdict, **ext_ids ).execute() def route_add(self, router, network, gw, policy="dst-ip"): @@ -708,6 +792,28 @@ def lb_remove_from_switches(self, lb, switches): for s in switches: txn.add(self.idl.ls_lb_del(s, lb.uuid, if_exists=True)) + def create_dhcp_options( + self, cidr: str, ext_ids: Optional[Dict] = None + ) -> DhcpOptions: + """Create entry in DHCP_Options table. + + :param cidr: DHCP address pool (i.e. '192.168.1.0/24') + :param ext_ids: Optional entries to 'external_ids' column + :return: DhcpOptions object + """ + ext_ids = {} if ext_ids is None else ext_ids + + log.info(f"Creating DHCP Options for {cidr}. External IDs: {ext_ids}") + add_command = self.idl.dhcp_options_add(cidr, **ext_ids) + add_command.execute() + + return DhcpOptions(add_command.result.uuid, cidr) + + def dhcp_options_set_options(self, uuid_: str, options: Dict) -> None: + """Set 'options' column for 'DHCP_Options' entry.""" + log.info(f"Setting DHCP options for {uuid_}: {options}") + self.idl.dhcp_options_set_options(uuid_, **options).execute() + def sync(self, wait="hv", timeout=DEFAULT_CTL_TIMEOUT): with self.idl.transaction( check_error=True, timeout=timeout, wait_type=wait diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index aeaac719..28f6af1a 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -3,13 +3,12 @@ import ovn_sandbox import ovn_stats import ovn_utils -import ovn_load_balancer as lb import time import netaddr from collections import namedtuple from collections import defaultdict -from randmac import RandMac from datetime import datetime +from typing import List, Optional log = logging.getLogger(__name__) @@ -153,23 +152,15 @@ def enable_trim_on_compaction(self): ) -class WorkerNode(Node): +class ChassisNode(Node): def __init__( self, phys_node, container, mgmt_ip, protocol, - int_net, - ext_net, - gw_net, - unique_id, ): super().__init__(phys_node, container, mgmt_ip, protocol) - self.int_net = int_net - self.ext_net = ext_net - self.gw_net = gw_net - self.id = unique_id self.switch = None self.gw_router = None self.ext_switch = None @@ -197,13 +188,6 @@ def configure_localnet(self, physical_net): 'ovn-bridge-mappings', f'{physical_net}:br-ex' ) - def configure(self, physical_net): - self.configure_localnet(physical_net) - phys_ctl = ovn_utils.PhysCtl(self) - phys_ctl.external_host_provision( - ip=self.ext_net.reverse(2), gw=self.ext_net.reverse() - ) - @ovn_stats.timeit def wait(self, sbctl, timeout_s): for _ in range(timeout_s * 10): @@ -212,119 +196,6 @@ def wait(self, sbctl, timeout_s): time.sleep(0.1) raise ovn_exceptions.OvnChassisTimeoutException() - @ovn_stats.timeit - def provision(self, cluster): - self.connect(cluster.get_relay_connection_string()) - self.wait(cluster.sbctl, cluster.cluster_cfg.node_timeout_s) - - # Create a node switch and connect it to the cluster router. - self.switch = cluster.nbctl.ls_add( - f'lswitch-{self.container}', net_s=self.int_net - ) - lrp_name = f'rtr-to-node-{self.container}' - ls_rp_name = f'node-to-rtr-{self.container}' - self.rp = cluster.nbctl.lr_port_add( - cluster.router, lrp_name, RandMac(), self.int_net.reverse() - ) - self.ls_rp = cluster.nbctl.ls_port_add( - self.switch, ls_rp_name, self.rp - ) - - # Make the lrp as distributed gateway router port. - cluster.nbctl.lr_port_set_gw_chassis(self.rp, self.container) - - # Create a gw router and connect it to the cluster join switch. - self.gw_router = cluster.nbctl.lr_add(f'gwrouter-{self.container}') - cluster.nbctl.lr_set_options( - self.gw_router, - { - 'always_learn_from_arp_request': 'false', - 'dynamic_neigh_routers': 'true', - 'chassis': self.container, - 'lb_force_snat_ip': 'router_ip', - 'snat-ct-zone': 0, - }, - ) - join_grp_name = f'gw-to-join-{self.container}' - join_ls_grp_name = f'join-to-gw-{self.container}' - - gr_gw = self.gw_net.reverse(self.id + 2) - self.gw_rp = cluster.nbctl.lr_port_add( - self.gw_router, join_grp_name, RandMac(), gr_gw - ) - self.join_gw_rp = cluster.nbctl.ls_port_add( - cluster.join_switch, join_ls_grp_name, self.gw_rp - ) - - # Create an external switch connecting the gateway router to the - # physnet. - self.ext_switch = cluster.nbctl.ls_add( - f'ext-{self.container}', net_s=self.ext_net - ) - ext_lrp_name = f'gw-to-ext-{self.container}' - ext_ls_rp_name = f'ext-to-gw-{self.container}' - self.ext_rp = cluster.nbctl.lr_port_add( - self.gw_router, ext_lrp_name, RandMac(), self.ext_net.reverse() - ) - self.ext_gw_rp = cluster.nbctl.ls_port_add( - self.ext_switch, ext_ls_rp_name, self.ext_rp - ) - - # Configure physnet. - self.physnet_port = cluster.nbctl.ls_port_add( - self.ext_switch, - f'provnet-{self.container}', - localnet=True, - ) - cluster.nbctl.ls_port_set_set_type(self.physnet_port, 'localnet') - cluster.nbctl.ls_port_set_set_options( - self.physnet_port, f'network_name={cluster.brex_cfg.physical_net}' - ) - - # Route for traffic entering the cluster. - cluster.nbctl.route_add( - self.gw_router, cluster.net, self.gw_net.reverse() - ) - - # Default route to get out of cluster via physnet. - cluster.nbctl.route_add( - self.gw_router, - ovn_utils.DualStackSubnet( - netaddr.IPNetwork("0.0.0.0/0"), netaddr.IPNetwork("::/0") - ), - self.ext_net.reverse(2), - ) - - # Route for traffic that needs to exit the cluster - # (via gw router). - cluster.nbctl.route_add( - cluster.router, self.int_net, gr_gw, policy="src-ip" - ) - - # SNAT traffic leaving the cluster. - cluster.nbctl.nat_add(self.gw_router, gr_gw, cluster.net) - - @ovn_stats.timeit - def provision_port(self, cluster, passive=False): - name = f'lp-{self.id}-{self.next_lport_index}' - - log.info(f'Creating lport {name}') - lport = cluster.nbctl.ls_port_add( - self.switch, - name, - mac=str(RandMac()), - ip=self.int_net.forward(self.next_lport_index + 1), - gw=self.int_net.reverse(), - ext_gw=self.ext_net.reverse(2), - metadata=self, - passive=passive, - security=True, - ) - - self.lports.append(lport) - self.next_lport_index += 1 - return lport - @ovn_stats.timeit def unprovision_port(self, cluster, port): cluster.nbctl.ls_port_del(port) @@ -332,50 +203,15 @@ def unprovision_port(self, cluster, port): self.lports.remove(port) @ovn_stats.timeit - def provision_load_balancers(self, cluster, ports, global_cfg): - # Add one port IP as a backend to the cluster load balancer. - if global_cfg.run_ipv4: - port_ips = ( - f'{port.ip}:{DEFAULT_BACKEND_PORT}' - for port in ports - if port.ip is not None - ) - cluster_vips = cluster.cluster_cfg.vips.keys() - cluster.load_balancer.add_backends_to_vip(port_ips, cluster_vips) - cluster.load_balancer.add_to_switches([self.switch.name]) - cluster.load_balancer.add_to_routers([self.gw_router.name]) - - if global_cfg.run_ipv6: - port_ips6 = ( - f'[{port.ip6}]:{DEFAULT_BACKEND_PORT}' - for port in ports - if port.ip6 is not None - ) - cluster_vips6 = cluster.cluster_cfg.vips6.keys() - cluster.load_balancer6.add_backends_to_vip( - port_ips6, cluster_vips6 - ) - cluster.load_balancer6.add_to_switches([self.switch.name]) - cluster.load_balancer6.add_to_routers([self.gw_router.name]) - - # GW Load balancer has no VIPs/backends configured on it, since - # this load balancer is used for hostnetwork services. We're not - # using those right now so the load blaancer is empty. - if global_cfg.run_ipv4: - self.gw_load_balancer = lb.OvnLoadBalancer( - f'lb-{self.gw_router.name}', cluster.nbctl - ) - self.gw_load_balancer.add_to_routers([self.gw_router.name]) - if global_cfg.run_ipv6: - self.gw_load_balancer6 = lb.OvnLoadBalancer( - f'lb-{self.gw_router.name}6', cluster.nbctl - ) - self.gw_load_balancer6.add_to_routers([self.gw_router.name]) - - @ovn_stats.timeit - def bind_port(self, port): + def bind_port(self, port, mtu_request: Optional[int] = None): log.info(f'Binding lport {port.name} on {self.container}') - self.vsctl.add_port(port, 'br-int', internal=True, ifaceid=port.name) + self.vsctl.add_port( + port, + 'br-int', + internal=True, + ifaceid=port.name, + mtu_request=mtu_request, + ) # Skip creating a netns for "passive" ports, we won't be sending # traffic on those. if not port.passive: @@ -421,13 +257,6 @@ def run_ping(self, cluster, src, dest): def ping_port(self, cluster, port, dest): self.run_ping(cluster, port.name, dest) - @ovn_stats.timeit - def ping_external(self, cluster, port): - if port.ip: - self.run_ping(cluster, 'ext-ns', port.ip) - if port.ip6: - self.run_ping(cluster, 'ext-ns', port.ip6) - def ping_ports(self, cluster, ports): for port in ports: if port.ip: @@ -438,361 +267,75 @@ def ping_ports(self, cluster, ports): def get_connection_string(self, port): return f"{self.protocol}:{self.mgmt_ip}:{port}" - -ACL_DEFAULT_DENY_PRIO = 1 -ACL_DEFAULT_ALLOW_ARP_PRIO = 2 -ACL_NETPOL_ALLOW_PRIO = 3 -DEFAULT_NS_VIP_SUBNET = netaddr.IPNetwork('30.0.0.0/16') -DEFAULT_NS_VIP_SUBNET6 = netaddr.IPNetwork('30::/32') -DEFAULT_VIP_PORT = 80 -DEFAULT_BACKEND_PORT = 8080 - - -class Namespace: - def __init__(self, clusters, name, global_cfg): - self.clusters = clusters - self.nbctl = [cluster.nbctl for cluster in clusters] - self.ports = [[] for _ in range(len(clusters))] - self.enforcing = False - self.pg_def_deny_igr = [ - nbctl.port_group_create(f'pg_deny_igr_{name}') - for nbctl in self.nbctl - ] - self.pg_def_deny_egr = [ - nbctl.port_group_create(f'pg_deny_egr_{name}') - for nbctl in self.nbctl - ] - self.pg = [ - nbctl.port_group_create(f'pg_{name}') for nbctl in self.nbctl - ] - self.addr_set4 = [ - ( - nbctl.address_set_create(f'as_{name}') - if global_cfg.run_ipv4 - else None - ) - for nbctl in self.nbctl - ] - self.addr_set6 = [ - ( - nbctl.address_set_create(f'as6_{name}') - if global_cfg.run_ipv6 - else None - ) - for nbctl in self.nbctl - ] - self.sub_as = [[] for _ in range(len(clusters))] - self.sub_pg = [[] for _ in range(len(clusters))] - self.load_balancer = None - for cluster in self.clusters: - cluster.n_ns += 1 - self.name = name - - @ovn_stats.timeit - def add_ports(self, ports, az=0): - self.ports[az].extend(ports) - # Always add port IPs to the address set but not to the PGs. - # Simulate what OpenShift does, which is: create the port groups - # when the first network policy is applied. - if self.addr_set4: - for i, nbctl in enumerate(self.nbctl): - nbctl.address_set_add_addrs( - self.addr_set4[i], [str(p.ip) for p in ports] - ) - if self.addr_set6: - for i, nbctl in enumerate(self.nbctl): - nbctl.address_set_add_addrs( - self.addr_set6[i], [str(p.ip6) for p in ports] - ) - if self.enforcing: - self.nbctl[az].port_group_add_ports( - self.pg_def_deny_igr[az], ports - ) - self.nbctl[az].port_group_add_ports( - self.pg_def_deny_egr[az], ports - ) - self.nbctl[az].port_group_add_ports(self.pg[az], ports) - - def unprovision(self): - # ACLs are garbage collected by OVSDB as soon as all the records - # referencing them are removed. - for i, cluster in enumerate(self.clusters): - cluster.unprovision_ports(self.ports[i]) - for i, nbctl in enumerate(self.nbctl): - nbctl.port_group_del(self.pg_def_deny_igr[i]) - nbctl.port_group_del(self.pg_def_deny_egr[i]) - nbctl.port_group_del(self.pg[i]) - if self.addr_set4: - nbctl.address_set_del(self.addr_set4[i]) - if self.addr_set6: - nbctl.address_set_del(self.addr_set6[i]) - nbctl.port_group_del(self.sub_pg[i]) - nbctl.address_set_del(self.sub_as[i]) - - def unprovision_ports(self, ports, az=0): - '''Unprovision a subset of ports in the namespace without having to - unprovision the entire namespace or any of its network policies.''' - - for port in ports: - self.ports[az].remove(port) - - self.clusters[az].unprovision_ports(ports) - - def enforce(self): - if self.enforcing: - return - self.enforcing = True - for i, nbctl in enumerate(self.nbctl): - nbctl.port_group_add_ports(self.pg_def_deny_igr[i], self.ports[i]) - nbctl.port_group_add_ports(self.pg_def_deny_egr[i], self.ports[i]) - nbctl.port_group_add_ports(self.pg[i], self.ports[i]) - - def create_sub_ns(self, ports, global_cfg, az=0): - n_sub_pgs = len(self.sub_pg[az]) - suffix = f'{self.name}_{n_sub_pgs}' - pg = self.nbctl[az].port_group_create(f'sub_pg_{suffix}') - self.nbctl[az].port_group_add_ports(pg, ports) - self.sub_pg[az].append(pg) - for i, nbctl in enumerate(self.nbctl): - if global_cfg.run_ipv4: - addr_set = nbctl.address_set_create(f'sub_as_{suffix}') - nbctl.address_set_add_addrs( - addr_set, [str(p.ip) for p in ports] - ) - self.sub_as[i].append(addr_set) - if global_cfg.run_ipv6: - addr_set = nbctl.address_set_create(f'sub_as_{suffix}6') - nbctl.address_set_add_addrs( - addr_set, [str(p.ip6) for p in ports] - ) - self.sub_as[i].append(addr_set) - return n_sub_pgs - - @ovn_stats.timeit - def default_deny(self, family, az=0): - self.enforce() - - addr_set = f'self.addr_set{family}.name' - self.nbctl[az].acl_add( - self.pg_def_deny_igr[az].name, - 'to-lport', - ACL_DEFAULT_DENY_PRIO, - 'port-group', - f'ip4.src == \\${addr_set} && ' - f'outport == @{self.pg_def_deny_igr[az].name}', - 'drop', - ) - self.nbctl[az].acl_add( - self.pg_def_deny_egr[az].name, - 'to-lport', - ACL_DEFAULT_DENY_PRIO, - 'port-group', - f'ip4.dst == \\${addr_set} && ' - f'inport == @{self.pg_def_deny_egr[az].name}', - 'drop', - ) - self.nbctl[az].acl_add( - self.pg_def_deny_igr[az].name, - 'to-lport', - ACL_DEFAULT_ALLOW_ARP_PRIO, - 'port-group', - f'outport == @{self.pg_def_deny_igr[az].name} && arp', - 'allow', - ) - self.nbctl[az].acl_add( - self.pg_def_deny_egr[az].name, - 'to-lport', - ACL_DEFAULT_ALLOW_ARP_PRIO, - 'port-group', - f'inport == @{self.pg_def_deny_egr[az].name} && arp', - 'allow', - ) - - @ovn_stats.timeit - def allow_within_namespace(self, family, az=0): - self.enforce() - - addr_set = f'self.addr_set{family}.name' - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.src == \\${addr_set} && ' f'outport == @{self.pg[az].name}', - 'allow-related', - ) - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.dst == \\${addr_set} && ' f'inport == @{self.pg[az].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def allow_cross_namespace(self, ns, family): - self.enforce() - - for az, nbctl in enumerate(self.nbctl): - if len(self.ports[az]) == 0: - continue - addr_set = f'self.addr_set{family}.name' - nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.src == \\${addr_set} && ' - f'outport == @{ns.pg[az].name}', - 'allow-related', - ) - ns_addr_set = f'ns.addr_set{family}.name' - nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.dst == \\${ns_addr_set} && ' - f'inport == @{self.pg[az].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def allow_sub_namespace(self, src, dst, family, az=0): - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip{family}.src == \\${self.sub_as[az][src].name} && ' - f'outport == @{self.sub_pg[az][dst].name}', - 'allow-related', - ) - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip{family}.dst == \\${self.sub_as[az][dst].name} && ' - f'inport == @{self.sub_pg[az][src].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def allow_from_external( - self, external_ips, include_ext_gw=False, family=4, az=0 - ): - self.enforce() - # If requested, include the ext-gw of the first port in the namespace - # so we can check that this rule is enforced. - if include_ext_gw: - assert len(self.ports) > 0 - if family == 4 and self.ports[az][0].ext_gw: - external_ips.append(self.ports[az][0].ext_gw) - elif family == 6 and self.ports[az][0].ext_gw6: - external_ips.append(self.ports[az][0].ext_gw6) - ips = [str(ip) for ip in external_ips] - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip.{family} == {{{",".join(ips)}}} && ' - f'outport == @{self.pg[az].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def check_enforcing_internal(self, az=0): - # "Random" check that first pod can reach last pod in the namespace. - if len(self.ports[az]) > 1: - src = self.ports[az][0] - dst = self.ports[az][-1] - worker = src.metadata - if src.ip: - worker.ping_port(self.clusters[az], src, dst.ip) - if src.ip6: - worker.ping_port(self.clusters[az], src, dst.ip6) + def configure(self, physical_net): + raise NotImplementedError @ovn_stats.timeit - def check_enforcing_external(self, az=0): - if len(self.ports[az]) > 0: - dst = self.ports[az][0] - worker = dst.metadata - worker.ping_external(self.clusters[az], dst) + def provision(self, cluster): + raise NotImplementedError @ovn_stats.timeit - def check_enforcing_cross_ns(self, ns, az=0): - if len(self.ports[az]) > 0 and len(ns.ports[az]) > 0: - dst = ns.ports[az][0] - src = self.ports[az][0] - worker = src.metadata - if src.ip and dst.ip: - worker.ping_port(self.clusters[az], src, dst.ip) - if src.ip6 and dst.ip6: - worker.ping_port(self.clusters[az], src, dst.ip6) - - def create_load_balancer(self, az=0): - self.load_balancer = lb.OvnLoadBalancer( - f'lb_{self.name}', self.nbctl[az] - ) + def provision_port(self, cluster, passive=False): + raise NotImplementedError @ovn_stats.timeit - def provision_vips_to_load_balancers(self, backend_lists, version, az=0): - vip_ns_subnet = DEFAULT_NS_VIP_SUBNET - if version == 6: - vip_ns_subnet = DEFAULT_NS_VIP_SUBNET6 - vip_net = vip_ns_subnet.next(self.clusters[az].n_ns) - n_vips = len(self.load_balancer.vips.keys()) - vip_ip = vip_net.ip.__add__(n_vips + 1) - - if version == 6: - vips = { - f'[{vip_ip + i}]:{DEFAULT_VIP_PORT}': [ - f'[{p.ip6}]:{DEFAULT_BACKEND_PORT}' for p in ports - ] - for i, ports in enumerate(backend_lists) - } - self.load_balancer.add_vips(vips) - else: - vips = { - f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ - f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports - ] - for i, ports in enumerate(backend_lists) - } - self.load_balancer.add_vips(vips) + def ping_external(self, cluster, port): + raise NotImplementedError class Cluster: - def __init__(self, central_nodes, relay_nodes, cluster_cfg, brex_cfg, az): + def __init__(self, cluster_cfg, central, brex_cfg, az): # In clustered mode use the first node for provisioning. - self.central_nodes = central_nodes - self.relay_nodes = relay_nodes self.worker_nodes = [] self.cluster_cfg = cluster_cfg self.brex_cfg = brex_cfg - self.nbctl = None - self.sbctl = None - self.icnbctl = None - self.net = cluster_cfg.cluster_net - self.gw_net = ovn_utils.DualStackSubnet.next( - cluster_cfg.gw_net, - az * (cluster_cfg.n_workers // cluster_cfg.n_az), - ) + self.nbctl: Optional[ovn_utils.OvnNbctl] = None + self.sbctl: Optional[ovn_utils.OvnSbctl] = None + self.icnbctl: Optional[ovn_utils.OvnIcNbctl] = None self.az = az - self.router = None - self.load_balancer = None - self.load_balancer6 = None - self.join_switch = None - self.last_selected_worker = 0 - self.n_ns = 0 - self.ts_switch = None + + protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" + db_containers = ( + [ + f'ovn-central-az{self.az+1}-1', + f'ovn-central-az{self.az+1}-2', + f'ovn-central-az{self.az+1}-3', + ] + if cluster_cfg.clustered_db + else [f'ovn-central-az{self.az+1}-1'] + ) + + mgmt_ip = cluster_cfg.node_net.ip + 2 + self.az * len(db_containers) + self.central_nodes = [ + CentralNode(central, c, mgmt_ip + i, protocol) + for i, c in enumerate(db_containers) + ] + + mgmt_ip = ( + cluster_cfg.node_net.ip + + 2 + + cluster_cfg.n_az * len(self.central_nodes) + + self.az * cluster_cfg.n_relays + ) + self.relay_nodes = [ + RelayNode( + central, + f'ovn-relay-az{self.az+1}-{i+1}', + mgmt_ip + i, + protocol, + ) + for i in range(cluster_cfg.n_relays) + ] + + def add_cluster_worker_nodes(self, workers): + raise NotImplementedError def add_workers(self, worker_nodes): self.worker_nodes.extend(worker_nodes) + def prepare_test(self): + self.start() + def start(self): for c in self.central_nodes: c.start( @@ -853,41 +396,6 @@ def get_relay_connection_string(self): ) return self.get_sb_connection_string() - def create_cluster_router(self, rtr_name): - self.router = self.nbctl.lr_add(rtr_name) - self.nbctl.lr_set_options( - self.router, - { - 'always_learn_from_arp_request': 'false', - }, - ) - - def create_cluster_load_balancer(self, lb_name, global_cfg): - if global_cfg.run_ipv4: - self.load_balancer = lb.OvnLoadBalancer( - lb_name, self.nbctl, self.cluster_cfg.vips - ) - self.load_balancer.add_vips(self.cluster_cfg.static_vips) - - if global_cfg.run_ipv6: - self.load_balancer6 = lb.OvnLoadBalancer( - f'{lb_name}6', self.nbctl, self.cluster_cfg.vips6 - ) - self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) - - def create_cluster_join_switch(self, sw_name): - self.join_switch = self.nbctl.ls_add(sw_name, net_s=self.gw_net) - - self.join_rp = self.nbctl.lr_port_add( - self.router, - f'rtr-to-{sw_name}', - RandMac(), - self.gw_net.reverse(), - ) - self.join_ls_rp = self.nbctl.ls_port_add( - self.join_switch, f'{sw_name}-to-rtr', self.join_rp - ) - def provision_ports(self, n_ports, passive=False): return [ self.select_worker_for_port().provision_ports(self, 1, passive)[0] @@ -907,37 +415,25 @@ def ping_ports(self, ports): w.ping_ports(self, ports) @ovn_stats.timeit - def provision_vips_to_load_balancers(self, backend_lists): - n_vips = len(self.load_balancer.vips.keys()) - vip_ip = self.cluster_cfg.vip_subnet.ip.__add__(n_vips + 1) + def mesh_ping_ports(self, ports: List[ovn_utils.LSPort]) -> None: + """Perform full-mesh ping test between ports.""" + all_ips = [port.ip for port in ports] - vips = { - f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ - f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports - ] - for i, ports in enumerate(backend_lists) - } - self.load_balancer.add_vips(vips) - - def unprovision_vips(self): - if self.load_balancer: - self.load_balancer.clear_vips() - self.load_balancer.add_vips(self.cluster_cfg.static_vips) - if self.load_balancer6: - self.load_balancer6.clear_vips() - self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) + for port in ports: + chassis: Optional[ChassisNode] = port.metadata + if chassis is None: + log.error( + f"Port {port.name} is missing 'metadata' attribute. " + f"Can't perform ping." + ) + continue + + for dest_ip in all_ips: + if dest_ip == port.ip: + continue + chassis.ping_port(self, port, dest_ip) def select_worker_for_port(self): self.last_selected_worker += 1 self.last_selected_worker %= len(self.worker_nodes) return self.worker_nodes[self.last_selected_worker] - - def provision_lb_group(self, name='cluster-lb-group'): - self.lb_group = lb.OvnLoadBalancerGroup(name, self.nbctl) - for w in self.worker_nodes: - self.nbctl.ls_add_lbg(w.switch, self.lb_group.lbg) - self.nbctl.lr_add_lbg(w.gw_router, self.lb_group.lbg) - - def provision_lb(self, lb): - log.info(f'Creating load balancer {lb.name}') - self.lb_group.add_lb(lb) diff --git a/test-scenarios/openstack-20-projects-10-vms.yml b/test-scenarios/openstack-20-projects-10-vms.yml new file mode 100644 index 00000000..5c793da9 --- /dev/null +++ b/test-scenarios/openstack-20-projects-10-vms.yml @@ -0,0 +1,13 @@ +global: + log_cmds: false + cms_name: openstack + +cluster: + clustered_db: true + log_txns_db: true + n_workers: 3 + +base_openstack: + n_projects: 20 + n_chassis_per_gw_lrp: 3 + n_vms_per_project: 10 diff --git a/test-scenarios/openstack-low-scale.yml b/test-scenarios/openstack-low-scale.yml new file mode 100644 index 00000000..17e1073d --- /dev/null +++ b/test-scenarios/openstack-low-scale.yml @@ -0,0 +1,13 @@ +global: + log_cmds: false + cms_name: openstack + +cluster: + clustered_db: true + log_txns_db: true + n_workers: 3 + +base_openstack: + n_projects: 2 + n_chassis_per_gw_lrp: 3 + n_vms_per_project: 3