diff --git a/holo-vrrp/src/instance.rs b/holo-vrrp/src/instance.rs index 6635f32b..5e834ecc 100644 --- a/holo-vrrp/src/instance.rs +++ b/holo-vrrp/src/instance.rs @@ -155,7 +155,7 @@ impl Instance { let mut packet = VrrpPacket { version: 2, hdr_type: 1, - vrid: u8::default(), + vrid: self.vrid, priority: self.config.priority, count_ip: self.config.virtual_addresses.len() as u8, auth_type: 0, diff --git a/holo-vrrp/src/interface.rs b/holo-vrrp/src/interface.rs index 4103e8d2..0b9d0061 100644 --- a/holo-vrrp/src/interface.rs +++ b/holo-vrrp/src/interface.rs @@ -21,7 +21,7 @@ use holo_utils::task::Task; use holo_utils::{Receiver, Sender, UnboundedSender}; use ipnetwork::{IpNetwork, Ipv4Network}; use tokio::sync::mpsc; -use tracing::{debug, debug_span}; +use tracing::{debug, debug_span, error_span}; use crate::error::{Error, IoError}; use crate::instance::{Instance, State}; @@ -36,8 +36,6 @@ pub struct Interface { pub name: String, // Interface system data. pub system: InterfaceSys, - // Interface raw sockets and Tx/Rx tasks. - pub net: InterfaceNet, // Interface VRRP instances. pub instances: BTreeMap, // Tx channels. @@ -62,7 +60,7 @@ pub struct MacVlanInterface { // Interface system data. pub system: InterfaceSys, // Interface raw sockets and Tx/Rx tasks. - pub net: Option, + pub net: Option, } #[derive(Debug, Default)] @@ -78,7 +76,7 @@ pub struct InterfaceSys { } #[derive(Debug)] -pub struct InterfaceNet { +pub struct MvlanInterfaceNet { // Raw sockets. pub socket_vrrp: Arc>, pub socket_arp: Arc>, @@ -146,10 +144,26 @@ impl Interface { debug_span!("change-state").in_scope(|| { if state == State::Backup { debug!(%vrid, "state to BACKUP."); - // change admin state of MacVlan to down. + if let Some(ifindex) = instance.mac_vlan.system.ifindex { + for addr in instance.config.virtual_addresses.clone() { + southbound::addr_del( + ifindex, + IpNetwork::V4(addr), + &self.tx.ibus, + ); + } + } } else if state == State::Master { debug!(%vrid, "state to MASTER."); - // change admin state of MacVlan to up. + if let Some(ifindex) = instance.mac_vlan.system.ifindex { + for addr in instance.config.virtual_addresses.clone() { + southbound::addr_add( + ifindex, + IpNetwork::V4(addr), + &self.tx.ibus, + ); + } + } } }); @@ -189,12 +203,18 @@ impl Interface { } pub(crate) fn delete_instance_virtual_address( - &self, + &mut self, vrid: u8, addr: Ipv4Network, ) { - if let Some(instance) = self.instances.get(&vrid) { + if let Some(instance) = self.instances.get_mut(&vrid) { if let Some(ifindex) = instance.mac_vlan.system.ifindex { + // remove address from the instance's configs + instance.config.virtual_addresses.remove(&addr); + + // netlink system call will be initiated to remove the address. + // when response is received, this will also be modified in our + // system's MacVlan southbound::addr_del( ifindex, IpNetwork::V4(addr), @@ -205,7 +225,12 @@ impl Interface { } pub(crate) fn send_vrrp_advert(&self, vrid: u8) { - if let Some(instance) = self.instances.get(&vrid) { + // check for the exists instance... + if let Some(instance) = self.instances.get(&vrid) + + // ...and confirm if the instance's parent Interface has an IP address + && let Some(addr) = self.system.addresses.first() + { let mut buf = BytesMut::new(); // ethernet frame @@ -213,9 +238,7 @@ impl Interface { buf.put(eth_frame); // ip packet - let ip_pkt: &[u8] = &instance - .adver_ipv4_pkt(self.system.addresses.first().unwrap().ip()) - .encode(); + let ip_pkt: &[u8] = &instance.adver_ipv4_pkt(addr.ip()).encode(); buf.put(ip_pkt); // vrrp packet @@ -226,7 +249,24 @@ impl Interface { ifname: instance.mac_vlan.name.clone(), buf: buf.to_vec(), }; - let _ = self.net.net_tx_packetp.send(msg); + if let Some(net) = &instance.mac_vlan.net { + net.net_tx_packetp.send(msg); + } + } else { + error_span!("send-vrrp").in_scope(|| { + tracing::error!(%vrid, "unable to send vrrp advertisement"); + }); + } + } + + // creates the MvlanInterfaceNet for the instance of said + // vrid. Must be done here to get some interface specifics. + pub(crate) fn create_mvlan_net(&mut self, vrid: u8) { + let net = MvlanInterfaceNet::new(self, vrid) + .expect("Failed to intialize VRRP tasks"); + + if let Some(instance) = self.instances.get_mut(&vrid) { + instance.mac_vlan.net = Some(net); } } } @@ -246,12 +286,9 @@ impl ProtocolInstance for Interface { tx: InstanceChannelsTx, ) -> Interface { // TODO: proper error handling - let net = InterfaceNet::new(&name, &tx) - .expect("Failed to initialize VRRP network tasks"); Interface { name, system: Default::default(), - net, instances: Default::default(), tx, shared, @@ -320,25 +357,14 @@ impl MacVlanInterface { net: None, } } - - pub fn create_net(&mut self, tx_channels: &InstanceChannelsTx) { - let net = InterfaceNet::new(&self.name, tx_channels) - .expect("Failed to initialize VRRP tasks"); - self.net = Some(net); - } } -// ===== impl InterfaceNet ===== - -impl InterfaceNet { - fn new( - ifname: &str, - instance_channels_tx: &InstanceChannelsTx, - ) -> Result { - // create socket_vrrp_rx and socket_vrrp_tx - // Currently being created separately so that we can customize - // our vrrp sockets with some virtual fields in the headers - // when sending out the VRRP advertisements +impl MvlanInterfaceNet { + fn new(parent_iface: &Interface, vrid: u8) -> Result { + let instance = parent_iface.instances.get(&vrid).unwrap(); + let ifname = &instance.mac_vlan.name; + let instance_channels_tx = &parent_iface.tx; + let socket_vrrp_rx = network::socket_vrrp_rx(ifname) .map_err(IoError::SocketError) .and_then(|socket| { @@ -346,8 +372,7 @@ impl InterfaceNet { }) .map(Arc::new)?; - // tx will change the creation to happen in it's own function. - let socket_vrrp_tx = network::socket_vrrp_tx(ifname) + let socket_vrrp_tx = network::socket_vrrp_tx(parent_iface, vrid) .map_err(IoError::SocketError) .and_then(|socket| { AsyncFd::new(socket).map_err(IoError::SocketError) @@ -375,7 +400,7 @@ impl InterfaceNet { &instance_channels_tx.protocol_input.vrrp_net_packet_tx, ); - Ok(InterfaceNet { + Ok(Self { socket_vrrp: socket_vrrp_rx, socket_arp, _net_tx_task: net_tx_task, diff --git a/holo-vrrp/src/network.rs b/holo-vrrp/src/network.rs index daefb75e..707fd4db 100644 --- a/holo-vrrp/src/network.rs +++ b/holo-vrrp/src/network.rs @@ -15,22 +15,50 @@ use libc::{if_nametoindex, AF_PACKET, ETH_P_ARP, ETH_P_IP}; use nix::sys::socket; use socket2::{Domain, InterfaceIndexOrAddress, Protocol, Type}; use tokio::sync::mpsc::error::SendError; +use tracing::{debug, debug_span}; use crate::error::IoError; +use crate::interface::Interface; use crate::packet::{ArpPacket, EthernetFrame, Ipv4Packet, VrrpPacket}; use crate::tasks::messages::input::VrrpNetRxPacketMsg; use crate::tasks::messages::output::NetTxPacketMsg; -pub fn socket_vrrp_tx(ifname: &str) -> Result { +pub fn socket_vrrp_tx( + interface: &Interface, + vrid: u8, +) -> Result { #[cfg(not(feature = "testing"))] { + let instance = interface.instances.get(&vrid).unwrap(); + let sock = capabilities::raise(|| { - Socket::new(Domain::PACKET, Type::RAW, Some(Protocol::from(112))) + Socket::new(Domain::IPV4, Type::RAW, Some(Protocol::from(112))) })?; - - capabilities::raise(|| sock.bind_device(Some(ifname.as_bytes())))?; - capabilities::raise(|| sock.set_broadcast(true))?; capabilities::raise(|| sock.set_nonblocking(true))?; + if let Some(addr) = instance.mac_vlan.system.addresses.first() { + capabilities::raise(|| { + match sock.set_multicast_if_v4(&addr.ip()) { + Ok(_res) => { + debug_span!("socket-vrrp").in_scope(|| { + debug!("successfully joined multicast interface"); + }); + } + Err(err) => { + debug_span!("socket-vrrp").in_scope(|| { + debug!(%addr, %err, "unable to join multicast interface"); + }); + } + } + }); + } + + // bind it to the primary interface's name + capabilities::raise(|| { + sock.bind_device(Some(interface.name.as_str().as_bytes())) + })?; + capabilities::raise(|| { + sock.set_reuse_address(true); + }); Ok(sock) } @@ -93,7 +121,7 @@ pub(crate) async fn send_packet_vrrp( unsafe { let ifindex = libc::if_nametoindex(c_ifname.as_ptr()); let mut sa = libc::sockaddr_ll { - sll_family: libc::AF_PACKET as u16, + sll_family: libc::AF_INET as u16, sll_protocol: (ETH_P_IP as u16).to_be(), sll_ifindex: ifindex as i32, sll_hatype: 0, diff --git a/holo-vrrp/src/southbound.rs b/holo-vrrp/src/southbound.rs index 16d164b4..7ba783b0 100644 --- a/holo-vrrp/src/southbound.rs +++ b/holo-vrrp/src/southbound.rs @@ -37,14 +37,15 @@ pub(crate) fn process_iface_update( // update names for all macvlans for (vrid, instance) in iface.instances.iter_mut() { - let name = format!("mvlan-vrrp-{}", vrid); - instance.mac_vlan.name = name; + instance.mac_vlan.name = format!("mvlan-vrrp-{}", vrid); } return; } - // check if it is one of the macvlans being updated. - for (vrid, instance) in iface.instances.iter_mut() { + let mut target_vrid: Option = None; + + //check if it is one of the macvlans being updated. + 'outer: for (vrid, instance) in iface.instances.iter_mut() { let name = format!("mvlan-vrrp-{}", vrid); let mvlan_iface = &mut instance.mac_vlan; @@ -61,36 +62,45 @@ pub(crate) fn process_iface_update( }); mvlan_iface.system.addresses = ips; + target_vrid = Some(*vrid); - mvlan_iface.create_net(&iface.tx); - return; + break 'outer; } } + + if let Some(vrid) = target_vrid { + iface.create_mvlan_net(vrid); + } } pub(crate) fn process_addr_add(iface: &mut Interface, msg: AddressMsg) { if msg.ifname != iface.name { - return; + if let IpNetwork::V4(addr) = msg.addr { + iface.system.addresses.insert(addr); + } } - if let IpNetwork::V4(addr) = msg.addr { - iface.system.addresses.insert(addr); - - // TODO: trigger protocol event? - } + // when this is some, it means that we need to rebind our + // transmission socket multicast address to the newly added address + let mut target_vrid: Option = None; // if the interface being updated is one of the macvlans for (vrid, instance) in iface.instances.iter_mut() { let name = format!("mvlan-vrrp-{}", vrid); let mvlan_iface = &mut instance.mac_vlan; - + if mvlan_iface.system.addresses.len() == 0 { + target_vrid = Some(*vrid); + } if mvlan_iface.name == name { if let IpNetwork::V4(addr) = msg.addr { - instance.config.virtual_addresses.insert(addr); mvlan_iface.system.addresses.insert(addr); } } } + + if let Some(vrid) = target_vrid { + iface.create_mvlan_net(vrid); + } } pub(crate) fn process_addr_del(iface: &mut Interface, msg: AddressMsg) { @@ -98,10 +108,22 @@ pub(crate) fn process_addr_del(iface: &mut Interface, msg: AddressMsg) { return; } + // remove the address from the addresses of parent interfaces if let IpNetwork::V4(addr) = msg.addr { iface.system.addresses.remove(&addr); + } - // TODO: trigger protocol event? + for (vrid, instance) in iface.instances.iter_mut() { + let name = format!("mvlan-vrrp-{}", vrid); + let mvlan_iface = &mut instance.mac_vlan; + + // if it is one of the macvlans being edited, we + // remove the macvlan's + if mvlan_iface.name == name { + if let IpNetwork::V4(addr) = msg.addr { + mvlan_iface.system.addresses.remove(&addr); + } + } } } diff --git a/holo-vrrp/src/tasks.rs b/holo-vrrp/src/tasks.rs index a28c4ae2..79840dc6 100644 --- a/holo-vrrp/src/tasks.rs +++ b/holo-vrrp/src/tasks.rs @@ -10,7 +10,7 @@ use std::time::Duration; use bytes::{BufMut, BytesMut}; use holo_utils::socket::{AsyncFd, Socket}; use holo_utils::task::{IntervalTask, Task, TimeoutTask}; -use holo_utils::{Sender, UnboundedReceiver}; +use holo_utils::{Sender, UnboundedReceiver, UnboundedSender}; use messages::input::MasterDownTimerMsg; use messages::output::NetTxPacketMsg; use tracing::{debug_span, Instrument}; @@ -183,9 +183,6 @@ pub(crate) fn set_timer( vrid: u8, master_down_tx: Sender, ) { - // the net producer used for sending VRRP messages outside. - let net_tx = interface.net.net_tx_packetp.clone(); - if let Some(instance) = interface.instances.get_mut(&vrid) { match instance.state.state { crate::instance::State::Initialize => { @@ -219,23 +216,25 @@ pub(crate) fn set_timer( let ifname = instance.mac_vlan.name.clone(); // ----------------- - - let timer = IntervalTask::new( - Duration::from_secs( - instance.config.advertise_interval as u64, - ), - true, - move || { - let ifname = ifname.clone(); - let buf = buf.to_vec(); - let net_tx = net_tx.clone(); - async move { - let msg = NetTxPacketMsg::Vrrp { ifname, buf }; - let _ = net_tx.send(msg); - } - }, - ); - instance.timer = VrrpTimer::AdverTimer(timer); + if let Some(net) = &instance.mac_vlan.net { + let net_tx = net.net_tx_packetp.clone(); + let timer = IntervalTask::new( + Duration::from_secs( + instance.config.advertise_interval as u64, + ), + true, + move || { + let ifname = ifname.clone(); + let buf = buf.to_vec(); + let net_tx = net_tx.clone(); + async move { + let msg = NetTxPacketMsg::Vrrp { ifname, buf }; + let _ = net_tx.send(msg); + } + }, + ); + instance.timer = VrrpTimer::AdverTimer(timer); + } } } }