diff --git a/app/gimlet/base.toml b/app/gimlet/base.toml index 39bcc2900..704ddbd1d 100644 --- a/app/gimlet/base.toml +++ b/app/gimlet/base.toml @@ -125,7 +125,7 @@ name = "task-power" features = ["gimlet"] priority = 6 max-sizes = {flash = 65536, ram = 8192 } -stacksize = 1504 +stacksize = 2504 start = true task-slots = ["i2c_driver", "sensor", "gimlet_seq"] notifications = ["timer"] diff --git a/drv/i2c-devices/src/max5970.rs b/drv/i2c-devices/src/max5970.rs index 7f480e181..2812f6f44 100644 --- a/drv/i2c-devices/src/max5970.rs +++ b/drv/i2c-devices/src/max5970.rs @@ -289,6 +289,10 @@ impl Max5970 { self.device.read_reg::(reg as u8) } + fn write_reg(&self, reg: Register, value: u8) -> Result<(), ResponseCode> { + self.device.write(&[reg as u8, value]) + } + pub fn i2c_device(&self) -> &I2cDevice { &self.device } @@ -327,6 +331,18 @@ impl Max5970 { Ok(Amperes(delta / self.rsense as f32)) } + fn peak_vout( + &self, + msb_reg: Register, + lsb_reg: Register, + ) -> Result { + Ok(self.convert_volts( + MonRange(self.read_reg(Register::mon_range)?), + self.read_reg(msb_reg)?, + self.read_reg(lsb_reg)?, + )) + } + pub fn max_vout(&self) -> Result { let (msb_reg, lsb_reg) = if self.rail == 0 { (Register::max_chx_mon_msb_ch1, Register::max_chx_mon_lsb_ch1) @@ -334,20 +350,24 @@ impl Max5970 { (Register::max_chx_mon_msb_ch2, Register::max_chx_mon_lsb_ch2) }; - Ok(self.convert_volts( - MonRange(self.read_reg(Register::mon_range)?), - self.read_reg(msb_reg)?, - self.read_reg(lsb_reg)?, - )) + self.peak_vout(msb_reg, lsb_reg) } - pub fn max_iout(&self) -> Result { + pub fn min_vout(&self) -> Result { let (msb_reg, lsb_reg) = if self.rail == 0 { - (Register::max_chx_cs_msb_ch1, Register::max_chx_cs_lsb_ch1) + (Register::min_chx_mon_msb_ch1, Register::min_chx_mon_lsb_ch1) } else { - (Register::max_chx_cs_msb_ch2, Register::max_chx_cs_lsb_ch2) + (Register::min_chx_mon_msb_ch2, Register::min_chx_mon_lsb_ch2) }; + self.peak_vout(msb_reg, lsb_reg) + } + + fn peak_iout( + &self, + msb_reg: Register, + lsb_reg: Register, + ) -> Result { self.convert_current( Status2(self.read_reg(Register::status2)?), self.read_reg(msb_reg)?, @@ -355,9 +375,36 @@ impl Max5970 { ) } + pub fn max_iout(&self) -> Result { + let (msb_reg, lsb_reg) = if self.rail == 0 { + (Register::max_chx_cs_msb_ch1, Register::max_chx_cs_lsb_ch1) + } else { + (Register::max_chx_cs_msb_ch2, Register::max_chx_cs_lsb_ch2) + }; + + self.peak_iout(msb_reg, lsb_reg) + } + + pub fn min_iout(&self) -> Result { + let (msb_reg, lsb_reg) = if self.rail == 0 { + (Register::min_chx_cs_msb_ch1, Register::min_chx_cs_lsb_ch1) + } else { + (Register::min_chx_cs_msb_ch2, Register::min_chx_cs_lsb_ch2) + }; + + self.peak_iout(msb_reg, lsb_reg) + } + pub fn status0(&self) -> Result { self.read_reg(Register::status0) } + + pub fn clear_peaks(&self) -> Result<(), ResponseCode> { + let rst = if self.rail == 0 { 0b00_11 } else { 0b11_00 }; + + self.write_reg(Register::peak_log_rst, rst)?; + self.write_reg(Register::peak_log_rst, 0) + } } impl Validate for Max5970 { diff --git a/idl/power.idol b/idl/power.idol index 503caa56b..7cec3e6cf 100644 --- a/idl/power.idol +++ b/idl/power.idol @@ -46,7 +46,7 @@ Interface( err: CLike("ResponseCode"), ), idempotent: true, - ), + ), "bmr491_event_log_read": ( doc: "reads an event from the BMR491's combined fault and lifecycle event log", args: { @@ -58,6 +58,14 @@ Interface( ), idempotent: true, ), + "bmr491_fault_log_clear": ( + doc: "clears the BMR491's persistent fault log", + reply: Result( + ok: "()", + err: CLike("ResponseCode"), + ), + idempotent: true, + ), "bmr491_max_fault_event_index": ( doc: "returns the index of the most recent fault event in the BMR491's event log", reply: Result( diff --git a/task/power/src/bsp/gimlet_bcde.rs b/task/power/src/bsp/gimlet_bcde.rs index a7b82230c..925f0d493 100644 --- a/task/power/src/bsp/gimlet_bcde.rs +++ b/task/power/src/bsp/gimlet_bcde.rs @@ -3,11 +3,17 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::{ - i2c_config, i2c_config::sensors, DeviceType, Ohms, PowerControllerConfig, - PowerState, + i2c_config, i2c_config::sensors, Device, DeviceType, Ohms, + PowerControllerConfig, PowerState, SensorId, }; +use drv_i2c_devices::max5970::*; +use ringbuf::*; +use userlib::units::*; + pub(crate) const CONTROLLER_CONFIG_LEN: usize = 37; +const MAX5970_CONFIG_LEN: usize = 22; + pub(crate) static CONTROLLER_CONFIG: [PowerControllerConfig; CONTROLLER_CONFIG_LEN] = [ rail_controller!(IBC, bmr491, v12_sys_a2, A2), @@ -71,8 +77,178 @@ pub(crate) fn get_state() -> PowerState { } } -pub fn preinit() { - // Nothing to do here +#[derive(Copy, Clone, PartialEq)] +enum Trace { + Now(u32), + Max5970 { + sensor: SensorId, + last_bounce_detected: Option, + status0: u8, + status1: u8, + status3: u8, + fault0: u8, + fault1: u8, + fault2: u8, + min_iout: f32, + max_iout: f32, + min_vout: f32, + max_vout: f32, + }, + None, +} + +ringbuf!(Trace, 64, Trace::None); + +fn trace_max5970( + dev: &Max5970, + sensor: SensorId, + peaks: &mut Max5970Peaks, + now: u32, +) { + let max_vout = match dev.max_vout() { + Ok(Volts(v)) => v, + _ => return, + }; + + let min_vout = match dev.min_vout() { + Ok(Volts(v)) => v, + _ => return, + }; + + let max_iout = match dev.max_iout() { + Ok(Amperes(a)) => a, + _ => return, + }; + + let min_iout = match dev.min_iout() { + Ok(Amperes(a)) => a, + _ => return, + }; + + if peaks.iout.bounced(min_iout, max_iout) + || peaks.vout.bounced(min_vout, max_vout) + { + peaks.last_bounce_detected = Some(now); + } + + ringbuf_entry!(Trace::Max5970 { + sensor, + last_bounce_detected: peaks.last_bounce_detected, + status0: match dev.read_reg(Register::status0) { + Ok(reg) => reg, + _ => return, + }, + status1: match dev.read_reg(Register::status1) { + Ok(reg) => reg, + _ => return, + }, + status3: match dev.read_reg(Register::status3) { + Ok(reg) => reg, + _ => return, + }, + fault0: match dev.read_reg(Register::fault0) { + Ok(reg) => reg, + _ => return, + }, + fault1: match dev.read_reg(Register::fault1) { + Ok(reg) => reg, + _ => return, + }, + fault2: match dev.read_reg(Register::fault2) { + Ok(reg) => reg, + _ => return, + }, + min_iout, + max_iout, + min_vout, + max_vout, + }); +} + +#[derive(Copy, Clone)] +struct Max5970Peak { + min: f32, + max: f32, +} + +impl Default for Max5970Peak { + fn default() -> Self { + Self { + min: f32::MAX, + max: f32::MIN, + } + } +} + +impl Max5970Peak { + /// + /// If we see the drives lose power, it is helpful to disambiguate PDN issues + /// from the power being explicitly disabled via system software (e.g., via + /// CEM_TO_PCIEHP_PWREN on Sharkfin). The MAX5970 doesn't have a way of + /// recording power cycles, but we know that if we see the peaks travel in + /// the wrong direction (that is, a max that is less than the previous max + /// or a minimum that is greater than our previous minimum) then there must + /// have been a power cycle. This can clearly yield false negatives, but + /// it will not yield false positives: if [`bounced`] returns true, one can + /// know with confidence that the power has been cycled. + /// + fn bounced(&mut self, min: f32, max: f32) -> bool { + let bounced = min > self.min || max < self.max; + self.min = min; + self.max = max; + bounced + } +} + +#[derive(Copy, Clone, Default)] +struct Max5970Peaks { + iout: Max5970Peak, + vout: Max5970Peak, + last_bounce_detected: Option, +} + +pub(crate) struct State { + fired: u32, + peaks: [Max5970Peaks; MAX5970_CONFIG_LEN], +} + +impl State { + pub(crate) fn init() -> Self { + Self { + fired: 0, + peaks: [Max5970Peaks::default(); MAX5970_CONFIG_LEN], + } + } + + pub(crate) fn handle_timer_fired( + &mut self, + devices: &[Device], + state: PowerState, + ) { + // + // Trace the detailed state every ten seconds, provided that we are in A0. + // + if state == PowerState::A0 && self.fired % 10 == 0 { + ringbuf_entry!(Trace::Now(self.fired)); + + for ((dev, sensor), peak) in CONTROLLER_CONFIG + .iter() + .zip(devices.iter()) + .filter_map(|(c, dev)| { + if let Device::Max5970(dev) = dev { + Some((dev, c.current)) + } else { + None + } + }) + .zip(self.peaks.iter_mut()) + { + trace_max5970(dev, sensor, peak, self.fired); + } + } + + self.fired += 1; + } } pub const HAS_RENDMP_BLACKBOX: bool = true; diff --git a/task/power/src/bsp/gimletlet_2.rs b/task/power/src/bsp/gimletlet_2.rs index d9510432f..c81309c90 100644 --- a/task/power/src/bsp/gimletlet_2.rs +++ b/task/power/src/bsp/gimletlet_2.rs @@ -19,8 +19,19 @@ pub(crate) fn get_state() -> PowerState { PowerState::A2 } -pub fn preinit() { - // Nothing to do here +pub(crate) struct State(()); + +impl State { + pub(crate) fn init() -> Self { + State(()) + } + + pub(crate) fn handle_timer_fired( + &self, + _devices: &[crate::Device], + _state: PowerState, + ) { + } } pub const HAS_RENDMP_BLACKBOX: bool = false; diff --git a/task/power/src/bsp/psc_abc.rs b/task/power/src/bsp/psc_abc.rs index 3ddf45b7e..b5c5a594c 100644 --- a/task/power/src/bsp/psc_abc.rs +++ b/task/power/src/bsp/psc_abc.rs @@ -28,22 +28,35 @@ pub(crate) fn get_state() -> PowerState { PowerState::A2 } -pub fn preinit() { - // Before talking to the power shelves, we have to enable an I2C buffer - userlib::task_slot!(SYS, sys); - use drv_stm32xx_sys_api::*; - - let sys_task = SYS.get_task_id(); - let sys = Sys::from(sys_task); - - let i2c_en = Port::E.pin(15); // SP_TO_BP_I2C_EN - sys.gpio_set(i2c_en); - sys.gpio_configure_output( - i2c_en, - OutputType::PushPull, - Speed::Low, - Pull::None, - ); +pub(crate) struct State(()); + +impl State { + pub(crate) fn init() -> Self { + // Before talking to the power shelves, we have to enable an I2C buffer + userlib::task_slot!(SYS, sys); + use drv_stm32xx_sys_api::*; + + let sys_task = SYS.get_task_id(); + let sys = Sys::from(sys_task); + + let i2c_en = Port::E.pin(15); // SP_TO_BP_I2C_EN + sys.gpio_set(i2c_en); + sys.gpio_configure_output( + i2c_en, + OutputType::PushPull, + Speed::Low, + Pull::None, + ); + + State(()) + } + + pub(crate) fn handle_timer_fired( + &self, + _devices: &[crate::Device], + _state: PowerState, + ) { + } } pub const HAS_RENDMP_BLACKBOX: bool = false; diff --git a/task/power/src/bsp/sidecar_bc.rs b/task/power/src/bsp/sidecar_bc.rs index a27029808..cfdc4c7ea 100644 --- a/task/power/src/bsp/sidecar_bc.rs +++ b/task/power/src/bsp/sidecar_bc.rs @@ -44,8 +44,19 @@ pub(crate) fn get_state() -> PowerState { } } -pub fn preinit() { - // Nothing to do here +pub(crate) struct State(()); + +impl State { + pub(crate) fn init() -> Self { + State(()) + } + + pub(crate) fn handle_timer_fired( + &self, + _devices: &[crate::Device], + _state: PowerState, + ) { + } } pub const HAS_RENDMP_BLACKBOX: bool = true; diff --git a/task/power/src/main.rs b/task/power/src/main.rs index 3b4aef316..c8641e22a 100644 --- a/task/power/src/main.rs +++ b/task/power/src/main.rs @@ -38,21 +38,10 @@ use drv_i2c_devices::{ enum Trace { GotVersion(u32), GotAddr(u32), - Max5970 { - sensor: SensorId, - status0: u8, - status1: u8, - status3: u8, - fault0: u8, - fault1: u8, - fault2: u8, - max_current: f32, - max_voltage: f32, - }, None, } -ringbuf!(Trace, 16, Trace::None); +ringbuf!(Trace, 2, Trace::None); use sensor_api::{NoData, SensorId}; @@ -360,54 +349,6 @@ macro_rules! max5970_controller { }; } -fn trace_max5970(dev: &Max5970, sensor: SensorId) { - if let Ok(Volts(volts)) = dev.max_vout() { - use drv_i2c_devices::max5970::Register; - - // - // We want to *not* trace the 3.3V rails on the MAX5970 on the - // Sharkfin (U8), so anything that has either never powered on or - // never seen a voltage greater than ~4V we will not record. - // - if volts < 4.0 { - return; - } - - ringbuf_entry!(Trace::Max5970 { - sensor, - status0: match dev.read_reg(Register::status0) { - Ok(reg) => reg, - _ => return, - }, - status1: match dev.read_reg(Register::status1) { - Ok(reg) => reg, - _ => return, - }, - status3: match dev.read_reg(Register::status3) { - Ok(reg) => reg, - _ => return, - }, - fault0: match dev.read_reg(Register::fault0) { - Ok(reg) => reg, - _ => return, - }, - fault1: match dev.read_reg(Register::fault1) { - Ok(reg) => reg, - _ => return, - }, - fault2: match dev.read_reg(Register::fault2) { - Ok(reg) => reg, - _ => return, - }, - max_current: match dev.max_iout() { - Ok(Amperes(amps)) => amps, - _ => return, - }, - max_voltage: volts, - }); - } -} - #[allow(unused_macros)] macro_rules! mwocp68_controller { ($which:ident, $rail:ident, $state:ident) => { @@ -463,15 +404,13 @@ mod bsp; #[export_name = "main"] fn main() -> ! { - bsp::preinit(); - let i2c_task = I2C.get_task_id(); let mut server = ServerImpl { i2c_task, sensor: sensor_api::Sensor::from(SENSOR.get_task_id()), devices: claim_devices(i2c_task), - fired: 0, + bsp: bsp::State::init(), }; let mut buffer = [0; idl::INCOMING_SIZE]; @@ -488,7 +427,7 @@ struct ServerImpl { i2c_task: TaskId, sensor: sensor_api::Sensor, devices: &'static mut [Device; bsp::CONTROLLER_CONFIG_LEN], - fired: u32, + bsp: bsp::State, } impl ServerImpl { @@ -561,18 +500,9 @@ impl ServerImpl { } } } - - // - // Every 10 seconds, gather max5970 data - // - if self.fired % 10 == 0 { - if let Device::Max5970(dev) = dev { - trace_max5970(dev, c.current); - } - } } - self.fired += 1; + self.bsp.handle_timer_fired(self.devices, state); } /// Find the BMR491 and return an `I2cDevice` handle @@ -909,6 +839,21 @@ impl idl::InOrderPowerImpl for ServerImpl { Ok(out) } + fn bmr491_fault_log_clear( + &mut self, + _msg: &userlib::RecvMessage, + ) -> Result<(), idol_runtime::RequestError> { + let dev = self.bmr491()?; + // Writing the special value 0xAA to the event index register + // results in the fault log being cleared. + dev.write(&[ + pmbus::commands::bmr491::CommandCode::MFR_EVENT_INDEX as u8, + 0xaa, + ])?; + + Ok(()) + } + fn bmr491_max_fault_event_index( &mut self, _msg: &userlib::RecvMessage,