From 4eaa912a9cacb13c77122b64158ee9e4befa8d0d Mon Sep 17 00:00:00 2001 From: tfx2001 Date: Thu, 15 Aug 2024 00:45:23 +0800 Subject: [PATCH 1/3] feat(trap): support atomic instructions emulation Since HPM6360 dose not support execute atomic load/store with SDRAM address range, we need to emulation atomic instructions execute in trap handler then the Linux kernel can runs without any modifications and user space thread also is supported. For AMO instructions, they will trap into Store/AMO access fault exception during execution, so we can emulate it in store fault exception handler. For lr/sc instructions, lr will trap into Load access fault during execution but sc won't, however. So we need to replace sc into an illegal instruction which I use csrrw zero, time, zero here, then it will trap into Illegal instruction exception and will be replaced back during exception handling. After replacing instructions we have to execute fence.i to make sure this data store is visible to the subsequent instruction fetch. Signed-off-by: tfx2001 --- Cargo.lock | 9 +- Cargo.toml | 2 +- src/main.rs | 2 + src/riscv_spec.rs | 9 +- src/trap.rs | 265 +++++++++++++++++++++++++++++++++++++++------- 5 files changed, 238 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb6c53f..6965161 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,8 +141,7 @@ dependencies = [ [[package]] name = "riscv-decode" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec7a6dc0b0bb96a4d23271864a45c0d24dcd9dde2a1b630a35f79fa29c588bf" +source = "git+https://github.com/fintelia/riscv-decode.git?rev=349b2a6b9fa608fc427aa46eaef8935557510d28#349b2a6b9fa608fc427aa46eaef8935557510d28" [[package]] name = "riscv-rt-macros" @@ -187,7 +186,7 @@ checksum = "a71347da9582cc6b6f3652c7d2c06516c9555690b3738ecdff7e84297f4e17fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.74", ] [[package]] @@ -224,9 +223,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.71" +version = "2.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 8c910a6..75047be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ hpm-rt = { git = "https://github.com/hpm-rs/hpm-rt.git", rev = "66ffb7d7a65d7251 riscv = "0.10" spin = "0.9" fast-trap = { version = "0.0.1", features = ["riscv-m"] } -riscv-decode = "0.2.1" +riscv-decode = { git = "https://github.com/fintelia/riscv-decode.git", rev = "349b2a6b9fa608fc427aa46eaef8935557510d28" } [build-dependencies] hpm-rt = { git = "https://github.com/hpm-rs/hpm-rt.git", rev = "66ffb7d7a65d7251d0b47db9599d36cefc4d6703" } diff --git a/src/main.rs b/src/main.rs index 76e4dc5..a66a4e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -84,6 +84,8 @@ fn main() -> ! { medeleg::clear_supervisor_env_call(); medeleg::clear_illegal_instruction(); medeleg::clear_machine_env_call(); + medeleg::clear_store_fault(); + medeleg::clear_load_fault(); mtvec::write(fast_trap::trap_entry as _, mtvec::TrapMode::Direct); asm!("j {trap_handler}", trap_handler = sym fast_trap::trap_entry, diff --git a/src/riscv_spec.rs b/src/riscv_spec.rs index 39c7c69..f584663 100644 --- a/src/riscv_spec.rs +++ b/src/riscv_spec.rs @@ -1,7 +1,7 @@ #![allow(unused, missing_docs)] -pub const CSR_TIME: u32 = 0xc01; -pub const CSR_TIMEH: u32 = 0xc81; +pub const CSR_TIME: usize = 0xc01; +pub const CSR_TIMEH: usize = 0xc81; pub mod mie { use core::arch::asm; @@ -102,3 +102,8 @@ pub mod mdcause { bits } } + +#[inline(always)] +pub unsafe fn fence_i() { + core::arch::asm!("fence.i"); +} diff --git a/src/trap.rs b/src/trap.rs index 68db243..dcc270d 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -1,15 +1,33 @@ -use fast_trap::{FastContext, FastResult}; +use fast_trap::{EntireContext, EntireContextSeparated, EntireResult, FastContext, FastResult}; use riscv::register::{ mcause::{self, Exception as E, Interrupt as I, Trap as T}, mip, mtval, scause, sepc, sstatus, stval, stvec, }; +use riscv_decode::{decode, Instruction}; use rustsbi::RustSBI; -use crate::board; use crate::extension::SBI; use crate::local_hsm; -use crate::print; use crate::riscv_spec::*; +use crate::{board, print}; + +static mut S_LR_ADDR: usize = 0; +/// `csrrw zero, time, zero` +const BKPT_INST: usize = 0xc0101073; +static mut BKPT_INST_ADDR: usize = 0; +static mut BKPT_RESERVED_INST: usize = 0; + +macro_rules! amo { + ($ctx:expr, $inst:ident, $operation:expr) => {{ + let tmp = read_register($ctx, $inst.rs1()); + let a = *(tmp as *const _); + let b = read_register($ctx, $inst.rs2()); + if ($inst.rd() != 0) { + write_register($ctx, $inst.rd(), a); + } + *(tmp as *mut _) = $operation(a, b); + }}; +} #[inline] fn boot(mut ctx: FastContext, start_addr: usize, opaque: usize) -> FastResult { @@ -24,54 +42,211 @@ fn boot(mut ctx: FastContext, start_addr: usize, opaque: usize) -> FastResult { } #[inline] -fn delegate() { - unsafe { - sepc::write(mepc::read()); - scause::write(mcause::read().bits()); - stval::write(mtval::read()); - sstatus::clear_sie(); - if mstatus::read() & mstatus::MPP == mstatus::MPP_SUPERVISOR { - sstatus::set_spp(sstatus::SPP::Supervisor); - } else { - sstatus::set_spp(sstatus::SPP::User); - } - mstatus::update(|bits| { - *bits &= !mstatus::MPP; - *bits |= mstatus::MPP_SUPERVISOR; - }); - mepc::write(stvec::read().address()); +fn check_trap_privilege_mode() { + if mstatus::read() & mstatus::MPP == mstatus::MPP_MACHINE { + panic!("{:?} from M-MODE", mcause::read().cause()); } } #[inline] -fn illegal_instruction_handler(ctx: &mut FastContext) -> bool { - use riscv_decode::{decode, Instruction}; +unsafe fn delegate() { + sepc::write(mepc::read()); + scause::write(mcause::read().bits()); + stval::write(mtval::read()); + sstatus::clear_sie(); + if mstatus::read() & mstatus::MPP == mstatus::MPP_SUPERVISOR { + sstatus::set_spp(sstatus::SPP::Supervisor); + } else { + sstatus::set_spp(sstatus::SPP::User); + } + mstatus::update(|bits| { + *bits &= !mstatus::MPP; + *bits |= mstatus::MPP_SUPERVISOR; + }); + mepc::write(stvec::read().address()); +} +#[inline] +fn illegal_instruction_handler(mut ctx: FastContext) -> Result { let inst = decode(mtval::read() as u32); match inst { - Ok(Instruction::Csrrs(csr)) => match csr.csr() { + Ok(Instruction::Csrrs(csr)) => match csr.csr() as usize { CSR_TIME => { - assert!( - 10 <= csr.rd() && csr.rd() <= 17, - "Unsupported CSR rd: {}", - csr.rd() - ); ctx.regs().a[(csr.rd() - 10) as usize] = SBI.timer.time() as usize; } CSR_TIMEH => { - assert!( - 10 <= csr.rd() && csr.rd() <= 17, - "Unsupported CSR rd: {}", - csr.rd() - ); ctx.regs().a[(csr.rd() - 10) as usize] = SBI.timer.timeh() as usize; } - _ => return false, + _ => return Err(ctx), + }, + Ok(Instruction::Csrrw(csr)) => unsafe { + if csr.csr() as usize == CSR_TIME && mepc::read() == BKPT_INST_ADDR { + clear_breakpoint(); + return Ok(ctx.continue_with(atomic_emulation_wrapper, ())); + } else { + return Err(ctx); + } + }, + _ => return Err(ctx), + } + mepc::next(); + Ok(ctx.restore()) +} + +unsafe fn find_next_sc(addr: usize) -> Result { + let mut addr = addr; + for _ in 0..16 { + let inst = (addr as *const u32).read(); + if let Ok(Instruction::ScW(_)) = decode(inst) { + return Ok(addr); + } else if (inst & 0xFF) != 0b11 { + // RVC instruction + addr += 2; + } else { + addr += 4; + } + } + Err(()) +} + +unsafe fn set_breakpoint(addr: usize) { + let addr = addr as *mut usize; + BKPT_RESERVED_INST = addr.read(); + BKPT_INST_ADDR = addr as usize; + *addr = BKPT_INST; + fence_i(); +} + +unsafe fn clear_breakpoint() { + if BKPT_INST_ADDR != 0 { + let addr = BKPT_INST_ADDR as *mut usize; + *addr = BKPT_RESERVED_INST; + BKPT_INST_ADDR = 0; + fence_i(); + } +} + +unsafe fn write_register(ctx: &mut EntireContextSeparated, r: u32, value: usize) { + let r = r as usize; + match r { + // x0 + 0 => {} + // gp + 3 => core::arch::asm!("c.mv gp, {}", in(reg) value), + // tp + 4 => core::arch::asm!("c.mv tp, {}", in(reg) value), + 5..=7 => ctx.regs().t[r - 5] = value, + 8..=9 => { + ctx.regs().s[r - 8] = value; + } + 10..=17 => { + ctx.regs().a[r - 10] = value; + } + 18..=27 => { + ctx.regs().s[r - 16] = value; + } + 28..=31 => { + ctx.regs().t[r - 25] = value; + } + _ => panic!("invalid register number: {}", r), + } +} + +fn read_register(ctx: &mut EntireContextSeparated, r: u32) -> usize { + let r = r as usize; + match r { + // x0 + 0 => 0, + // gp + 3 => unsafe { + let value: usize; + core::arch::asm!("c.mv {}, gp", out(reg) value); + value }, - _ => return false, + 4 => unsafe { + let value: usize; + core::arch::asm!("c.mv {}, tp", out(reg) value); + value + }, + 5..=7 => ctx.regs().t[r - 5], + 8..=9 => ctx.regs().s[r - 8], + 10..=17 => ctx.regs().a[r - 10], + 18..=27 => ctx.regs().s[r - 16], + 28..=31 => ctx.regs().t[r - 25], + _ => panic!("invalid register number: {}", r), + } +} + +unsafe fn atomic_emulation(mut ctx: EntireContextSeparated) -> EntireResult { + let inst = (mepc::read() as *const u32).read_unaligned(); + let decoded_inst = decode(inst); + match decoded_inst { + Ok(Instruction::LrW(lr)) => { + let rs1 = lr.rs1(); + let rd = lr.rd(); + S_LR_ADDR = read_register(&mut ctx, rs1); + let tmp: usize = *(S_LR_ADDR as *const _); + write_register(&mut ctx, rd, tmp); + + // Clear old breakpoint and set a new one + clear_breakpoint(); + let sc_inst_addr = find_next_sc(mepc::read()).unwrap_or_else(|_| { + panic!("[rustsbi] unable to find matching sc instruction"); + }); + set_breakpoint(sc_inst_addr); + } + Ok(Instruction::ScW(sc)) => { + let rs1 = sc.rs1(); + let rs2 = sc.rs2(); + let rd = sc.rd(); + let tmp: usize = read_register(&mut ctx, rs1); + if tmp != S_LR_ADDR { + write_register(&mut ctx, rd, 1); + } else { + *(S_LR_ADDR as *mut _) = read_register(&mut ctx, rs2); + write_register(&mut ctx, rd, 0); + S_LR_ADDR = 0; + } + } + Ok(Instruction::AmoswapW(amo)) => { + amo!(&mut ctx, amo, |_, b| b); + } + Ok(Instruction::AmoaddW(amo)) => { + amo!(&mut ctx, amo, |a, b| a + b); + } + Ok(Instruction::AmoxorW(amo)) => { + amo!(&mut ctx, amo, |a, b| a ^ b); + } + Ok(Instruction::AmoandW(amo)) => { + amo!(&mut ctx, amo, |a, b| a & b); + } + Ok(Instruction::AmoorW(amo)) => { + amo!(&mut ctx, amo, |a, b| a | b); + } + Ok(Instruction::AmominW(amo)) => { + amo!(&mut ctx, amo, |a, b| (a as isize).min(b as isize)); + } + Ok(Instruction::AmomaxW(amo)) => { + amo!(&mut ctx, amo, |a, b| (a as isize).max(b as isize)); + } + Ok(Instruction::AmominuW(amo)) => { + amo!(&mut ctx, amo, |a: usize, b| a.min(b)); + } + Ok(Instruction::AmomaxuW(amo)) => { + amo!(&mut ctx, amo, |a: usize, b| a.max(b)); + } + _ => { + delegate(); + return ctx.restore(); + } } mepc::next(); - true + ctx.restore() +} + +extern "C" fn atomic_emulation_wrapper(ctx: EntireContext) -> EntireResult { + let (ctx, _) = ctx.split(); + unsafe { atomic_emulation(ctx) } } #[no_mangle] @@ -145,14 +320,22 @@ pub extern "C" fn fast_handler( break ctx.restore(); } T::Exception(E::IllegalInstruction) => { - if mstatus::read() & mstatus::MPP == mstatus::MPP_MACHINE { - panic!("Illegal instruction exception from M-MODE"); - } + check_trap_privilege_mode(); ctx.regs().a = [ctx.a0(), a1, a2, a3, a4, a5, a6, a7]; - if !illegal_instruction_handler(&mut ctx) { + break illegal_instruction_handler(ctx).unwrap_or_else(|ctx| unsafe { delegate(); - } - break ctx.restore(); + ctx.restore() + }); + } + T::Exception(E::LoadFault) => { + check_trap_privilege_mode(); + ctx.regs().a = [ctx.a0(), a1, a2, a3, a4, a5, a6, a7]; + break ctx.continue_with(atomic_emulation_wrapper, ()); + } + T::Exception(E::StoreFault) => { + check_trap_privilege_mode(); + ctx.regs().a = [ctx.a0(), a1, a2, a3, a4, a5, a6, a7]; + break ctx.continue_with(atomic_emulation_wrapper, ()); } T::Interrupt(I::MachineTimer) => { ctx.regs().a = [ctx.a0(), a1, a2, a3, a4, a5, a6, a7]; From 69deaa3b50422c75eb6f61c10b2c50660a274e63 Mon Sep 17 00:00:00 2001 From: tfx2001 Date: Thu, 15 Aug 2024 01:27:05 +0800 Subject: [PATCH 2/3] chore(loader): rename load_test_kernel to load_kernel Signed-off-by: tfx2001 --- src/loader.rs | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loader.rs b/src/loader.rs index b226911..38ef8c4 100644 --- a/src/loader.rs +++ b/src/loader.rs @@ -47,7 +47,7 @@ impl BlobInfo { } } -pub unsafe fn load_test_kernel() { +pub unsafe fn load_kernel() { let info: &BlobInfo = &BLOB_TABLE[0]; assert!(info.type_ == BlobType::Kernel); assert!(info.start + info.length <= BLOB_TABLE[1].start); diff --git a/src/main.rs b/src/main.rs index a66a4e8..7ab25f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,7 +65,7 @@ fn main() -> ! { trap_stack::prepare_for_trap(); unsafe { // 加载内核 - loader::load_test_kernel(); + loader::load_kernel(); // 加载设备树 loader::load_dtb() }; From d4f04d9094ce90605acee22a408dea16141b0979 Mon Sep 17 00:00:00 2001 From: tfx2001 Date: Thu, 15 Aug 2024 01:30:49 +0800 Subject: [PATCH 3/3] chore(pmp): move set_pmp to pmp module Signed-off-by: tfx2001 --- src/main.rs | 14 +------------- src/pmp.rs | 13 +++++++++++-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index 7ab25f8..c9c681e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,7 +58,7 @@ fn main() -> ! { firmware_address = _start as usize, ); // 初始化 PMP - set_pmp(); + pmp::set_pmp(); // 显示 PMP 配置 pmp::print_pmps(); // 设置陷入栈 @@ -94,18 +94,6 @@ fn main() -> ! { } } -/// 设置 PMP。 -fn set_pmp() { - use riscv::register::*; - unsafe { - // 1. SDRAM - pmpcfg0::set_pmp(0, Range::OFF, Permission::NONE, false); - pmpaddr0::write((0x4000_0000) >> 2); - pmpcfg0::set_pmp(1, Range::TOR, Permission::RWX, false); - pmpaddr1::write(usize::MAX >> 2); - } -} - #[panic_handler] fn panic(info: &core::panic::PanicInfo) -> ! { println!( diff --git a/src/pmp.rs b/src/pmp.rs index 52cf546..0ad4bdb 100644 --- a/src/pmp.rs +++ b/src/pmp.rs @@ -1,4 +1,15 @@ use crate::println; +use riscv::register::*; + +pub fn set_pmp() { + unsafe { + // 1. SDRAM + pmpcfg0::set_pmp(0, Range::OFF, Permission::NONE, false); + pmpaddr0::write((0x4000_0000) >> 2); + pmpcfg0::set_pmp(1, Range::TOR, Permission::RWX, false); + pmpaddr1::write(usize::MAX >> 2); + } +} pub(crate) fn print_pmps() { const ITEM_PER_CFG: usize = core::mem::size_of::(); @@ -47,7 +58,6 @@ fn dump_pmp(i: usize, s: usize, e: usize, cfg: usize) { } fn pmpcfg(i: usize) -> usize { - use riscv::register::*; match i { 0 => pmpcfg0::read().bits, #[cfg(target_arch = "riscv32")] @@ -60,7 +70,6 @@ fn pmpcfg(i: usize) -> usize { } fn pmpaddr(i: usize) -> usize { - use riscv::register::*; match i { 0x0 => pmpaddr0::read(), 0x1 => pmpaddr1::read(),