[feat] add nvme support

api/axfeat/Cargo.toml

@@ -55,6 +55,7 @@ bus-pci = ["axdriver?/bus-pci"]
 driver-ramdisk = ["axdriver?/ramdisk", "axfs?/use-ramdisk"]
 driver-ixgbe = ["axdriver?/ixgbe"]
 driver-bcm2835-sdhci = ["axdriver?/bcm2835-sdhci"]
+driver-nvme = ["axdriver?/nvme"]
 
 # Logging
 log-level-off = ["axlog/log-level-off"]

examples/shell/README.md

@@ -0,0 +1,10 @@
+# Useage
+
+## Example
+
+use nvme as block device.
+
+```bash
+make disk_img
+make A=examples/shell ARCH=x86_64 LOG=info SMP=4 ACCEL=N FEATURES=driver-nvme APP_FEATURES=use-ramfs BLK=y run
+```

modules/axdriver/Cargo.toml

@@ -2,7 +2,10 @@
 name = "axdriver"
 version.workspace = true
 edition = "2021"
-authors = ["Yuekai Jia <[email protected]>", "ChengXiang Qi <[email protected]>"]
+authors = [
+    "Yuekai Jia <[email protected]>",
+    "ChengXiang Qi <[email protected]>",
+]
 description = "ArceOS device drivers"
 license.workspace = true
 homepage.workspace = true
@@ -12,7 +15,7 @@ documentation = "https://arceos-org.github.io/arceos/axdriver/index.html"
 [features]
 dyn = []
 bus-mmio = []
-bus-pci = ["dep:axdriver_pci", "dep:axhal", "dep:axconfig"]
+bus-pci = ["dep:axdriver_pci", "dep:axhal", "dep:axconfig", "dep:pcie"]
 net = ["axdriver_net"]
 block = ["axdriver_block"]
 display = ["axdriver_display"]
@@ -27,6 +30,7 @@ virtio-gpu = ["display", "virtio", "axdriver_virtio/gpu"]
 ramdisk = ["block", "axdriver_block/ramdisk"]
 bcm2835-sdhci = ["block", "axdriver_block/bcm2835-sdhci"]
 ixgbe = ["net", "axdriver_net/ixgbe", "dep:axalloc", "dep:axhal", "dep:axdma"]
+nvme = ["block", "bus-pci", "dep:nvme-driver", "dep:axalloc", "dep:axhal", "axhal/alloc"]
 # more devices example: e1000 = ["net", "axdriver_net/e1000"]
 
 default = ["bus-pci"]
@@ -43,4 +47,7 @@ axdriver_virtio = { git = "https://github.com/arceos-org/axdriver_crates.git", t
 axalloc = { workspace = true, optional = true }
 axhal = { workspace = true, optional = true }
 axconfig = { workspace = true, optional = true }
-axdma = { workspace = true, optional = true }
+axdma = { workspace = true, optional = true }
+pcie = { version = "0.2", optional = true }
+nvme-driver = { version = "0.2", optional = true }
+

modules/axdriver/build.rs

@@ -1,5 +1,5 @@
 const NET_DEV_FEATURES: &[&str] = &["ixgbe", "virtio-net"];
-const BLOCK_DEV_FEATURES: &[&str] = &["ramdisk", "bcm2835-sdhci", "virtio-blk"];
+const BLOCK_DEV_FEATURES: &[&str] = &["nvme", "ramdisk", "bcm2835-sdhci", "virtio-blk"];
 const DISPLAY_DEV_FEATURES: &[&str] = &["virtio-gpu"];
 
 fn make_cfg_values(str_list: &[&str]) -> String {

modules/axdriver/src/bus/pci.rs

@@ -1,8 +1,12 @@
+use alloc::vec::Vec;
+use core::ptr::NonNull;
+
 use crate::{prelude::*, AllDevices};
 use axdriver_pci::{
     BarInfo, Cam, Command, DeviceFunction, HeaderType, MemoryBarType, PciRangeAllocator, PciRoot,
 };
 use axhal::mem::phys_to_virt;
+use pcie::{Header, RootComplexGeneric};
 
 const PCI_BAR_NUM: u8 = 6;
 
@@ -84,6 +88,35 @@ fn config_pci_device(
 
 impl AllDevices {
     pub(crate) fn probe_bus_devices(&mut self) {
+        self.probe_pci_devices();
+        self.probe_pcie_devices();
+    }
+
+    fn probe_pcie_devices(&mut self) {
+        let base_vaddr = phys_to_virt(axconfig::PCI_ECAM_BASE.into());
+        let mut root = RootComplexGeneric::new(NonNull::new(base_vaddr.as_mut_ptr()).unwrap());
+
+        let headers = root.enumerate_no_modify(None).collect::<Vec<_>>();
+
+        for header in headers {
+            if let Header::Endpoint(ep) = header {
+                for_each_drivers!(type Driver, {
+                    if let Some(dev) = Driver::probe_pcie(&mut root, ep.clone()) {
+                        info!(
+                            "registered a new {:?} device at {:?}: {:?}",
+                            dev.device_type(),
+                            ep.address,
+                            dev.device_name(),
+                        );
+                        self.add_device(dev);
+                        continue; // skip to the next device
+                    }
+                })
+            }
+        }
+    }
+
+    fn probe_pci_devices(&mut self) {
         let base_vaddr = phys_to_virt(axconfig::PCI_ECAM_BASE.into());
         let mut root = unsafe { PciRoot::new(base_vaddr.as_mut_ptr(), Cam::Ecam) };
 

modules/axdriver/src/drivers.rs

@@ -31,6 +31,14 @@ pub trait DriverProbe {
     ) -> Option<AxDeviceEnum> {
         None
     }
+
+    #[cfg(bus = "pci")]
+    fn probe_pcie(
+        _root: &mut pcie::RootComplexGeneric,
+        _ep: pcie::Endpoint,
+    ) -> Option<AxDeviceEnum> {
+        None
+    }
 }
 
 #[cfg(net_dev = "virtio-net")]
@@ -128,3 +136,23 @@ cfg_if::cfg_if! {
         }
     }
 }
+
+cfg_if::cfg_if! {
+    if #[cfg(block_dev="nvme")] {
+        use crate::nvme::Nvme;
+        use pcie::{Endpoint, RootComplexGeneric};
+        pub struct NvmeDriver;
+        register_block_driver!(NvmeDriver, Nvme);
+
+        impl DriverProbe for NvmeDriver {
+            #[cfg(bus = "pci")]
+            fn probe_pcie(root: &mut RootComplexGeneric, ep: Endpoint) -> Option<crate::AxDeviceEnum> {
+                use crate::AxDeviceEnum;
+
+                let dev = Nvme::new(root, ep)?;
+
+                Some(AxDeviceEnum::from_block(dev))
+            }
+        }
+    }
+}

modules/axdriver/src/lib.rs

@@ -61,7 +61,7 @@
 #[macro_use]
 extern crate log;
 
-#[cfg(feature = "dyn")]
+#[cfg(any(feature = "dyn", feature = "bus-pci"))]
 extern crate alloc;
 
 #[macro_use]
@@ -78,6 +78,9 @@ mod virtio;
 #[cfg(feature = "ixgbe")]
 mod ixgbe;
 
+#[cfg(feature = "nvme")]
+mod nvme;
+
 pub mod prelude;
 
 #[allow(unused_imports)]

modules/axdriver/src/macros.rs

@@ -64,5 +64,10 @@ macro_rules! for_each_drivers {
             type $drv_type = crate::drivers::IxgbeDriver;
             $code
         }
+        #[cfg(block_dev = "nvme")]
+        {
+            type $drv_type = crate::drivers::NvmeDriver;
+            $code
+        }
     }};
 }

modules/axdriver/src/nvme.rs

@@ -0,0 +1,98 @@
+use core::ptr::NonNull;
+
+use axhal::mem::phys_to_virt;
+use nvme_driver::{Config, Namespace};
+use pcie::{Chip, CommandRegister, DeviceType, Endpoint, RootComplex};
+
+use crate::{BaseDriverOps, BlockDriverOps};
+
+pub struct Nvme {
+    inner: nvme_driver::Nvme,
+    ns: Namespace,
+}
+
+unsafe impl Send for Nvme {}
+unsafe impl Sync for Nvme {}
+
+impl Nvme {
+    pub fn new<C: Chip>(root: &mut RootComplex<C>, ep: Endpoint) -> Option<Self> {
+        info!("probe: {:?}", ep);
+
+        ep.update_command(root, |cmd| {
+            cmd | CommandRegister::IO_ENABLE
+                | CommandRegister::MEMORY_ENABLE
+                | CommandRegister::BUS_MASTER_ENABLE
+        });
+
+        if ep.device_type() == DeviceType::NvmeController {
+            let bar_addr = match ep.bar {
+                pcie::BarVec::Memory32(bar_vec_t) => {
+                    let bar0 = bar_vec_t[0].as_ref().unwrap();
+                    bar0.address as usize
+                }
+                pcie::BarVec::Memory64(bar_vec_t) => {
+                    let bar0 = bar_vec_t[0].as_ref().unwrap();
+                    bar0.address as usize
+                }
+                pcie::BarVec::Io(_bar_vec_t) => return None,
+            };
+
+            let addr = phys_to_virt(bar_addr.into());
+
+            let mut nvme = nvme_driver::Nvme::new(
+                unsafe { NonNull::new_unchecked(addr.as_mut_ptr()) },
+                Config {
+                    page_size: 0x1000,
+                    io_queue_pair_count: 1,
+                },
+            )
+            .inspect_err(|e| error!("{:?}", e))
+            .unwrap();
+            let ns_list = nvme.namespace_list().ok()?;
+            let ns = ns_list.first()?;
+
+            return Some(Self {
+                inner: nvme,
+                ns: *ns,
+            });
+        }
+
+        None
+    }
+}
+
+impl BaseDriverOps for Nvme {
+    fn device_name(&self) -> &str {
+        "NVME"
+    }
+
+    fn device_type(&self) -> crate::DeviceType {
+        crate::DeviceType::Block
+    }
+}
+
+impl BlockDriverOps for Nvme {
+    fn num_blocks(&self) -> u64 {
+        self.ns.lba_count as _
+    }
+
+    fn block_size(&self) -> usize {
+        self.ns.lba_size
+    }
+
+    fn read_block(&mut self, block_id: u64, buf: &mut [u8]) -> crate::DevResult {
+        self.inner
+            .block_read_sync(&self.ns, block_id as _, buf)
+            .map_err(|_e| crate::DevError::Io)
+    }
+
+    fn write_block(&mut self, block_id: u64, buf: &[u8]) -> crate::DevResult {
+        self.inner
+            .block_write_sync(&self.ns, block_id, buf)
+            .map_err(|_e| crate::DevError::Io)
+    }
+
+    fn flush(&mut self) -> crate::DevResult {
+        Ok(())
+    }
+}

modules/axhal/Cargo.toml

@@ -11,7 +11,7 @@ documentation = "https://arceos-org.github.io/arceos/axhal/index.html"
 
 [features]
 smp = []
-alloc = []
+alloc = ["dep:dma-api"]
 fp_simd = []
 paging = ["axalloc", "page_table_multiarch"]
 irq = []
@@ -37,6 +37,7 @@ page_table_multiarch = { version = "0.4", optional = true }
 axlog = { workspace = true }
 axconfig = { workspace = true }
 axalloc = { workspace = true, optional = true }
+dma-api = { version = "0.1", optional = true }
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
 x86 = "0.52"

modules/axhal/src/arch/aarch64/cache.rs

@@ -0,0 +1,49 @@
+#![allow(unused)]
+
+use core::{arch::asm, ptr::NonNull};
+
+fn dcache_line_size() -> usize {
+    unsafe {
+        let result;
+        asm!(
+            "mrs    x8, CTR_EL0",
+            "ubfm x8, x8, #16, #19",	// cache line size encoding
+            "mov		{0}, #4",		// bytes per word
+            "lsl		{0}, {0}, x8",	// actual cache line size""",
+            out(reg) result);
+
+        result
+    }
+}
+
+/// Invalidate data cache
+pub fn dcache_invalidate_range(addr: NonNull<u8>, size: usize) {
+    let addr = addr.as_ptr() as usize;
+    unsafe {
+        let line_size = dcache_line_size();
+        let start = addr & !(line_size - 1);
+        let end = (addr + size + line_size - 1) & !(line_size - 1);
+
+        for addr in (start..end).step_by(line_size) {
+            asm!("dc ivac, {0}", in(reg) addr);
+        }
+
+        asm!("dsb sy");
+    }
+}
+
+/// Clean data cache
+pub fn dcache_clean_range(addr: NonNull<u8>, size: usize) {
+    let addr = addr.as_ptr() as usize;
+    unsafe {
+        let line_size = dcache_line_size();
+        let start = addr & !(line_size - 1);
+        let end = (addr + size + line_size - 1) & !(line_size - 1);
+
+        for addr in (start..end).step_by(line_size) {
+            asm!("dc cvac, {0}", in(reg) addr);
+        }
+
+        asm!("dsb sy");
+    }
+}

modules/axhal/src/arch/aarch64/mod.rs

@@ -1,3 +1,4 @@
+pub(crate) mod cache;
 mod context;
 pub(crate) mod trap;
 

modules/axhal/src/arch/riscv/cache.rs

@@ -0,0 +1,13 @@
+#![allow(unused)]
+
+use core::ptr::NonNull;
+
+/// Invalidate data cache
+pub fn dcache_invalidate_range(_addr: NonNull<u8>, _size: usize) {
+    //TODO
+}
+
+/// Clean data cache
+pub fn dcache_clean_range(_addr: NonNull<u8>, _size: usize) {
+    //TODO
+}

modules/axhal/src/arch/riscv/mod.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 mod macros;
 
+pub mod cache;
 mod context;
 mod trap;
 

modules/axhal/src/arch/x86_64/cache.rs

@@ -0,0 +1,13 @@
+#![allow(unused)]
+
+use core::ptr::NonNull;
+
+/// Invalidate data cache
+pub fn dcache_invalidate_range(_addr: NonNull<u8>, _size: usize) {
+    //TODO
+}
+
+/// Clean data cache
+pub fn dcache_clean_range(_addr: NonNull<u8>, _size: usize) {
+    //TODO
+}

modules/axhal/src/arch/x86_64/mod.rs

@@ -1,3 +1,4 @@
+pub(crate) mod cache;
 mod context;
 mod gdt;
 mod idt;