diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index aa294bcf99b2a0..7b52141077b4eb 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -435,7 +435,7 @@ impl RawAllocation for SimpleAllocation { pub(crate) struct SimpleAllocator { dev: AsahiDevRef, range: Range, - prot: u32, + prot: mmu::Prot, vm: mmu::Vm, min_align: usize, cpu_maps: bool, @@ -450,7 +450,7 @@ impl SimpleAllocator { vm: &mmu::Vm, range: Range, min_align: usize, - prot: u32, + prot: mmu::Prot, _block_size: usize, mut cpu_maps: bool, _name: fmt::Arguments<'_>, @@ -642,7 +642,7 @@ pub(crate) struct HeapAllocator { dev: AsahiDevRef, range: Range, top: u64, - prot: u32, + prot: mmu::Prot, vm: mmu::Vm, min_align: usize, block_size: usize, @@ -662,7 +662,7 @@ impl HeapAllocator { vm: &mmu::Vm, range: Range, min_align: usize, - prot: u32, + prot: mmu::Prot, block_size: usize, mut cpu_maps: bool, name: fmt::Arguments<'_>, diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 5f1f4f181e0cce..34a3c70f75a360 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -20,6 +20,7 @@ mod mem; mod microseq; mod mmu; mod object; +mod pgtable; mod queue; mod regs; mod slotalloc; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index a323a6611ec86a..6aa07ca765750f 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -77,7 +77,7 @@ impl ObjectRef { vm: &crate::mmu::Vm, range: Range, alignment: u64, - prot: u32, + prot: mmu::Prot, guard: bool, ) -> Result { // Only used for kernel objects now @@ -94,7 +94,7 @@ impl ObjectRef { obj_range: Range, range: Range, alignment: u64, - prot: u32, + prot: mmu::Prot, guard: bool, ) -> Result { if obj_range.end > self.gem.size() { @@ -113,7 +113,7 @@ impl ObjectRef { &mut self, vm: &crate::mmu::Vm, addr: u64, - prot: u32, + prot: mmu::Prot, guard: bool, ) -> Result { if self.gem.flags & uapi::ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(&self.gem) { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index a2121bed8da564..3a759d93ba2ca4 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -19,11 +19,10 @@ use core::sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}; use core::time::Duration; use kernel::{ + addr::PhysicalAddr, bindings, c_str, delay, device, drm, drm::{gem::BaseObject, gpuvm, mm}, error::{to_result, Result}, - io_pgtable, - io_pgtable::{prot, AppleUAT, IoPageTable}, prelude::*, static_lock_class, sync::{ @@ -31,12 +30,19 @@ use kernel::{ Arc, Mutex, }, time::{clock, Now}, - types::{ARef, ForeignOwnable}, + types::ARef, }; use crate::debug::*; use crate::no_debug; -use crate::{driver, fw, gem, hw, mem, slotalloc, util::RangeExt}; +use crate::{driver, fw, gem, hw, mem, pgtable, slotalloc, util::RangeExt}; + +// KernelMapping protection types +pub(crate) use crate::pgtable::Prot; +pub(crate) use pgtable::prot::*; +pub(crate) use pgtable::{UatPageTable, UAT_PGBIT, UAT_PGMSK, UAT_PGSZ}; + +use pgtable::UAT_IAS; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -75,51 +81,9 @@ pub(crate) const IOVA_UNK_PAGE: u64 = IOVA_USER_TOP - 2 * UAT_PGSZ as u64; /// User VA range excluding the unk page pub(crate) const IOVA_USER_USABLE_RANGE: Range = IOVA_USER_BASE..IOVA_UNK_PAGE; -// KernelMapping protection types - -// Note: prot::CACHE means "cache coherency", which for UAT means *uncached*, -// since uncached mappings from the GFX ASC side are cache coherent with the AP cache. -// Not having that flag means *cached noncoherent*. - -/// Firmware MMIO R/W -pub(crate) const PROT_FW_MMIO_RW: u32 = - prot::PRIV | prot::READ | prot::WRITE | prot::CACHE | prot::MMIO; -/// Firmware MMIO R/O -pub(crate) const PROT_FW_MMIO_RO: u32 = prot::PRIV | prot::READ | prot::CACHE | prot::MMIO; -/// Firmware shared (uncached) RW -pub(crate) const PROT_FW_SHARED_RW: u32 = prot::PRIV | prot::READ | prot::WRITE | prot::CACHE; -/// Firmware shared (uncached) RO -pub(crate) const PROT_FW_SHARED_RO: u32 = prot::PRIV | prot::READ | prot::CACHE; -/// Firmware private (cached) RW -pub(crate) const PROT_FW_PRIV_RW: u32 = prot::PRIV | prot::READ | prot::WRITE; -/* -/// Firmware private (cached) RO -pub(crate) const PROT_FW_PRIV_RO: u32 = prot::PRIV | prot::READ; -*/ -/// Firmware/GPU shared (uncached) RW -pub(crate) const PROT_GPU_FW_SHARED_RW: u32 = prot::READ | prot::WRITE | prot::CACHE; -/// Firmware/GPU shared (private) RW -pub(crate) const PROT_GPU_FW_PRIV_RW: u32 = prot::READ | prot::WRITE; -/// Firmware-RW/GPU-RO shared (private) RW -pub(crate) const PROT_GPU_RO_FW_PRIV_RW: u32 = prot::PRIV | prot::WRITE; -/// GPU shared/coherent RW -pub(crate) const PROT_GPU_SHARED_RW: u32 = prot::READ | prot::WRITE | prot::CACHE | prot::NOEXEC; -/// GPU shared/coherent RO -pub(crate) const PROT_GPU_SHARED_RO: u32 = prot::READ | prot::CACHE | prot::NOEXEC; -/// GPU shared/coherent WO -pub(crate) const PROT_GPU_SHARED_WO: u32 = prot::WRITE | prot::CACHE | prot::NOEXEC; -/* -/// GPU private/noncoherent RW -pub(crate) const PROT_GPU_PRIV_RW: u32 = prot::READ | prot::WRITE | prot::NOEXEC; -/// GPU private/noncoherent RO -pub(crate) const PROT_GPU_PRIV_RO: u32 = prot::READ | prot::NOEXEC; -*/ - -type PhysAddr = bindings::phys_addr_t; - /// A pre-allocated memory region for UAT management struct UatRegion { - base: PhysAddr, + base: PhysicalAddr, map: NonNull, } @@ -173,7 +137,7 @@ struct VmInner { dev: driver::AsahiDevRef, is_kernel: bool, va_range: Range, - page_table: AppleUAT, + page_table: UatPageTable, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, binding: Arc>, @@ -209,7 +173,7 @@ struct StepContext { prev_va: Option>>>, next_va: Option>>>, vm_bo: Option>>, - prot: u32, + prot: Prot, } impl gpuvm::DriverGpuVm for VmInner { @@ -260,7 +224,8 @@ impl gpuvm::DriverGpuVm for VmInner { iova ); - self.map_pages(iova, addr, UAT_PGSZ, len >> UAT_PGBIT, ctx.prot)?; + self.page_table + .map_pages(iova..(iova + len as u64), addr as PhysicalAddr, ctx.prot)?; left -= len; iova += len as u64; @@ -296,7 +261,8 @@ impl gpuvm::DriverGpuVm for VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); - self.unmap_pages(va.addr(), UAT_PGSZ, (va.range() >> UAT_PGBIT) as usize)?; + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -341,18 +307,18 @@ impl gpuvm::DriverGpuVm for VmInner { orig_addr + orig_range }; - let unmap_range = unmap_end - unmap_start; - mod_dev_dbg!( self.dev, - "MMU: unmap for remap: {:#x}:{:#x} (from {:#x}:{:#x})\n", + "MMU: unmap for remap: {:#x}..{:#x} (from {:#x}:{:#x})\n", unmap_start, - unmap_range, + unmap_end, orig_addr, orig_range ); - self.unmap_pages(unmap_start, UAT_PGSZ, (unmap_range >> UAT_PGBIT) as usize)?; + let unmap_range = unmap_end - unmap_start; + + self.page_table.unmap_pages(unmap_start..unmap_end)?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -414,78 +380,22 @@ impl VmInner { /// Returns the translation table base for this Vm fn ttb(&self) -> u64 { - self.page_table.cfg().ttbr - } - - /// Map an IOVA to the shifted address the underlying io_pgtable uses. - fn map_iova(&self, iova: u64, size: usize) -> Result { - if !self.va_range.is_superset(iova..(iova + size as u64)) { - Err(EINVAL) - } else if self.is_kernel { - Ok(iova - self.va_range.start) - } else { - Ok(iova) - } - } - - /// Map a contiguous range of virtual->physical pages. - fn map_pages( - &mut self, - mut iova: u64, - mut paddr: usize, - pgsize: usize, - pgcount: usize, - prot: u32, - ) -> Result { - let mut left = pgcount; - while left > 0 { - let mapped_iova = self.map_iova(iova, pgsize * left)?; - let mapped = - self.page_table - .map_pages(mapped_iova as usize, paddr, pgsize, left, prot)?; - assert!(mapped <= left * pgsize); - - left -= mapped / pgsize; - paddr += mapped; - iova += mapped as u64; - } - Ok(pgcount * pgsize) - } - - /// Unmap a contiguous range of pages. - fn unmap_pages(&mut self, mut iova: u64, pgsize: usize, pgcount: usize) -> Result { - let mut left = pgcount; - while left > 0 { - let mapped_iova = self.map_iova(iova, pgsize * left)?; - let mut unmapped = self - .page_table - .unmap_pages(mapped_iova as usize, pgsize, left); - if unmapped == 0 { - dev_err!( - self.dev.as_ref(), - "unmap_pages {:#x}:{:#x} returned 0\n", - mapped_iova, - left - ); - unmapped = pgsize; // Pretend we unmapped one page and try again... - } - assert!(unmapped <= left * pgsize); - - left -= unmapped / pgsize; - iova += unmapped as u64; - } - - Ok(pgcount * pgsize) + self.page_table.ttb() as u64 } /// Map an `mm::Node` representing an mapping in VA space. - fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: u32) -> Result { + fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().sgt.lock(); let sgt = guard.as_ref().ok_or(EINVAL)?; let mut offset = node.offset; + let mut left = node.mapped_size; for range in sgt.iter() { + if left == 0 { + break; + } + let mut addr = range.dma_address(); let mut len = range.dma_len(); @@ -507,6 +417,8 @@ impl VmInner { offset -= skip; } + len = len.min(left); + if len == 0 { continue; } @@ -519,9 +431,11 @@ impl VmInner { iova ); - self.map_pages(iova, addr, UAT_PGSZ, len >> UAT_PGBIT, prot)?; + self.page_table + .map_pages(iova..(iova + len as u64), addr as PhysicalAddr, prot)?; iova += len as u64; + left -= len; } Ok(()) } @@ -598,7 +512,7 @@ pub(crate) struct KernelMappingInner { _gem: Option>, owner: ARef>, uat_inner: Arc, - prot: u32, + prot: Prot, offset: usize, mapped_size: usize, } @@ -618,13 +532,18 @@ impl KernelMapping { self.0.mapped_size } + /// Returns the IOVA base of this mapping + pub(crate) fn iova_range(&self) -> Range { + self.0.start()..(self.0.start() + self.0.mapped_size as u64) + } + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { let mut owner = self .0 .owner - .exec_lock(None) + .exec_lock(None, false) .expect("Failed to exec_lock in remap_uncached_and_flush"); mod_dev_dbg!( @@ -634,23 +553,14 @@ impl KernelMapping { self.size() ); - // The IOMMU API does not allow us to remap things in-place... - // just do an unmap and map again for now. - // Do not try to unmap guard page (-1) + // Remap in-place as uncached. + // Do not try to unmap the guard page (-1) + let prot = self.0.prot.as_uncached(); if owner - .unmap_pages(self.iova(), UAT_PGSZ, self.size() >> UAT_PGBIT) + .page_table + .reprot_pages(self.iova_range(), prot) .is_err() { - dev_err!( - owner.dev.as_ref(), - "MMU: unmap for remap {:#x}:{:#x} failed\n", - self.iova(), - self.size() - ); - } - - let prot = self.0.prot | prot::CACHE; - if owner.map_node(&self.0, prot).is_err() { dev_err!( owner.dev.as_ref(), "MMU: remap {:#x}:{:#x} failed\n", @@ -779,15 +689,19 @@ impl Drop for KernelMapping { // 4. Unmap // 5. Flush the TLB range again - // prot::CACHE means "cache coherent" which means *uncached* here. - if self.0.prot & prot::CACHE == 0 { + if self.0.prot.is_cached_noncoherent() { + mod_pr_debug!( + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); self.remap_uncached_and_flush(); } let mut owner = self .0 .owner - .exec_lock(None) + .exec_lock(None, false) .expect("exec_lock failed in KernelMapping::drop"); mod_dev_dbg!( owner.dev, @@ -796,10 +710,7 @@ impl Drop for KernelMapping { self.size() ); - if owner - .unmap_pages(self.iova(), UAT_PGSZ, self.size() >> UAT_PGBIT) - .is_err() - { + if owner.page_table.unmap_pages(self.iova_range()).is_err() { dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -873,7 +784,6 @@ impl UatInner { pub(crate) struct Uat { dev: driver::AsahiDevRef, cfg: &'static hw::HwConfig, - pagetables_rgn: UatRegion, inner: Arc, slots: slotalloc::SlotAllocator, @@ -995,27 +905,6 @@ impl HandoffFlush { } } -// We do not implement FlushOps, since we flush manually in this module after -// page table operations. Just provide dummy implementations. -impl io_pgtable::FlushOps for Uat { - type Data = (); - - fn tlb_flush_all(_data: ::Borrowed<'_>) {} - fn tlb_flush_walk( - _data: ::Borrowed<'_>, - _iova: usize, - _size: usize, - _granule: usize, - ) { - } - fn tlb_add_page( - _data: ::Borrowed<'_>, - _iova: usize, - _granule: usize, - ) { - } -} - impl Vm { /// Create a new virtual memory address space fn new( @@ -1023,22 +912,18 @@ impl Vm { uat_inner: Arc, kernel_range: Range, cfg: &'static hw::HwConfig, - is_kernel: bool, + ttb: Option, id: u64, ) -> Result { - let dummy_obj = gem::new_kernel_object(dev, 0x4000)?; - - let page_table = AppleUAT::new( - dev.as_ref(), - io_pgtable::Config { - pgsize_bitmap: UAT_PGSZ, - ias: if is_kernel { UAT_IAS_KERN } else { UAT_IAS }, - oas: cfg.uat_oas, - coherent_walk: true, - quirks: 0, - }, - (), - )?; + let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; + let is_kernel = ttb.is_some(); + + let page_table = if let Some(ttb) = ttb { + UatPageTable::new_with_ttb(ttb, IOVA_KERN_RANGE, cfg.uat_oas)? + } else { + UatPageTable::new(cfg.uat_oas)? + }; + let (va_range, gpuvm_range) = if is_kernel { (IOVA_KERN_RANGE, kernel_range.clone()) } else { @@ -1053,7 +938,7 @@ impl Vm { binding: None, bind_token: None, active_users: 0, - ttb: page_table.cfg().ttbr, + ttb: page_table.ttb(), }, c_str!("VmBinding"), ), @@ -1098,12 +983,12 @@ impl Vm { object_range: Range, alignment: u64, range: Range, - prot: u32, + prot: Prot, guard: bool, ) -> Result { let size = object_range.range(); let sgt = gem.sg_table()?; - let mut inner = self.inner.exec_lock(Some(gem))?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = inner.obtain_bo()?; let mut vm_bo_guard = vm_bo.inner().sgt.lock(); @@ -1131,7 +1016,11 @@ impl Vm { mm::InsertMode::Best, )?; - inner.map_node(&node, prot)?; + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; Ok(KernelMapping(node)) } @@ -1142,11 +1031,11 @@ impl Vm { addr: u64, size: usize, gem: &gem::Object, - prot: u32, + prot: Prot, guard: bool, ) -> Result { let sgt = gem.sg_table()?; - let mut inner = self.inner.exec_lock(Some(gem))?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = inner.obtain_bo()?; @@ -1172,7 +1061,11 @@ impl Vm { 0, )?; - inner.map_node(&node, prot)?; + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; Ok(KernelMapping(node)) } @@ -1184,7 +1077,7 @@ impl Vm { addr: u64, size: u64, offset: u64, - prot: u32, + prot: Prot, ) -> Result { // Mapping needs a complete context let mut ctx = StepContext { @@ -1196,7 +1089,7 @@ impl Vm { }; let sgt = gem.sg_table()?; - let mut inner = self.inner.exec_lock(Some(gem))?; + let mut inner = self.inner.exec_lock(Some(gem), true)?; let vm_bo = inner.obtain_bo()?; @@ -1235,9 +1128,9 @@ impl Vm { iova: u64, phys: usize, size: usize, - prot: u32, + prot: Prot, ) -> Result { - let mut inner = self.inner.exec_lock(None)?; + let mut inner = self.inner.exec_lock(None, false)?; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -1274,8 +1167,14 @@ impl Vm { 0, )?; - inner.map_pages(iova, phys, UAT_PGSZ, size >> UAT_PGBIT, prot)?; - + let ret = + inner + .page_table + .map_pages(iova..(iova + size as u64), phys as PhysicalAddr, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; Ok(KernelMapping(node)) } @@ -1289,7 +1188,7 @@ impl Vm { ..Default::default() }; - let mut inner = self.inner.exec_lock(None)?; + let mut inner = self.inner.exec_lock(None, false)?; mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); inner.sm_unmap(&mut ctx, iova, size) @@ -1300,7 +1199,7 @@ impl Vm { // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); - let mut inner = self.inner.exec_lock(Some(gem))?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; if let Some(bo) = inner.find_bo() { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); @@ -1375,13 +1274,7 @@ impl Drop for VmInner { } impl Uat { - /// Map a bootloader-preallocated memory region - fn map_region( - dev: &device::Device, - name: &CStr, - size: usize, - cached: bool, - ) -> Result { + fn get_region(dev: &device::Device, name: &CStr) -> Result<(PhysicalAddr, usize)> { let mut res = core::mem::MaybeUninit::::uninit(); let res = unsafe { @@ -1407,6 +1300,18 @@ impl Uat { let rgn_size: usize = unsafe { bindings::resource_size(&res) } as usize; + Ok((res.start, rgn_size)) + } + + /// Map a bootloader-preallocated memory region + fn map_region( + dev: &device::Device, + name: &CStr, + size: usize, + cached: bool, + ) -> Result { + let (start, rgn_size) = Self::get_region(dev, name)?; + if size > rgn_size { dev_err!( dev, @@ -1423,7 +1328,7 @@ impl Uat { } else { bindings::MEMREMAP_WC }; - let map = unsafe { bindings::memremap(res.start, rgn_size, flags.into()) }; + let map = unsafe { bindings::memremap(start, rgn_size, flags.into()) }; let map = NonNull::new(map); match map { @@ -1431,19 +1336,10 @@ impl Uat { dev_err!(dev, "Failed to remap {} region\n", name); Err(ENOMEM) } - Some(map) => Ok(UatRegion { - base: res.start, - map, - }), + Some(map) => Ok(UatRegion { base: start, map }), } } - /// Returns a view into the root kernel (upper half) page table - fn kpt0(&self) -> &[Pte; UAT_NPTE] { - // SAFETY: pointer is non-null per the type invariant - unsafe { (self.pagetables_rgn.map.as_ptr() as *mut [Pte; UAT_NPTE]).as_ref() }.unwrap() - } - /// Returns a reference to the global kernel (upper half) `Vm` pub(crate) fn kernel_vm(&self) -> &Vm { &self.kernel_vm @@ -1501,8 +1397,8 @@ impl Uat { idx ); } - ttbs[idx].ttb0.store(ttb, Ordering::Relaxed); - ttbs[idx].ttb1.store(ttb1, Ordering::Relaxed); + ttbs[idx].ttb0.store(ttb, Ordering::Release); + ttbs[idx].ttb1.store(ttb1, Ordering::Release); uat_inner.handoff().unlock(); core::mem::drop(uat_inner); @@ -1529,7 +1425,7 @@ impl Uat { self.inner.clone(), kernel_range, self.cfg, - false, + None, id, ) } @@ -1574,22 +1470,23 @@ impl Uat { let inner = Self::make_inner(dev)?; - let pagetables_rgn = - Self::map_region(dev.as_ref(), c_str!("pagetables"), PAGETABLES_SIZE, true)?; + let (ttb1, ttb1size) = Self::get_region(dev.as_ref(), c_str!("pagetables"))?; + if ttb1size < PAGETABLES_SIZE { + dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); + return Err(ENOMEM); + } dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); - let kernel_lower_vm = Vm::new(dev, inner.clone(), IOVA_USER_RANGE, cfg, false, 1)?; - let kernel_vm = Vm::new(dev, inner.clone(), IOVA_KERN_RANGE, cfg, true, 0)?; + let kernel_lower_vm = Vm::new(dev, inner.clone(), IOVA_USER_RANGE, cfg, None, 1)?; + let kernel_vm = Vm::new(dev, inner.clone(), IOVA_KERN_RANGE, cfg, Some(ttb1), 0)?; dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); let ttb0 = kernel_lower_vm.ttb(); - let ttb1 = kernel_vm.ttb(); let uat = Self { dev: dev.into(), cfg, - pagetables_rgn, kernel_vm, kernel_lower_vm, inner, @@ -1606,7 +1503,7 @@ impl Uat { let mut inner = uat.inner.lock(); inner.map_kernel_to_user = map_kernel_to_user; - inner.kernel_ttb1 = uat.pagetables_rgn.base; + inner.kernel_ttb1 = ttb1; inner.handoff().init()?; @@ -1616,10 +1513,8 @@ impl Uat { let ttbs = inner.ttbs(); - ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::Relaxed); - ttbs[0] - .ttb1 - .store(uat.pagetables_rgn.base | TTBR_VALID, Ordering::Relaxed); + ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::SeqCst); + ttbs[0].ttb1.store(ttb1 | TTBR_VALID, Ordering::SeqCst); for ctx in &ttbs[1..] { ctx.ttb0.store(0, Ordering::Relaxed); @@ -1630,8 +1525,6 @@ impl Uat { core::mem::drop(inner); - uat.kpt0()[2].store(ttb1 | PTE_TABLE, Ordering::Relaxed); - dev_info!(dev.as_ref(), "MMU: initialized\n"); Ok(uat) @@ -1640,9 +1533,6 @@ impl Uat { impl Drop for Uat { fn drop(&mut self) { - // Unmap what we mapped - self.kpt0()[2].store(0, Ordering::Relaxed); - // Make sure we flush the TLBs fence(Ordering::SeqCst); mem::tlbi_all(); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs new file mode 100644 index 00000000000000..7871b038e53512 --- /dev/null +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! UAT Page Table management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the actual page tables by allocating raw memory pages from +//! the kernel page allocator. + +use core::fmt::Debug; +use core::mem::size_of; +use core::ops::Range; +use core::sync::atomic::{AtomicU64, Ordering}; + +use kernel::addr::PhysicalAddr; +use kernel::{error::Result, page::Page, prelude::*}; + +use crate::debug::*; +use crate::util::align; + +const DEBUG_CLASS: DebugFlags = DebugFlags::PgTable; + +/// Number of bits in a page offset. +pub(crate) const UAT_PGBIT: usize = 14; +/// UAT page size. +pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; +/// UAT page offset mask. +pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; + +type Pte = AtomicU64; + +const PTE_BIT: usize = 3; // log2(sizeof(Pte)) +const PTE_SIZE: usize = 1 << PTE_BIT; + +/// Number of PTEs per page. +const UAT_NPTE: usize = UAT_PGSZ / size_of::(); + +/// Number of address bits to address a level +const UAT_LVBIT: usize = UAT_PGBIT - PTE_BIT; +/// Number of entries per level +const UAT_LVSZ: usize = UAT_NPTE; +/// Mask of level bits +const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; + +const UAT_LEVELS: usize = 3; + +/// UAT input address space +pub(crate) const UAT_IAS: usize = 39; +const UAT_IASMSK: u64 = ((1u64 << UAT_IAS) - 1) as u64; + +const PTE_TYPE_BITS: u64 = 3; +const PTE_TYPE_LEAF_TABLE: u64 = 3; + +const UAT_AP_SHIFT: u32 = 6; +const UAT_AP_BITS: u64 = 3 << UAT_AP_SHIFT; +const UAT_HIGH_BITS_SHIFT: u32 = 53; +const UAT_HIGH_BITS: u64 = 7 << UAT_HIGH_BITS_SHIFT; +const UAT_MEMATTR_SHIFT: u32 = 2; +const UAT_MEMATTR_BITS: u64 = 7 << UAT_MEMATTR_SHIFT; + +const UAT_PROT_BITS: u64 = UAT_AP_BITS | UAT_MEMATTR_BITS | UAT_HIGH_BITS; + +const UAT_AF: u64 = 1 << 10; + +const MEMATTR_CACHED: u8 = 0; +const MEMATTR_DEV: u8 = 1; +const MEMATTR_UNCACHED: u8 = 2; + +const AP_FW_GPU: u8 = 0; +const AP_FW: u8 = 1; +const AP_GPU: u8 = 2; + +const HIGH_BITS_PXN: u8 = 1 << 0; +const HIGH_BITS_UXN: u8 = 1 << 1; +const HIGH_BITS_GPU_ACCESS: u8 = 1 << 2; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct Prot { + memattr: u8, + ap: u8, + high_bits: u8, +} + +// Firmware + GPU access +const PROT_FW_GPU_NA: Prot = Prot::from_bits(AP_FW_GPU, 0, 0); +const _PROT_FW_GPU_RO: Prot = Prot::from_bits(AP_FW_GPU, 0, 1); +const _PROT_FW_GPU_WO: Prot = Prot::from_bits(AP_FW_GPU, 1, 0); +const PROT_FW_GPU_RW: Prot = Prot::from_bits(AP_FW_GPU, 1, 1); + +// Firmware only access +const PROT_FW_RO: Prot = Prot::from_bits(AP_FW, 0, 0); +const _PROT_FW_NA: Prot = Prot::from_bits(AP_FW, 0, 1); +const PROT_FW_RW: Prot = Prot::from_bits(AP_FW, 1, 0); +const PROT_FW_RW_GPU_RO: Prot = Prot::from_bits(AP_FW, 1, 1); + +// GPU only access +const PROT_GPU_RO: Prot = Prot::from_bits(AP_GPU, 0, 0); +const PROT_GPU_WO: Prot = Prot::from_bits(AP_GPU, 0, 1); +const PROT_GPU_RW: Prot = Prot::from_bits(AP_GPU, 1, 0); +const _PROT_GPU_NA: Prot = Prot::from_bits(AP_GPU, 1, 1); + +pub(crate) mod prot { + pub(crate) use super::Prot; + use super::*; + + /// Firmware MMIO R/W + pub(crate) const PROT_FW_MMIO_RW: Prot = PROT_FW_RW.memattr(MEMATTR_DEV); + /// Firmware MMIO R/O + pub(crate) const PROT_FW_MMIO_RO: Prot = PROT_FW_RO.memattr(MEMATTR_DEV); + /// Firmware shared (uncached) RW + pub(crate) const PROT_FW_SHARED_RW: Prot = PROT_FW_RW.memattr(MEMATTR_UNCACHED); + /// Firmware shared (uncached) RO + pub(crate) const PROT_FW_SHARED_RO: Prot = PROT_FW_RO.memattr(MEMATTR_UNCACHED); + /// Firmware private (cached) RW + pub(crate) const PROT_FW_PRIV_RW: Prot = PROT_FW_RW.memattr(MEMATTR_CACHED); + /// Firmware/GPU shared (uncached) RW + pub(crate) const PROT_GPU_FW_SHARED_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_UNCACHED); + /// Firmware/GPU shared (private) RW + pub(crate) const PROT_GPU_FW_PRIV_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_CACHED); + /// Firmware-RW/GPU-RO shared (private) RW + pub(crate) const PROT_GPU_RO_FW_PRIV_RW: Prot = PROT_FW_RW_GPU_RO.memattr(MEMATTR_CACHED); + /// GPU shared/coherent RW + pub(crate) const PROT_GPU_SHARED_RW: Prot = PROT_GPU_RW; + /// GPU shared/coherent RO + pub(crate) const PROT_GPU_SHARED_RO: Prot = PROT_GPU_RO; + /// GPU shared/coherent WO + pub(crate) const PROT_GPU_SHARED_WO: Prot = PROT_GPU_WO; +} + +impl Prot { + const fn from_bits(ap: u8, uxn: u8, pxn: u8) -> Self { + assert!(uxn <= 1); + assert!(pxn <= 1); + assert!(ap <= 3); + + Prot { + high_bits: HIGH_BITS_GPU_ACCESS | (pxn * HIGH_BITS_PXN) | (uxn * HIGH_BITS_UXN), + memattr: 0, + ap, + } + } + + const fn memattr(&self, memattr: u8) -> Self { + Self { memattr, ..*self } + } + + const fn as_pte(&self) -> u64 { + (self.ap as u64) << UAT_AP_SHIFT + | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT + | (self.memattr as u64) << UAT_MEMATTR_SHIFT + | UAT_AF + } + + pub(crate) const fn is_cached_noncoherent(&self) -> bool { + self.ap != AP_GPU && self.memattr == MEMATTR_CACHED + } + + pub(crate) const fn as_uncached(&self) -> Self { + self.memattr(MEMATTR_UNCACHED) + } +} + +impl Default for Prot { + fn default() -> Self { + PROT_FW_GPU_NA + } +} + +pub(crate) struct UatPageTable { + ttb: PhysicalAddr, + ttb_owned: bool, + va_range: Range, + oas_mask: u64, +} + +impl UatPageTable { + pub(crate) fn new(oas: usize) -> Result { + mod_pr_debug!("UATPageTable::new: oas={}\n", oas); + let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let ttb = Page::into_phys(ttb_page); + Ok(UatPageTable { + ttb, + ttb_owned: true, + va_range: 0..(1u64 << UAT_IAS), + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn new_with_ttb( + ttb: PhysicalAddr, + va_range: Range, + oas: usize, + ) -> Result { + mod_pr_debug!( + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + ttb, + va_range, + oas + ); + if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + Ok(UatPageTable { + ttb, + ttb_owned: false, + va_range, + oas_mask: (1 << oas) - 1, + }) + } + + pub(crate) fn ttb(&self) -> PhysicalAddr { + self.ttb + } + + fn with_pages(&mut self, iova_range: Range, free: bool, mut cb: F) -> Result + where + F: FnMut(u64, &[Pte]), + { + mod_pr_debug!("UATPageTable::with_pages: {:#x?} {}\n", iova_range, free); + if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { + pr_err!( + "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", + iova_range + ); + return Err(EINVAL); + } + + if iova_range.is_empty() { + return Ok(()); + } + + let mut iova = iova_range.start & UAT_IASMSK; + let mut last_iova = iova; + let end = ((iova_range.end - 1) & UAT_IASMSK) + 1; + + let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + pt_addr[UAT_LEVELS - 1] = Some(self.ttb); + + 'outer: while iova < end { + mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); + for level in (0..UAT_LEVELS - 1).rev() { + let last_upidx = ((last_iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) + & UAT_LVMSK) as usize; + let upidx = + ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; + // If the index has changed, invalidate the physaddr + if upidx != last_upidx { + if let Some(phys) = pt_addr[level] { + if free { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); + } + pt_addr[level] = None; + } + // Fetch the page table base address for this level + if pt_addr[level].is_none() { + let phys = pt_addr[level + 1].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", + level, + phys + ); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { + Page::borrow_phys(&phys).ok_or(ENOMEM).map_err(|a| { + pr_err!( + "UATPageTable::with_pages: Failed to borrow physical address {:#x}", + phys, + ); + a + })? + }; + mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && !free { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + upte.store(upte_val, Ordering::Relaxed); + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + })?; + mod_pr_debug!( + "UATPageTable::with_pages: level {} PT {:#x?}\n", + level, + pt_addr[level] + ); + } + // If we don't have a page table, skip this entire level + if pt_addr[level].is_none() { + let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); + last_iova = iova; + iova = align(iova + 1, block); + mod_pr_debug!( + "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", + block, + last_iova, + iova + ); + continue 'outer; + } + } + + let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; + let max_count = UAT_NPTE - idx; + let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); + let phys = pt_addr[0].unwrap(); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + mod_pr_debug!( + "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", + phys, + idx, + count, + iova + ); + let pt = unsafe { + Page::borrow_phys(&phys).ok_or(ENOMEM).map_err(|a| { + pr_err!( + "UATPageTable::with_pages: Failed to borrow physical address {:#x}", + phys, + ); + a + })? + }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; + // SAFETY: We know this is a valid pointer to PTEs and the range is valid and + // checked by with_pointer_into_page(). + let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; + cb(iova, ptes); + Ok(()) + })?; + + let block = 1 << (UAT_PGBIT + UAT_LVBIT); + last_iova = iova; + iova = align(iova + 1, block); + } + + if free { + for level in (0..UAT_LEVELS - 1).rev() { + if let Some(phys) = pt_addr[level] { + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + unsafe { Page::from_phys(phys) }; + } + } + } + + Ok(()) + } + + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, false, |_, _| {}) + } + + pub(crate) fn map_pages( + &mut self, + iova_range: Range, + mut phys: PhysicalAddr, + prot: Prot, + ) -> Result { + mod_pr_debug!( + "UATPageTable::map_pages: {:#x?} {:#x?} {:?}\n", + iova_range, + phys, + prot + ); + if phys & (UAT_PGMSK as PhysicalAddr) != 0 { + pr_err!("UATPageTable::map_pages: phys not aligned: {:#x?}\n", phys); + return Err(EINVAL); + } + + self.with_pages(iova_range, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev != 0 { + pr_err!( + "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + } + pte.store( + phys | prot.as_pte() | PTE_TYPE_LEAF_TABLE, + Ordering::Relaxed, + ); + phys += UAT_PGSZ as PhysicalAddr; + } + }) + } + + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { + mod_pr_debug!( + "UATPageTable::reprot_pages: {:#x?} {:?}\n", + iova_range, + prot + ); + self.with_pages(iova_range, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + pr_err!( + "UATPageTable::reprot_pages: Page at IOVA {:#x} is unmapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + continue; + } + pte.store((ptev & !UAT_PROT_BITS) | prot.as_pte(), Ordering::Relaxed); + } + }) + } + + pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { + pr_err!( + "UATPageTable::unmap_pages: Page at IOVA {:#x} already unmapped\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + pte.store(0, Ordering::Relaxed); + } + }) + } +} + +impl Drop for UatPageTable { + fn drop(&mut self) { + mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); + if self + .with_pages(self.va_range.clone(), true, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) != 0 { + pr_err!( + "UATPageTable::drop: Leaked page at IOVA {:#x}\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + } + }) + .is_err() + { + pr_err!("UATPageTable::drop failed to free page tables\n",); + } + if self.ttb_owned { + // SAFETY: If we own the ttb, it was allocated with Page::into_phys(). + mod_pr_debug!("UATPageTable::drop: Free TTB {:#x}\n", self.ttb); + unsafe { + Page::from_phys(self.ttb); + } + } + } +}