diff --git a/arch/arm64/include/arch.h b/arch/arm64/include/arch.h index 281d7cf504fbe..14190d4262241 100644 --- a/arch/arm64/include/arch.h +++ b/arch/arm64/include/arch.h @@ -45,9 +45,9 @@ # error Only pages sizes of 4096 are currently supported (CONFIG_ARCH_ADDRENV) #endif -/* All implementations have 3 levels of page tables */ +/* All implementations have 4 levels of page tables */ -#define ARCH_PGT_MAX_LEVELS (3) +#define ARCH_PGT_MAX_LEVELS (4) #define ARCH_SPGTS (ARCH_PGT_MAX_LEVELS - 1) #endif /* CONFIG_ARCH_ADDRENV */ diff --git a/arch/arm64/src/common/arm64_addrenv.c b/arch/arm64/src/common/arm64_addrenv.c index 1f88929a9be88..e71ec2c244221 100644 --- a/arch/arm64/src/common/arm64_addrenv.c +++ b/arch/arm64/src/common/arm64_addrenv.c @@ -116,24 +116,28 @@ static void map_spgtables(arch_addrenv_t *addrenv, uintptr_t vaddr) { int i; uintptr_t prev; + uintptr_t l0; - /* Start from L1, and connect until max level - 1 */ + /* Get the base page table level and the page table associated with it */ - prev = arm64_pgvaddr(addrenv->spgtables[0]); + l0 = mmu_get_base_pgt_level(); + prev = arm64_pgvaddr(addrenv->spgtables[l0]); - /* Check if the mapping already exists */ + /* Start from the base level, and connect until max level - 1 */ - if (mmu_ln_getentry(1, prev, vaddr) != 0) + for (i = l0; i < (ARCH_SPGTS - 1); i++) { - return; - } + uintptr_t next = addrenv->spgtables[i + 1]; - /* No mapping yet, create it */ + /* Check if the mapping already exists */ + + if (mmu_ln_getentry(i, prev, vaddr) == 0) + { + /* No mapping yet, create it */ + + mmu_ln_setentry(i, prev, next, vaddr, MMU_UPGT_FLAGS); + } - for (i = 0; i < (ARCH_SPGTS - 1); i++) - { - uintptr_t next = addrenv->spgtables[i + 1]; - mmu_ln_setentry(i + 1, prev, next, vaddr, MMU_UPGT_FLAGS); prev = arm64_pgvaddr(next); } } @@ -158,7 +162,7 @@ static int create_spgtables(arch_addrenv_t *addrenv) int i; uintptr_t paddr; - for (i = 0; i < ARCH_SPGTS; i++) + for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++) { paddr = mm_pgalloc(1); if (!paddr) @@ -198,20 +202,37 @@ static int create_spgtables(arch_addrenv_t *addrenv) static int copy_kernel_mappings(arch_addrenv_t *addrenv) { - uintptr_t user_mappings = arm64_pgvaddr(addrenv->spgtables[0]); + uintptr_t kpgt; + uintptr_t upgt; + uintptr_t l0; - /* Copy the L1 references */ + /* Determine the base page table level */ - if (user_mappings == 0) + l0 = mmu_get_base_pgt_level(); + kpgt = g_kernel_mappings; + + /* Don't copy L0 references, as those encompass 512GB each */ + + if (l0 == 0) + { + upgt = arm64_pgvaddr(addrenv->spgtables[1]); + kpgt = arm64_pgvaddr(mmu_pte_to_paddr(((uintptr_t *)kpgt)[0])); + } + else + { + upgt = arm64_pgvaddr(addrenv->spgtables[l0]); + } + + if (upgt == 0 || kpgt == 0) { return -EINVAL; } - memcpy((void *)user_mappings, (void *)g_kernel_mappings, MMU_PAGE_SIZE); + memcpy((void *)upgt, (void *)kpgt, MMU_PAGE_SIZE); /* Update with memory by flushing the cache */ - up_flush_dcache(user_mappings, user_mappings + MMU_PAGE_SIZE); + up_flush_dcache(upgt, upgt + MMU_PAGE_SIZE); return OK; } @@ -249,8 +270,8 @@ static int create_region(arch_addrenv_t *addrenv, uintptr_t vaddr, nmapped = 0; npages = MM_NPAGES(size); - ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); - ptlevel = ARCH_SPGTS; + ptlevel = MMU_PGT_LEVEL_MAX - 1; + ptprev = arm64_pgvaddr(addrenv->spgtables[ptlevel]); /* Create mappings for the lower level tables */ @@ -385,6 +406,7 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize, uintptr_t textbase; uintptr_t database; uintptr_t heapbase; + uintptr_t l0; DEBUGASSERT(addrenv); DEBUGASSERT(MM_ISALIGNED(ARCH_ADDRENV_VBASE)); @@ -492,7 +514,8 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize, /* Provide the ttbr0 value for context switch */ - addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[0], 0); + l0 = mmu_get_base_pgt_level(); + addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0); /* Synchronize data and instruction pipelines */ @@ -539,17 +562,14 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv) /* Things start from the beginning of the user virtual memory */ vaddr = ARCH_ADDRENV_VBASE; - pgsize = mmu_get_region_size(ARCH_SPGTS); + pgsize = mmu_get_region_size(MMU_PGT_LEVEL_MAX - 1); /* First destroy the allocated memory and the final level page table */ ptprev = (uintptr_t *)arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); if (ptprev) { - /* walk user space only */ - - i = (ARCH_SPGTS < 2) ? vaddr / pgsize : 0; - for (; i < ENTRIES_PER_PGT; i++, vaddr += pgsize) + for (i = 0; i < ENTRIES_PER_PGT; i++, vaddr += pgsize) { ptlast = (uintptr_t *)arm64_pgvaddr(mmu_pte_to_paddr(ptprev[i])); if (ptlast) @@ -577,7 +597,7 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv) /* Then destroy the static tables */ - for (i = 0; i < ARCH_SPGTS; i++) + for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++) { paddr = addrenv->spgtables[i]; if (paddr) diff --git a/arch/arm64/src/common/arm64_addrenv_perms.c b/arch/arm64/src/common/arm64_addrenv_perms.c index f8081d56b7a1d..441b0154d122f 100644 --- a/arch/arm64/src/common/arm64_addrenv_perms.c +++ b/arch/arm64/src/common/arm64_addrenv_perms.c @@ -70,8 +70,8 @@ static int modify_region(uintptr_t vstart, uintptr_t vend, uintptr_t setmask) for (vaddr = vstart; vaddr < vend; vaddr += MM_PGSIZE) { - for (ptlevel = 1, lnvaddr = l1vaddr; - ptlevel < MMU_PGT_LEVELS; + for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = l1vaddr; + ptlevel < MMU_PGT_LEVEL_MAX; ptlevel++) { paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr)); diff --git a/arch/arm64/src/common/arm64_addrenv_pgmap.c b/arch/arm64/src/common/arm64_addrenv_pgmap.c index 4f39be0b4dd3d..e53528b97fdbe 100644 --- a/arch/arm64/src/common/arm64_addrenv_pgmap.c +++ b/arch/arm64/src/common/arm64_addrenv_pgmap.c @@ -90,7 +90,9 @@ uintptr_t up_addrenv_find_page(arch_addrenv_t *addrenv, uintptr_t vaddr) /* Make table walk to find the page */ - for (ptlevel = 1, lnvaddr = pgdir; ptlevel < MMU_PGT_LEVELS; ptlevel++) + for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = pgdir; + ptlevel < MMU_PGT_LEVEL_MAX; + ptlevel++) { paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr)); lnvaddr = arm64_pgvaddr(paddr); @@ -189,6 +191,7 @@ int up_addrenv_kmap_init(void) struct arch_addrenv_s *addrenv; uintptr_t next; uintptr_t vaddr; + uintptr_t l0; int i; /* Populate the static page tables one by one */ @@ -196,19 +199,20 @@ int up_addrenv_kmap_init(void) addrenv = &g_kernel_addrenv; next = g_kernel_pgt_pbase; vaddr = CONFIG_ARCH_KMAP_VBASE; + l0 = mmu_get_base_pgt_level(); - for (i = 0; i < ARCH_SPGTS; i++) + for (i = l0; i < ARCH_SPGTS; i++) { /* Connect the static page tables */ uintptr_t lnvaddr = arm64_pgvaddr(next); addrenv->spgtables[i] = next; - next = mmu_pte_to_paddr(mmu_ln_getentry(i + 1, lnvaddr, vaddr)); + next = mmu_pte_to_paddr(mmu_ln_getentry(i, lnvaddr, vaddr)); } /* Set the page directory root */ - addrenv->ttbr0 = mmu_ttbr_reg(g_kernel_pgt_pbase, 0); + addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0); /* When all is set and done, flush the data caches */ diff --git a/arch/arm64/src/common/arm64_addrenv_utils.c b/arch/arm64/src/common/arm64_addrenv_utils.c index 9d9e00ef7928e..2a95765148506 100644 --- a/arch/arm64/src/common/arm64_addrenv_utils.c +++ b/arch/arm64/src/common/arm64_addrenv_utils.c @@ -66,8 +66,8 @@ uintptr_t arm64_get_pgtable(arch_addrenv_t *addrenv, uintptr_t vaddr) /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */ - ptlevel = ARCH_SPGTS; - ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + ptlevel = MMU_PGT_LEVEL_MAX - 1; + ptprev = arm64_pgvaddr(addrenv->spgtables[ptlevel]); if (!ptprev) { /* Something is very wrong */ @@ -134,7 +134,7 @@ int arm64_map_pages(arch_addrenv_t *addrenv, uintptr_t *pages, uintptr_t ptlevel; uintptr_t paddr; - ptlevel = MMU_PGT_LEVELS; + ptlevel = MMU_PGT_LEVEL_MAX; /* Add the references to pages[] into the caller's address environment */ @@ -184,7 +184,10 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr, uintptr_t ptlevel; uintptr_t paddr; - ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */ + + ptlevel = MMU_PGT_LEVEL_MAX - 1; + ptprev = arm64_pgvaddr(addrenv->spgtables[ptlevel]); if (!ptprev) { /* Something is very wrong */ @@ -192,8 +195,6 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr, return -EFAULT; } - ptlevel = ARCH_SPGTS; - /* Remove the references from the caller's address environment */ for (; npages > 0; npages--) diff --git a/arch/arm64/src/common/arm64_internal.h b/arch/arm64/src/common/arm64_internal.h index 274bfdb6f49ab..59caefc266486 100644 --- a/arch/arm64/src/common/arm64_internal.h +++ b/arch/arm64/src/common/arm64_internal.h @@ -260,6 +260,8 @@ EXTERN uint8_t g_idle_topstack[]; /* End+1 of heap */ ****************************************************************************/ void arm64_new_task(struct tcb_s *tak_new); +void arm64_jump_to_user(uint64_t entry, uint64_t x0, uint64_t x1, + uint64_t *regs) noreturn_function; /* Low level initialization provided by chip logic */ diff --git a/arch/arm64/src/common/arm64_mmu.c b/arch/arm64/src/common/arm64_mmu.c index bd1283a63789e..c9d158c96a3ab 100644 --- a/arch/arm64/src/common/arm64_mmu.c +++ b/arch/arm64/src/common/arm64_mmu.c @@ -82,7 +82,7 @@ #define XLAT_TABLE_SIZE (1U << XLAT_TABLE_SIZE_SHIFT) #define XLAT_TABLE_ENTRY_SIZE_SHIFT 3U /* Each table entry is 8 bytes */ -#define XLAT_TABLE_LEVEL_MAX MMU_PGT_LEVELS +#define XLAT_TABLE_LEVEL_MAX MMU_PGT_LEVEL_MAX #define XLAT_TABLE_ENTRIES_SHIFT \ (XLAT_TABLE_SIZE_SHIFT - XLAT_TABLE_ENTRY_SIZE_SHIFT) @@ -207,6 +207,7 @@ static const struct arm_mmu_config g_mmu_nxrt_config = static const size_t g_pgt_sizes[] = { + MMU_L0_PAGE_SIZE, MMU_L1_PAGE_SIZE, MMU_L2_PAGE_SIZE, MMU_L3_PAGE_SIZE @@ -709,7 +710,8 @@ void mmu_ln_setentry(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t paddr, uintptr_t *lntable = (uintptr_t *)lnvaddr; uint32_t index; - DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL && + ptlevel <= XLAT_TABLE_LEVEL_MAX); /* Calculate index for lntable */ @@ -735,7 +737,8 @@ uintptr_t mmu_ln_getentry(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t *lntable = (uintptr_t *)lnvaddr; uint32_t index; - DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL && + ptlevel <= XLAT_TABLE_LEVEL_MAX); index = XLAT_TABLE_VA_IDX(vaddr, ptlevel); @@ -753,7 +756,8 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr, uintptr_t *lntable = (uintptr_t *)lnvaddr; uint32_t index; - DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL && + ptlevel <= XLAT_TABLE_LEVEL_MAX); index = XLAT_TABLE_VA_IDX(vaddr, ptlevel); @@ -771,7 +775,13 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr, size_t mmu_get_region_size(uint32_t ptlevel) { - DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL && + ptlevel <= XLAT_TABLE_LEVEL_MAX); - return g_pgt_sizes[ptlevel - 1]; + return g_pgt_sizes[ptlevel]; +} + +uintptr_t mmu_get_base_pgt_level(void) +{ + return XLAT_TABLE_BASE_LEVEL; } diff --git a/arch/arm64/src/common/arm64_mmu.h b/arch/arm64/src/common/arm64_mmu.h index 80bd75301fa51..4de718311abc8 100644 --- a/arch/arm64/src/common/arm64_mmu.h +++ b/arch/arm64/src/common/arm64_mmu.h @@ -233,13 +233,15 @@ /* Amount of page table levels */ -#define MMU_PGT_LEVELS (3U) +#define MMU_PGT_LEVELS (4U) +#define MMU_PGT_LEVEL_MAX (3U) /* Levels go from 0-3 */ /* Page sizes per page table level */ -#define MMU_L1_PAGE_SIZE (0x40000000) /* 1G */ -#define MMU_L2_PAGE_SIZE (0x200000) /* 2M */ -#define MMU_L3_PAGE_SIZE (0x1000) /* 4K */ +#define MMU_L0_PAGE_SIZE (0x8000000000) /* 512G */ +#define MMU_L1_PAGE_SIZE (0x40000000) /* 1G */ +#define MMU_L2_PAGE_SIZE (0x200000) /* 2M */ +#define MMU_L3_PAGE_SIZE (0x1000) /* 4K */ /* Flags for user page tables */ @@ -620,6 +622,28 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr, size_t mmu_get_region_size(uint32_t ptlevel); +/**************************************************************************** + * Name: mmu_get_base_pgt_level + * + * Description: + * Get the base translation table level. The ARM64 MMU implementation + * optimizes the amount of translation table levels in use, based on the + * configured virtual address range (CONFIG_ARM64_VA_BITS). + * + * Table indices range from 0...3 and the lowest table indices are dropped + * as needed. If CONFIG_ARM64_VA_BITS >= 40, all 4 translation table levels + * are needed. + * + * Input Parameters: + * None. + * + * Returned Value: + * The base translation table level. + * + ****************************************************************************/ + +uintptr_t mmu_get_base_pgt_level(void); + #endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM64_SRC_COMMON_ARM64_MMU_H */ diff --git a/arch/arm64/src/common/arm64_pgalloc.c b/arch/arm64/src/common/arm64_pgalloc.c index 22115ffcd41c2..758e094e4dce2 100644 --- a/arch/arm64/src/common/arm64_pgalloc.c +++ b/arch/arm64/src/common/arm64_pgalloc.c @@ -92,6 +92,7 @@ uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages) struct tcb_s *tcb = nxsched_self(); struct arch_addrenv_s *addrenv; uintptr_t ptlast; + uintptr_t ptlevel; uintptr_t paddr; uintptr_t vaddr; @@ -113,7 +114,8 @@ uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages) /* Start mapping from the old heap break address */ - vaddr = brkaddr; + vaddr = brkaddr; + ptlevel = MMU_PGT_LEVEL_MAX; /* Sanity checks */ @@ -144,7 +146,7 @@ uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages) /* Then add the reference */ - mmu_ln_setentry(MMU_PGT_LEVELS, ptlast, paddr, vaddr, MMU_UDATA_FLAGS); + mmu_ln_setentry(ptlevel, ptlast, paddr, vaddr, MMU_UDATA_FLAGS); vaddr += MM_PGSIZE; } diff --git a/arch/arm64/src/common/arm64_pthread_start.c b/arch/arm64/src/common/arm64_pthread_start.c index db80db228b630..8132884cf8394 100644 --- a/arch/arm64/src/common/arm64_pthread_start.c +++ b/arch/arm64/src/common/arm64_pthread_start.c @@ -68,12 +68,6 @@ void up_pthread_start(pthread_trampoline_t startup, pthread_startroutine_t entrypt, pthread_addr_t arg) { - uint64_t *regs = this_task()->xcp.initregs; - - /* This must be performed atomically, the C-section ends upon user entry */ - - enter_critical_section(); - /* Set up to enter the user-space pthread start-up function in * unprivileged mode. We need: * @@ -83,24 +77,8 @@ void up_pthread_start(pthread_trampoline_t startup, * SPSR = user mode */ - regs[REG_ELR] = (uint64_t)startup; - regs[REG_X0] = (uint64_t)entrypt; - regs[REG_X1] = (uint64_t)arg; - regs[REG_SPSR] = (regs[REG_SPSR] & ~SPSR_MODE_MASK) | SPSR_MODE_EL0T; - - /* Fully unwind the kernel stack and drop to user space */ - - asm - ( - "mov x0, %0\n" /* Get context registers */ - "mov sp, x0\n" /* Stack pointer = context */ - "b arm64_exit_exception\n" - : - : "r" (regs) - : "x0", "memory" - ); - - PANIC(); + arm64_jump_to_user((uint64_t)startup, (uint64_t)entrypt, (uint64_t)arg, + this_task()->xcp.initregs); } #endif /* !CONFIG_BUILD_FLAT && __KERNEL__ && !CONFIG_DISABLE_PTHREAD */ diff --git a/arch/arm64/src/common/arm64_task_start.c b/arch/arm64/src/common/arm64_task_start.c index 5e3501eee6f52..0d20f693e5a04 100644 --- a/arch/arm64/src/common/arm64_task_start.c +++ b/arch/arm64/src/common/arm64_task_start.c @@ -65,12 +65,6 @@ void up_task_start(main_t taskentry, int argc, char *argv[]) { - uint64_t *regs = this_task()->xcp.initregs; - - /* This must be performed atomically, the C-section ends upon user entry */ - - enter_critical_section(); - /* Set up to return to the user-space _start function in * unprivileged mode. We need: * @@ -80,24 +74,8 @@ void up_task_start(main_t taskentry, int argc, char *argv[]) * SPSR = user mode */ - regs[REG_ELR] = (uint64_t)taskentry; - regs[REG_X0] = (uint64_t)argc; - regs[REG_X1] = (uint64_t)argv; - regs[REG_SPSR] = (regs[REG_SPSR] & ~SPSR_MODE_MASK) | SPSR_MODE_EL0T; - - /* Fully unwind the kernel stack and drop to user space */ - - asm - ( - "mov x0, %0\n" /* Get context registers */ - "mov sp, x0\n" /* Stack pointer = context */ - "b arm64_exit_exception\n" - : - : "r" (regs) - : "x0", "memory" - ); - - PANIC(); + arm64_jump_to_user((uint64_t)taskentry, (uint64_t)argc, (uint64_t)argv, + this_task()->xcp.initregs); } #endif /* !CONFIG_BUILD_FLAT */ diff --git a/arch/arm64/src/common/arm64_vectors.S b/arch/arm64/src/common/arm64_vectors.S index 025e4cb3372d4..9ce01a771bed9 100644 --- a/arch/arm64/src/common/arm64_vectors.S +++ b/arch/arm64/src/common/arm64_vectors.S @@ -174,6 +174,36 @@ restore_new: ret +/**************************************************************************** + * Function: arm64_jump_to_user + * + * Description: + * Routine to jump to user space, called when a user process is started and + * the kernel is ready to give control to the user task in user space. + * + * arm64_jump_to_user(entry, x0, x1, regs) + * entry: process entry point + * x0: parameter 0 for process + * x1: parameter 1 for process + * regs: integer register save area to use + * + ****************************************************************************/ + +#ifndef CONFIG_BUILD_FLAT +GTEXT(arm64_jump_to_user) +SECTION_FUNC(text, arm64_jump_to_user) + msr daifset, #IRQ_DAIF_MASK + mov sp, x3 + str x0, [sp, #8 * REG_ELR] + str x1, [sp, #8 * REG_X0] + str x2, [sp, #8 * REG_X1] + mrs x0, spsr_el1 + and x0, x0, #~SPSR_MODE_MASK + #orr x0, x0, #SPSR_MODE_EL0T # EL0T=0x00, out of range for orr + str x0, [sp, #8 * REG_SPSR] + b arm64_exit_exception +#endif + /**************************************************************************** * Function: arm64_sync_exc *