mm: Add active_mm and rework mm's

mm_address_space now has two decoupled refcounts, as with Linux.
active_mm is used to delay TLB flushing when switching to kernel
threads. They keep a ref (mm_count) which holds the top level
pgd (any kernel mappings) and the mm_address_space in place.
User-level mappings are wiped out when mm_users = 0. Also make vm.cpp ->
vm.c

Signed-off-by: Pedro Falcato <pedro.falcato@gmail.com>

kernel/arch/riscv64/mmu.cpp

@@ -451,11 +451,12 @@ void riscv_invalidate_tlb(void *context)
     auto addr = info->addr;
     auto pages = info->pages;
     auto addr_space = info->mm;
+    struct mm_address_space *mm;
 
     auto curr_thread = get_current_thread();
+    mm = curr_thread->active_mm ?: curr_thread->aspace;
 
-    if (is_higher_half(addr) ||
-        (curr_thread->owner && curr_thread->owner->get_aspace() == addr_space))
+    if (is_higher_half(addr) || mm == addr_space)
     {
         paging_invalidate((void *) addr, pages);
         add_per_cpu(tlb_nr_invals, 1);

kernel/arch/riscv64/stubs.cpp

@@ -6,6 +6,7 @@
  * SPDX-License-Identifier: GPL-2.0-only
  */
 
+#include <onyx/arch.h>
 #include <onyx/irq.h>
 #include <onyx/ktrace.h>
 #include <onyx/panic.h>

kernel/arch/riscv64/syscall_table.json

@@ -180,7 +180,8 @@
                 "offset"
             ]
         ],
-        "return_type": "void *"
+        "return_type": "void *",
+        "abi": "c"
     },
     {
         "name": "munmap",
@@ -196,7 +197,8 @@
                 "length"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "mprotect",
@@ -216,7 +218,8 @@
                 "prot"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "mount",
@@ -277,7 +280,8 @@
                 "addr"
             ]
         ],
-        "return_type": "uint64_t"
+        "return_type": "uint64_t",
+        "abi": "c"
     },
     {
         "name": "kill",
@@ -608,7 +612,8 @@
                 "new_address"
             ]
         ],
-        "return_type": "void *"
+        "return_type": "void *",
+        "abi": "c"
     },
     {
         "name": "sigreturn",
@@ -847,7 +852,8 @@
                 "memstat"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "chdir",
@@ -2594,7 +2600,8 @@
                 "flags"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "statfs",

kernel/arch/x86_64/code_patch.cpp

@@ -12,6 +12,8 @@
 #include <onyx/smp.h>
 #include <onyx/x86/control_regs.h>
 
+#include <onyx/atomic.hpp>
+
 namespace code_patch
 {
 

kernel/arch/x86_64/isr.cpp

@@ -225,7 +225,7 @@ void stack_segment_fault(struct registers *ctx)
 #ifdef CONFIG_VERBOSE_SEGV
 #undef REQUIRES_SHARED
 #define REQUIRES_SHARED(...)
-vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length)
+extern "C" vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length)
     REQUIRES_SHARED(mm->vm_lock);
 
 #define dumpprint printk

kernel/arch/x86_64/mmu.cpp

@@ -679,10 +679,13 @@ void x86_invalidate_tlb(void *context)
     auto addr = info->addr;
     auto pages = info->pages;
     auto addr_space = info->mm;
-
+    struct mm_address_space *mm = NULL;
     auto curr_thread = get_current_thread();
 
-    if (is_higher_half(addr) || (curr_thread->owner && curr_thread->get_aspace() == addr_space))
+    if (curr_thread)
+        mm = curr_thread->active_mm ?: curr_thread->aspace;
+
+    if (is_higher_half(addr) || mm == addr_space)
     {
         paging_invalidate((void *) addr, pages);
         add_per_cpu(tlb_nr_invals, 1);

kernel/arch/x86_64/syscall_table.json

@@ -180,7 +180,8 @@
                 "offset"
             ]
         ],
-        "return_type": "void *"
+        "return_type": "void *",
+        "abi": "c"
     },
     {
         "name": "munmap",
@@ -196,7 +197,8 @@
                 "length"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "mprotect",
@@ -216,7 +218,8 @@
                 "prot"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "mount",
@@ -277,7 +280,8 @@
                 "addr"
             ]
         ],
-        "return_type": "uint64_t"
+        "return_type": "uint64_t",
+        "abi": "c"
     },
     {
         "name": "kill",
@@ -608,7 +612,8 @@
                 "new_address"
             ]
         ],
-        "return_type": "void *"
+        "return_type": "void *",
+        "abi": "c"
     },
     {
         "name": "sigreturn",
@@ -863,7 +868,8 @@
                 "memstat"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "chdir",
@@ -2638,7 +2644,8 @@
                 "flags"
             ]
         ],
-        "return_type": "int"
+        "return_type": "int",
+        "abi": "c"
     },
     {
         "name": "statfs",

kernel/drivers/firmware/efi/efi.cpp

@@ -9,6 +9,7 @@
 
 #include <stdio.h>
 
+#include <onyx/err.h>
 #include <onyx/mm/slab.h>
 #include <onyx/process.h>
 #include <onyx/utf8.h>
@@ -17,7 +18,7 @@
 
 #include <efi/efi.h>
 
-static ref_guard<mm_address_space> efi_aspace;
+static mm_address_space *efi_aspace;
 
 namespace efi::internal
 {
@@ -37,7 +38,7 @@ class efi_guard
 public:
     efi_guard()
     {
-        old_aspace = vm_set_aspace(efi_aspace.get());
+        old_aspace = vm_set_aspace(efi_aspace);
     }
 
     EFI_SYSTEM_TABLE *system_table() const
@@ -96,8 +97,8 @@ void efi_remap_efi_region(EFI_MEMORY_DESCRIPTOR &desc)
 
     auto flags = efi_memory_desc_flags_to_vm(desc.Attribute);
     auto ptr =
-        __map_pages_to_vaddr(efi_aspace.get(), (void *) desc.PhysicalStart,
-                             (void *) desc.PhysicalStart, desc.NumberOfPages << PAGE_SHIFT, flags);
+        __map_pages_to_vaddr(efi_aspace, (void *) desc.PhysicalStart, (void *) desc.PhysicalStart,
+                             desc.NumberOfPages << PAGE_SHIFT, flags);
     desc.VirtualStart = desc.PhysicalStart;
     if (!ptr && !mapping_over_null)
         panic("Failed to map EFI region [%016lx, %016lx] attributes %016lx\n", desc.PhysicalStart,
@@ -115,7 +116,7 @@ void efi_remap_efi_region(EFI_MEMORY_DESCRIPTOR &desc)
  */
 void efi_quirk_map_zero_region()
 {
-    __map_pages_to_vaddr(efi_aspace.get(), nullptr, nullptr, PAGE_SIZE, VM_READ | VM_NOFLUSH);
+    __map_pages_to_vaddr(efi_aspace, nullptr, nullptr, PAGE_SIZE, VM_READ | VM_NOFLUSH);
 }
 
 /**
@@ -140,7 +141,7 @@ static void efi_print_info()
 
 static void efi_dump_mem_desc(const EFI_MEMORY_DESCRIPTOR *desc)
 {
-    pr_debug(
+    pr_warn(
         "Descriptor type %u physical start %lx virtual start %lx nr_pages %lx attributes %08lx\n",
         desc->Type, desc->PhysicalStart, desc->VirtualStart, desc->NumberOfPages, desc->Attribute);
 }
@@ -198,7 +199,8 @@ void efi_init(EFI_SYSTEM_TABLE *system_table, EFI_MEMORY_DESCRIPTOR *descriptors
     efi_set_enabled();
 
     efi::internal::system_table = system_table;
-    efi_aspace = mm_address_space::create().unwrap();
+    efi_aspace = mm_create();
+    CHECK(!IS_ERR(efi_aspace));
     // EFI firmware out in the wild frequently "accidentally" touch NULL,
     // such that we need to map NULL
     efi_quirk_map_zero_region();

kernel/drivers/net/rtl8168/rtl8168.cpp

@@ -15,6 +15,7 @@
 #include <onyx/driver.h>
 #include <onyx/net/ethernet.h>
 #include <onyx/net/netif.h>
+#include <onyx/refcount.h>
 #include <onyx/wait_queue.h>
 
 #include <pci/pci.h>

kernel/include/onyx/arch.h

@@ -11,7 +11,9 @@
 #include <onyx/process.h>
 #include <onyx/thread.h>
 
+__BEGIN_CDECLS
 void arch_vm_init(void);
+__END_CDECLS
 
 #ifdef __cplusplus
 

kernel/include/onyx/bootmem.h

@@ -12,13 +12,18 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <onyx/compiler.h>
+
 #define BOOTMEM_FLAG_LOW_MEM (1 << 0)
 
+__BEGIN_CDECLS
+
 void *alloc_boot_page(size_t nr_pgs, long flags);
 
 void bootmem_reserve(unsigned long start, size_t size);
 void bootmem_add_range(unsigned long start, size_t size);
 
 void for_every_phys_region(void (*callback)(unsigned long start, size_t size));
+__END_CDECLS
 
 #endif

kernel/include/onyx/exec.h

@@ -10,8 +10,13 @@
 
 struct exec_state
 {
-    ref_guard<mm_address_space> new_address_space{};
+    mm_address_space *new_address_space{};
     bool flushed{false};
+    ~exec_state()
+    {
+        if (new_address_space)
+            mmput(new_address_space);
+    }
 };
 
 int exec_state_create(struct exec_state *state);

kernel/include/onyx/mm/shmem.h

@@ -11,8 +11,11 @@
 
 #include <stddef.h>
 
+#include <onyx/compiler.h>
+
 struct file;
 
+__BEGIN_CDECLS
 /**
  * @brief Create a new shmem file
  *
@@ -21,4 +24,6 @@ struct file;
  */
 struct file *anon_get_shmem(size_t len);
 
+__END_CDECLS
+
 #endif