WIP: aarch64

src/arch/aarch64/interrupt/exception.rs

@@ -1,8 +1,7 @@
-use core::arch::asm;
+use rmm::VirtualAddress;
 
+use crate::memory::{ArchIntCtx, GenericPfFlags};
 use crate::{
-    context,
-    cpu_id,
     interrupt::stack_trace,
     syscall,
     syscall::flag::*,
@@ -11,6 +10,8 @@ use crate::{
     exception_stack,
 };
 
+use super::InterruptStack;
+
 exception_stack!(synchronous_exception_at_el1_with_sp0, |stack| {
     println!("Synchronous exception at EL1 with SP0");
     stack.dump();
@@ -25,52 +26,71 @@ fn iss(esr: usize) -> u32 {
     (esr & 0x01ff_ffff) as u32
 }
 
+unsafe fn far_el1() -> usize {
+    let ret: usize;
+    core::arch::asm!("mrs {}, far_el1", out(reg) ret);
+    ret
+}
+
+unsafe fn instr_data_abort_inner(stack: &mut InterruptStack, from_user: bool, instr_not_data: bool) -> bool {
+    let iss = iss(stack.iret.esr_el1);
+    let fsc = iss & 0x3F;
+
+    let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111;
+    //let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111;
+    let write_not_read_if_data = iss & (1 << 6) != 0;
+
+    let mut flags = GenericPfFlags::empty();
+    flags.set(GenericPfFlags::PRESENT, !was_translation_fault);
+
+    // TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64
+    // doesn't appear to require that flag to be set if the read alone would trigger a fault.
+    flags.set(GenericPfFlags::INVOLVED_WRITE, write_not_read_if_data && !instr_not_data);
+    flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data);
+    flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user);
+
+    let faulting_addr = VirtualAddress::new(far_el1());
+
+    crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok()
+}
+
 exception_stack!(synchronous_exception_at_el1_with_spx, |stack| {
-    if exception_code(stack.iret.esr_el1) == 0b100101 {
-        // "Data Abort taken without a change in Exception level"
-
-        let iss = iss(stack.iret.esr_el1);
-
-        let was_translation_fault = iss >= 0b000100 && iss <= 0b000111;
-        let was_permission_fault = iss >= 0b001101 && iss <= 0b001111;
-
-        extern "C" {
-            static __usercopy_start: u8;
-            static __usercopy_end: u8;
-        }
-        let usercopy = (&__usercopy_start as *const _ as usize)..(&__usercopy_end as *const _ as usize);
-
-        if (was_translation_fault || was_permission_fault) && usercopy.contains(&{stack.iret.elr_el1}) {
-            // This was a usercopy page fault. Set the return value to nonzero to indicate usercopy
-            // failure (EFAULT), and emulate the return instruction by setting the return pointer
-            // to the saved LR value.
-
-            stack.iret.elr_el1 = stack.preserved.x30;
-            stack.scratch.x0 = 1;
-
-            return;
-        }
+    if !pf_inner(stack, exception_code(stack.iret.esr_el1)) {
+        println!("Synchronous exception at EL1 with SPx");
+        stack.dump();
+        stack_trace();
+        loop {}
     }
-
-    println!("Synchronous exception at EL1 with SPx");
-    stack.dump();
-    stack_trace();
-    loop {}
 });
+unsafe fn pf_inner(stack: &mut InterruptStack, ty: u8) -> bool {
+    match ty {
+        // "Data Abort taken from a lower Exception level"
+        0b100100 => instr_data_abort_inner(stack, true, false),
+        // "Data Abort taken without a change in Exception level"
+        0b100101 => instr_data_abort_inner(stack, false, false),
+        // "Instruction Abort taken from a lower Exception level"
+        0b100000 => instr_data_abort_inner(stack, true, true),
+        // "Instruction Abort taken without a change in Exception level"
+        0b100001 => instr_data_abort_inner(stack, false, true),
+
+        _ => return false,
+    }
+}
 
 exception_stack!(synchronous_exception_at_el0, |stack| {
-    with_exception_stack!(|stack| {
-        if exception_code(stack.iret.esr_el1) != 0b010101 {
+    match stack.iret.esr_el1 {
+        0b010101 => with_exception_stack!(|stack| {
+            let scratch = &stack.scratch;
+            syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, stack)
+        }),
+
+        ty => if !pf_inner(stack, ty as u8) {
             println!("FATAL: Not an SVC induced synchronous exception");
             stack.dump();
             stack_trace();
             crate::ksignal(SIGSEGV);
-            stack.scratch.x0
-        } else {
-            let scratch = &stack.scratch;
-            syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, stack)
         }
-    });
+    }
 });
 
 exception_stack!(unhandled_exception, |stack| {
@@ -79,3 +99,16 @@ exception_stack!(unhandled_exception, |stack| {
     stack_trace();
     loop {}
 });
+
+impl ArchIntCtx for InterruptStack {
+    fn ip(&self) -> usize {
+        self.iret.elr_el1
+    }
+    fn recover_and_efault(&mut self) {
+        // Set the return value to nonzero to indicate usercopy failure (EFAULT), and emulate the
+        // return instruction by setting the return pointer to the saved LR value.
+
+        self.iret.elr_el1 = self.preserved.x30;
+        self.scratch.x0 = 1;
+    }
+}

src/arch/aarch64/interrupt/syscall.rs

@@ -61,9 +61,3 @@ macro_rules! with_exception_stack {
             (*$stack).scratch.x0 = $code;
     }}
 }
-
-function!(clone_ret => {
-    "ldp x29, x30, [sp], #16\n",
-    "mov sp, x29\n",
-    "ret\n",
-});

src/arch/aarch64/mod.rs

@@ -62,3 +62,9 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
         ret
     ", options(noreturn));
 }
+pub unsafe fn bootstrap_mem(bootstrap: &crate::Bootstrap) -> &'static [u8] {
+    use ::rmm::Arch;
+    use crate::memory::PAGE_SIZE;
+
+    core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE)
+}

src/arch/aarch64/paging/mod.rs

@@ -215,6 +215,9 @@ impl Page {
             number: self.number + n,
         }
     }
+    pub fn offset_from(self, other: Self) -> usize {
+        self.number - other.number
+    }
 }
 
 pub struct PageIter {

src/arch/aarch64/rmm.rs

@@ -2,7 +2,7 @@ use core::{
     cmp,
     mem,
     slice,
-    sync::atomic::{self, AtomicUsize, Ordering},
+    sync::atomic::{self, AtomicUsize, Ordering}, cell::SyncUnsafeCell,
 };
 use rmm::{
     KILOBYTE,
@@ -251,10 +251,19 @@ impl core::fmt::Debug for LockedAllocator {
     }
 }
 
-static mut AREAS: [MemoryArea; 512] = [MemoryArea {
+static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new([MemoryArea {
     base: PhysicalAddress::new(0),
     size: 0,
-}; 512];
+}; 512]);
+static AREA_COUNT: SyncUnsafeCell<u16> = SyncUnsafeCell::new(0);
+
+// TODO: Share code
+pub fn areas() -> &'static [MemoryArea] {
+    // SAFETY: Both AREAS and AREA_COUNT are initialized once and then never changed.
+    //
+    // TODO: Memory hotplug?
+    unsafe { &(&*AREAS.get())[..AREA_COUNT.get().read().into()] }
+}
 
 pub static FRAME_ALLOCATOR: LockedAllocator = LockedAllocator;
 
@@ -368,6 +377,8 @@ pub unsafe fn init(
         areas_size / mem::size_of::<BootloaderMemoryEntry>()
     );
 
+    let areas = &mut *AREAS.get();
+
     // Copy memory map from bootloader location, and page align it
     let mut area_i = 0;
     for bootloader_area in bootloader_areas.iter() {
@@ -443,13 +454,14 @@ pub unsafe fn init(
             continue;
         }
 
-        AREAS[area_i].base = PhysicalAddress::new(base);
-        AREAS[area_i].size = size;
+        areas[area_i].base = PhysicalAddress::new(base);
+        areas[area_i].size = size;
         area_i += 1;
     }
+    AREA_COUNT.get().write(area_i as u16);
 
     let allocator = inner::<A>(
-        &AREAS,
+        areas,
         kernel_base, kernel_size_aligned,
         stack_base, stack_size_aligned,
         env_base, env_size_aligned,

src/arch/aarch64/start.rs

@@ -171,6 +171,8 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! {
         // Initialize all of the non-core devices not otherwise needed to complete initialization
         device::init_noncore();
 
+        crate::memory::init_mm();
+
         // Stop graphical debug
         #[cfg(feature = "graphical_debug")]
         graphical_debug::fini();

src/arch/x86_64/mod.rs

@@ -84,6 +84,7 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
 }
 pub use arch_copy_to_user as arch_copy_from_user;
 
+// TODO: This doesn't need to be arch-specific, right?
 pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] {
     core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE)
 }

src/context/memory.rs

@@ -1447,7 +1447,7 @@ fn cow(old_frame: Frame, old_info: &PageInfo, initial_ref_kind: RefKind) -> Resu
 
 pub fn init_frame(init_rc: RefCount) -> Result<Frame, PfError> {
     let new_frame = crate::memory::allocate_frames(1).ok_or(PfError::Oom)?;
-    let page_info = get_page_info(new_frame).expect("all allocated frames need an associated page info");
+    let page_info = get_page_info(new_frame).unwrap_or_else(|| panic!("all allocated frames need an associated page info, {:?} didn't", new_frame));
     assert_eq!(page_info.refcount(), RefCount::Zero);
     page_info.refcount.store(init_rc.to_raw(), Ordering::Relaxed);
 
@@ -1466,6 +1466,10 @@ fn map_zeroed(mapper: &mut PageMapper, page: Page, page_flags: PageFlags<RmmA>,
 
 pub unsafe fn copy_frame_to_frame_directly(dst: Frame, src: Frame) {
     // Optimized exact-page-size copy function?
+
+    // TODO: For new frames, when the kernel's linear phys=>virt mappings are 4k, this is almost
+    // guaranteed to cause either one (or two) TLB misses.
+
     let dst = unsafe { RmmA::phys_to_virt(dst.start_address()).data() as *mut u8 };
     let src = unsafe { RmmA::phys_to_virt(src.start_address()).data() as *const u8 };
 

src/debugger.rs

@@ -32,12 +32,11 @@ pub unsafe fn debugger(target_id: Option<crate::context::ContextId>) {
                 let space = space.read();
                 if ! space.grants.is_empty() {
                     println!("grants:");
-                    for grant in space.grants.iter() {
-                        let region = grant.region();
+                    for (base, grant) in space.grants.iter() {
                         println!(
-                            "    virt 0x{:016x}:0x{:016x} size 0x{:08x} {}",
-                            region.start_address().data(), region.final_address().data(), region.size(),
-                            if grant.is_owned() { "owned" } else { "borrowed" },
+                            "    virt 0x{:016x}:0x{:016x} size 0x{:08x} {:?}",
+                            base.start_address().data(), base.next_by(grant.page_count()).start_address().data(), grant.page_count() * PAGE_SIZE, grant.provider,
+
                         );
                     }
                 }

src/scheme/memory.rs

@@ -9,7 +9,9 @@ use crate::memory::{free_frames, used_frames, PAGE_SIZE, Frame};
 use crate::context::memory::{AddrSpace, Grant, PageSpan, handle_notify_files};
 use crate::paging::VirtualAddress;
 
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use crate::paging::entry::EntryFlags;
+
 use crate::syscall::data::{Map, StatVfs};
 use crate::syscall::flag::MapFlags;
 use crate::syscall::error::*;

src/syscall/driver.rs

@@ -6,8 +6,6 @@ use crate::scheme::memory::{MemoryScheme, MemoryType};
 use crate::syscall::error::{Error, EFAULT, EINVAL, ENOMEM, EPERM, ESRCH, Result};
 use crate::syscall::flag::{MapFlags, PhysallocFlags, PartialAllocStrategy, PhysmapFlags};
 
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-
 use alloc::sync::Arc;
 use alloc::vec::Vec;