diff --git a/src/arch/aarch64/interrupt/exception.rs b/src/arch/aarch64/interrupt/exception.rs index dda39e751f..d6ddba7464 100644 --- a/src/arch/aarch64/interrupt/exception.rs +++ b/src/arch/aarch64/interrupt/exception.rs @@ -1,8 +1,7 @@ -use core::arch::asm; +use rmm::VirtualAddress; +use crate::memory::{ArchIntCtx, GenericPfFlags}; use crate::{ - context, - cpu_id, interrupt::stack_trace, syscall, syscall::flag::*, @@ -11,6 +10,8 @@ use crate::{ exception_stack, }; +use super::InterruptStack; + exception_stack!(synchronous_exception_at_el1_with_sp0, |stack| { println!("Synchronous exception at EL1 with SP0"); stack.dump(); @@ -25,52 +26,71 @@ fn iss(esr: usize) -> u32 { (esr & 0x01ff_ffff) as u32 } +unsafe fn far_el1() -> usize { + let ret: usize; + core::arch::asm!("mrs {}, far_el1", out(reg) ret); + ret +} + +unsafe fn instr_data_abort_inner(stack: &mut InterruptStack, from_user: bool, instr_not_data: bool) -> bool { + let iss = iss(stack.iret.esr_el1); + let fsc = iss & 0x3F; + + let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111; + //let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111; + let write_not_read_if_data = iss & (1 << 6) != 0; + + let mut flags = GenericPfFlags::empty(); + flags.set(GenericPfFlags::PRESENT, !was_translation_fault); + + // TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64 + // doesn't appear to require that flag to be set if the read alone would trigger a fault. + flags.set(GenericPfFlags::INVOLVED_WRITE, write_not_read_if_data && !instr_not_data); + flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data); + flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user); + + let faulting_addr = VirtualAddress::new(far_el1()); + + crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok() +} + exception_stack!(synchronous_exception_at_el1_with_spx, |stack| { - if exception_code(stack.iret.esr_el1) == 0b100101 { - // "Data Abort taken without a change in Exception level" - - let iss = iss(stack.iret.esr_el1); - - let was_translation_fault = iss >= 0b000100 && iss <= 0b000111; - let was_permission_fault = iss >= 0b001101 && iss <= 0b001111; - - extern "C" { - static __usercopy_start: u8; - static __usercopy_end: u8; - } - let usercopy = (&__usercopy_start as *const _ as usize)..(&__usercopy_end as *const _ as usize); - - if (was_translation_fault || was_permission_fault) && usercopy.contains(&{stack.iret.elr_el1}) { - // This was a usercopy page fault. Set the return value to nonzero to indicate usercopy - // failure (EFAULT), and emulate the return instruction by setting the return pointer - // to the saved LR value. - - stack.iret.elr_el1 = stack.preserved.x30; - stack.scratch.x0 = 1; - - return; - } + if !pf_inner(stack, exception_code(stack.iret.esr_el1)) { + println!("Synchronous exception at EL1 with SPx"); + stack.dump(); + stack_trace(); + loop {} } - - println!("Synchronous exception at EL1 with SPx"); - stack.dump(); - stack_trace(); - loop {} }); +unsafe fn pf_inner(stack: &mut InterruptStack, ty: u8) -> bool { + match ty { + // "Data Abort taken from a lower Exception level" + 0b100100 => instr_data_abort_inner(stack, true, false), + // "Data Abort taken without a change in Exception level" + 0b100101 => instr_data_abort_inner(stack, false, false), + // "Instruction Abort taken from a lower Exception level" + 0b100000 => instr_data_abort_inner(stack, true, true), + // "Instruction Abort taken without a change in Exception level" + 0b100001 => instr_data_abort_inner(stack, false, true), + + _ => return false, + } +} exception_stack!(synchronous_exception_at_el0, |stack| { - with_exception_stack!(|stack| { - if exception_code(stack.iret.esr_el1) != 0b010101 { + match stack.iret.esr_el1 { + 0b010101 => with_exception_stack!(|stack| { + let scratch = &stack.scratch; + syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, stack) + }), + + ty => if !pf_inner(stack, ty as u8) { println!("FATAL: Not an SVC induced synchronous exception"); stack.dump(); stack_trace(); crate::ksignal(SIGSEGV); - stack.scratch.x0 - } else { - let scratch = &stack.scratch; - syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, stack) } - }); + } }); exception_stack!(unhandled_exception, |stack| { @@ -79,3 +99,16 @@ exception_stack!(unhandled_exception, |stack| { stack_trace(); loop {} }); + +impl ArchIntCtx for InterruptStack { + fn ip(&self) -> usize { + self.iret.elr_el1 + } + fn recover_and_efault(&mut self) { + // Set the return value to nonzero to indicate usercopy failure (EFAULT), and emulate the + // return instruction by setting the return pointer to the saved LR value. + + self.iret.elr_el1 = self.preserved.x30; + self.scratch.x0 = 1; + } +} diff --git a/src/arch/aarch64/interrupt/syscall.rs b/src/arch/aarch64/interrupt/syscall.rs index 948cd5d458..311a2ae881 100644 --- a/src/arch/aarch64/interrupt/syscall.rs +++ b/src/arch/aarch64/interrupt/syscall.rs @@ -61,9 +61,3 @@ macro_rules! with_exception_stack { (*$stack).scratch.x0 = $code; }} } - -function!(clone_ret => { - "ldp x29, x30, [sp], #16\n", - "mov sp, x29\n", - "ret\n", -}); diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 32504507cf..e7dd3609d9 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -62,3 +62,9 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - ret ", options(noreturn)); } +pub unsafe fn bootstrap_mem(bootstrap: &crate::Bootstrap) -> &'static [u8] { + use ::rmm::Arch; + use crate::memory::PAGE_SIZE; + + core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) +} diff --git a/src/arch/aarch64/paging/mod.rs b/src/arch/aarch64/paging/mod.rs index f3c5e7ba62..8dfe8f6695 100644 --- a/src/arch/aarch64/paging/mod.rs +++ b/src/arch/aarch64/paging/mod.rs @@ -215,6 +215,9 @@ impl Page { number: self.number + n, } } + pub fn offset_from(self, other: Self) -> usize { + self.number - other.number + } } pub struct PageIter { diff --git a/src/arch/aarch64/rmm.rs b/src/arch/aarch64/rmm.rs index 5af88af91f..dc24c38f9a 100644 --- a/src/arch/aarch64/rmm.rs +++ b/src/arch/aarch64/rmm.rs @@ -2,7 +2,7 @@ use core::{ cmp, mem, slice, - sync::atomic::{self, AtomicUsize, Ordering}, + sync::atomic::{self, AtomicUsize, Ordering}, cell::SyncUnsafeCell, }; use rmm::{ KILOBYTE, @@ -251,10 +251,19 @@ impl core::fmt::Debug for LockedAllocator { } } -static mut AREAS: [MemoryArea; 512] = [MemoryArea { +static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new([MemoryArea { base: PhysicalAddress::new(0), size: 0, -}; 512]; +}; 512]); +static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); + +// TODO: Share code +pub fn areas() -> &'static [MemoryArea] { + // SAFETY: Both AREAS and AREA_COUNT are initialized once and then never changed. + // + // TODO: Memory hotplug? + unsafe { &(&*AREAS.get())[..AREA_COUNT.get().read().into()] } +} pub static FRAME_ALLOCATOR: LockedAllocator = LockedAllocator; @@ -368,6 +377,8 @@ pub unsafe fn init( areas_size / mem::size_of::() ); + let areas = &mut *AREAS.get(); + // Copy memory map from bootloader location, and page align it let mut area_i = 0; for bootloader_area in bootloader_areas.iter() { @@ -443,13 +454,14 @@ pub unsafe fn init( continue; } - AREAS[area_i].base = PhysicalAddress::new(base); - AREAS[area_i].size = size; + areas[area_i].base = PhysicalAddress::new(base); + areas[area_i].size = size; area_i += 1; } + AREA_COUNT.get().write(area_i as u16); let allocator = inner::( - &AREAS, + areas, kernel_base, kernel_size_aligned, stack_base, stack_size_aligned, env_base, env_size_aligned, diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index cd127f16e7..f76a7e3c43 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -171,6 +171,8 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { // Initialize all of the non-core devices not otherwise needed to complete initialization device::init_noncore(); + crate::memory::init_mm(); + // Stop graphical debug #[cfg(feature = "graphical_debug")] graphical_debug::fini(); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5fbb64eab0..704ca0eac0 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -84,6 +84,7 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - } pub use arch_copy_to_user as arch_copy_from_user; +// TODO: This doesn't need to be arch-specific, right? pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] { core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) } diff --git a/src/context/memory.rs b/src/context/memory.rs index 82088019c8..0c9d82c955 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -1447,7 +1447,7 @@ fn cow(old_frame: Frame, old_info: &PageInfo, initial_ref_kind: RefKind) -> Resu pub fn init_frame(init_rc: RefCount) -> Result { let new_frame = crate::memory::allocate_frames(1).ok_or(PfError::Oom)?; - let page_info = get_page_info(new_frame).expect("all allocated frames need an associated page info"); + let page_info = get_page_info(new_frame).unwrap_or_else(|| panic!("all allocated frames need an associated page info, {:?} didn't", new_frame)); assert_eq!(page_info.refcount(), RefCount::Zero); page_info.refcount.store(init_rc.to_raw(), Ordering::Relaxed); @@ -1466,6 +1466,10 @@ fn map_zeroed(mapper: &mut PageMapper, page: Page, page_flags: PageFlags, pub unsafe fn copy_frame_to_frame_directly(dst: Frame, src: Frame) { // Optimized exact-page-size copy function? + + // TODO: For new frames, when the kernel's linear phys=>virt mappings are 4k, this is almost + // guaranteed to cause either one (or two) TLB misses. + let dst = unsafe { RmmA::phys_to_virt(dst.start_address()).data() as *mut u8 }; let src = unsafe { RmmA::phys_to_virt(src.start_address()).data() as *const u8 }; diff --git a/src/debugger.rs b/src/debugger.rs index c425f30f3e..c8b6b22020 100644 --- a/src/debugger.rs +++ b/src/debugger.rs @@ -32,12 +32,11 @@ pub unsafe fn debugger(target_id: Option) { let space = space.read(); if ! space.grants.is_empty() { println!("grants:"); - for grant in space.grants.iter() { - let region = grant.region(); + for (base, grant) in space.grants.iter() { println!( - " virt 0x{:016x}:0x{:016x} size 0x{:08x} {}", - region.start_address().data(), region.final_address().data(), region.size(), - if grant.is_owned() { "owned" } else { "borrowed" }, + " virt 0x{:016x}:0x{:016x} size 0x{:08x} {:?}", + base.start_address().data(), base.next_by(grant.page_count()).start_address().data(), grant.page_count() * PAGE_SIZE, grant.provider, + ); } } diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs index c728a4f231..d86d267046 100644 --- a/src/scheme/memory.rs +++ b/src/scheme/memory.rs @@ -9,7 +9,9 @@ use crate::memory::{free_frames, used_frames, PAGE_SIZE, Frame}; use crate::context::memory::{AddrSpace, Grant, PageSpan, handle_notify_files}; use crate::paging::VirtualAddress; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use crate::paging::entry::EntryFlags; + use crate::syscall::data::{Map, StatVfs}; use crate::syscall::flag::MapFlags; use crate::syscall::error::*; diff --git a/src/syscall/driver.rs b/src/syscall/driver.rs index 4121a1b903..852dfed08e 100644 --- a/src/syscall/driver.rs +++ b/src/syscall/driver.rs @@ -6,8 +6,6 @@ use crate::scheme::memory::{MemoryScheme, MemoryType}; use crate::syscall::error::{Error, EFAULT, EINVAL, ENOMEM, EPERM, ESRCH, Result}; use crate::syscall::flag::{MapFlags, PhysallocFlags, PartialAllocStrategy, PhysmapFlags}; -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - use alloc::sync::Arc; use alloc::vec::Vec;