diff --git a/src/arch/x86/interrupt/exception.rs b/src/arch/x86/interrupt/exception.rs index 6331314f74..d0e8f437f5 100644 --- a/src/arch/x86/interrupt/exception.rs +++ b/src/arch/x86/interrupt/exception.rs @@ -1,6 +1,8 @@ use rmm::TableKind; use x86::irq::PageFaultError; +use crate::memory::GenericPfFlags; +use crate::paging::VirtualAddress; use crate::{ interrupt::stack_trace, ptrace, @@ -133,60 +135,23 @@ interrupt_error!(protection, |stack| { ksignal(SIGSEGV); }); -#[naked] -unsafe extern "C" fn usercopy_trampoline() { - core::arch::asm!(" - mov eax, 1 - - pop esi - pop edi - - ret 4 - ", options(noreturn)); -} - interrupt_error!(page, |stack| { - // TODO: Share code with x86_64? - let cr2 = unsafe { x86::controlregs::cr2() }; - let flags = PageFaultError::from_bits_truncate(stack.code as u32); + let cr2 = VirtualAddress::new(unsafe { x86::controlregs::cr2() }); + let arch_flags = PageFaultError::from_bits_truncate(stack.code as u32); + let mut generic_flags = GenericPfFlags::empty(); - extern "C" { - static __usercopy_start: u8; - static __usercopy_end: u8; + generic_flags.set(GenericPfFlags::PRESENT, arch_flags.contains(PageFaultError::P)); + generic_flags.set(GenericPfFlags::INVOLVED_WRITE, arch_flags.contains(PageFaultError::WR)); + generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, arch_flags.contains(PageFaultError::US)); + generic_flags.set(GenericPfFlags::INVL, arch_flags.contains(PageFaultError::RSVD)); + generic_flags.set(GenericPfFlags::INSTR_NOT_DATA, arch_flags.contains(PageFaultError::ID)); + + if crate::memory::page_fault_handler(&mut stack.inner, generic_flags, cr2).is_err() { + println!("Page fault: {:>08X} {:#?}", cr2.data(), arch_flags); + stack.dump(); + stack_trace(); + ksignal(SIGSEGV); } - let usercopy_region = (&__usercopy_start as *const u8 as usize)..(&__usercopy_end as *const u8 as usize); - - // TODO: Most likely not necessary, but maybe also check that cr2 is not too close to USER_END. - let address_is_user = rmm::VirtualAddress::new(cr2).kind() == TableKind::User; - - let invalid_page_tables = flags.contains(PageFaultError::RSVD); - let caused_by_user = flags.contains(PageFaultError::US); - let caused_by_instr_fetch = flags.contains(PageFaultError::ID); - - if address_is_user && !caused_by_user && !caused_by_instr_fetch && !invalid_page_tables && usercopy_region.contains(&{ stack.inner.iret.eip }) { - // Unlike on x86_64, Protected Mode interrupts will not save/restore esp and ss unless - // privilege rings changed, which they won't here as we are catching a kernel-induced page - // fault. - // - // Thus, it is only possible to change scratch/preserved registers, and EIP. While it may - // be feasible to set ECX to zero to stop the REP MOVSB, or increase EIP by 2 (REP MOVSB is - // f3 a4, i.e. 2 bytes), this trampoline allows any memcpy implementation, that reasonably - // pushes preserved registers to the stack. - stack.inner.iret.eip = usercopy_trampoline as usize; - - return; - } - - println!("Page fault: {:>016X}", cr2); - println!(" Present: {}", flags.contains(PageFaultError::P)); - println!(" Write: {}", flags.contains(PageFaultError::WR)); - println!(" User: {}", flags.contains(PageFaultError::US)); - println!(" Reserved write: {}", flags.contains(PageFaultError::RSVD)); - println!(" Instruction fetch: {}", flags.contains(PageFaultError::ID)); - - stack.dump(); - stack_trace(); - ksignal(SIGSEGV); }); interrupt_stack!(fpu_fault, |stack| { diff --git a/src/arch/x86/interrupt/handler.rs b/src/arch/x86/interrupt/handler.rs index 07710653d8..0c7f807dd6 100644 --- a/src/arch/x86/interrupt/handler.rs +++ b/src/arch/x86/interrupt/handler.rs @@ -1,5 +1,6 @@ use core::mem; +use crate::memory::ArchIntCtx; use crate::syscall::IntRegisters; use super::super::flags::*; @@ -404,3 +405,31 @@ macro_rules! interrupt_error { } }; } +#[naked] +unsafe extern "C" fn usercopy_trampoline() { + core::arch::asm!(" + mov eax, 1 + + pop esi + pop edi + + ret 4 + ", options(noreturn)); +} + +impl ArchIntCtx for InterruptStack { + fn ip(&self) -> usize { + self.iret.eip + } + fn recover_and_efault(&mut self) { + // Unlike on x86_64, Protected Mode interrupts will not save/restore esp and ss unless + // privilege rings changed, which they won't here as we are catching a kernel-induced page + // fault. + // + // Thus, it is only possible to change scratch/preserved registers, and EIP. While it may + // be feasible to set ECX to zero to stop the REP MOVSB, or increase EIP by 2 (REP MOVSB is + // f3 a4, i.e. 2 bytes), this trampoline allows any memcpy implementation, that reasonably + // pushes preserved registers to the stack. + self.iret.eip = usercopy_trampoline as usize; + } +} diff --git a/src/arch/x86/mod.rs b/src/arch/x86/mod.rs index 9d68c43ec2..471e9d7a99 100644 --- a/src/arch/x86/mod.rs +++ b/src/arch/x86/mod.rs @@ -43,6 +43,8 @@ pub mod stop; pub mod time; pub use ::rmm::X86Arch as CurrentRmmArch; +use ::rmm::Arch; +use crate::{Bootstrap, memory::PAGE_SIZE}; // Flags pub mod flags { @@ -81,3 +83,6 @@ pub unsafe extern "fastcall" fn arch_copy_to_user_inner(len: usize, dst: usize, ret 4 ", options(noreturn)); } +pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] { + core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) +} diff --git a/src/arch/x86/paging/mod.rs b/src/arch/x86/paging/mod.rs index ea3562b6de..97a66d20e3 100644 --- a/src/arch/x86/paging/mod.rs +++ b/src/arch/x86/paging/mod.rs @@ -116,6 +116,9 @@ impl Page { number: self.number + n, } } + pub fn offset_from(self, other: Self) -> usize { + self.number - other.number + } } pub struct PageIter { diff --git a/src/arch/x86/rmm.rs b/src/arch/x86/rmm.rs index 4c635aaeac..92f8394cf9 100644 --- a/src/arch/x86/rmm.rs +++ b/src/arch/x86/rmm.rs @@ -1,4 +1,5 @@ use core::{ + cell::SyncUnsafeCell, cmp, mem, slice, @@ -249,10 +250,16 @@ impl core::fmt::Debug for LockedAllocator { } } -static mut AREAS: [MemoryArea; 512] = [MemoryArea { +static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new([MemoryArea { base: PhysicalAddress::new(0), size: 0, -}; 512]; +}; 512]); +static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); + +pub fn areas() -> &'static [MemoryArea] { + // SAFETY: Both areas and AREA_COUNT are initialized once and then never changed. + unsafe { &(&*AREAS.get())[..AREA_COUNT.get().read().into()] } +} pub static FRAME_ALLOCATOR: LockedAllocator = LockedAllocator; @@ -361,6 +368,8 @@ pub unsafe fn init( let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; let initfs_end = initfs_base + initfs_size_aligned; + let areas = &mut *AREAS.get(); + let bootloader_areas = slice::from_raw_parts( areas_base as *const BootloaderMemoryEntry, areas_size / mem::size_of::() @@ -452,15 +461,15 @@ pub unsafe fn init( // Combine areas that overlap for other_i in 0..area_i { - let other = &AREAS[other_i]; + let other = &areas[other_i]; let other_base = other.base.data(); let other_end = other_base + other.size; if base < other_end && base + size > other_base { let new_base = cmp::min(base, other_base); let new_size = cmp::max(base + size, other_end).checked_sub(new_base).unwrap_or(0); log::warn!("{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", base, size, other_base, other.size, new_base, new_size); - AREAS[other_i].base = PhysicalAddress::new(new_base); - AREAS[other_i].size = new_size; + areas[other_i].base = PhysicalAddress::new(new_base); + areas[other_i].size = new_size; size = 0; // Skip area } } @@ -470,13 +479,14 @@ pub unsafe fn init( continue; } - AREAS[area_i].base = PhysicalAddress::new(base); - AREAS[area_i].size = size; + areas[area_i].base = PhysicalAddress::new(base); + areas[area_i].size = size; area_i += 1; } + AREA_COUNT.get().write(area_i as u16); let allocator = inner::( - &AREAS, + areas, kernel_base, kernel_size_aligned, stack_base, stack_size_aligned, env_base, env_size_aligned, diff --git a/src/arch/x86/start.rs b/src/arch/x86/start.rs index 8a72ae9f9f..94170d16eb 100644 --- a/src/arch/x86/start.rs +++ b/src/arch/x86/start.rs @@ -18,6 +18,7 @@ use crate::gdt; use crate::idt; use crate::interrupt; use crate::log::{self, info}; +use crate::memory; use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKind}; /// Test of zero values in BSS. @@ -185,6 +186,9 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Initialize all of the non-core devices not otherwise needed to complete initialization device::init_noncore(); + // Initialize data structures used to track pages. + memory::init_mm(); + // Stop graphical debug #[cfg(feature = "graphical_debug")] graphical_debug::fini(); diff --git a/src/debugger.rs b/src/debugger.rs index ad7f7c0d2a..c425f30f3e 100644 --- a/src/debugger.rs +++ b/src/debugger.rs @@ -107,12 +107,11 @@ pub unsafe fn debugger(target_id: Option) { let addr_space = addr_space.read(); if ! addr_space.grants.is_empty() { println!("grants:"); - for grant in addr_space.grants.iter() { - let region = grant.region(); + for (base, grant) in addr_space.grants.iter() { println!( - " virt 0x{:08x}:0x{:08x} size 0x{:08x} {}", - region.start_address().data(), region.final_address().data(), region.size(), - if grant.is_owned() { "owned" } else { "borrowed" }, + " virt 0x{:08x}:0x{:08x} size 0x{:08x} {:?}", + base.start_address().data(), base.next_by(grant.page_count()).start_address().data() + 0xFFF, grant.page_count() * crate::memory::PAGE_SIZE, + grant.provider, ); } } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 7b0e6eb4a1..41195a3619 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -8,7 +8,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use crate::arch::rmm::LockedAllocator; use crate::common::try_box_slice_new; use crate::context; -use crate::context::memory::{init_frame, AccessMode, try_correcting_page_tables, PfError}; +use crate::context::memory::{init_frame, AccessMode, PfError}; use crate::kernel_executable_offsets::{__usercopy_start, __usercopy_end}; use crate::paging::Page; pub use crate::paging::{PAGE_SIZE, PhysicalAddress}; diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 5bedfc0dfd..d40e3e9fe8 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -501,7 +501,10 @@ impl UserInner { d: map.flags.bits(), // The uid and gid can be obtained by the proc scheme anyway, if the pid is provided. uid: map.offset as u32, + #[cfg(target_pointer_width = "64")] gid: (map.offset >> 32) as u32, + #[cfg(target_pointer_width = "32")] + gid: 0, })?; let mapping_is_lazy = map.flags.contains(MapFlags::MAP_LAZY); @@ -840,7 +843,13 @@ impl KernelScheme for UserScheme { let res = inner.call_extended(CallerCtx { pid: context::context_id().into(), uid: offset as u32, + #[cfg(target_pointer_width = "64")] gid: (offset >> 32) as u32, + + // TODO: saturating_shr? + #[cfg(not(target_pointer_width = "64"))] + gid: 0, + }, [KSMSG_MUNMAP, number, size, flags.bits()])?; match res {