From 7c0995ee24a5b991d108e25c0439cda7ac2fdd43 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Thu, 11 Jul 2024 23:41:22 +0300 Subject: [PATCH] Revived tests Also fixed debug build failure on i686, and reformatted the code --- src/allocator/frame/buddy.rs | 37 ++++------ src/allocator/frame/bump.rs | 11 +-- src/allocator/frame/mod.rs | 10 ++- src/arch/aarch64.rs | 26 +++---- src/arch/emulate.rs | 137 +++++++++++++++++++++-------------- src/arch/mod.rs | 31 +++----- src/arch/riscv64/sv39.rs | 19 ++--- src/arch/riscv64/sv48.rs | 26 +++---- src/arch/x86.rs | 10 +-- src/arch/x86_64.rs | 13 +--- src/lib.rs | 6 +- src/main.rs | 90 ++++++++++++----------- src/page/entry.rs | 11 ++- src/page/flags.rs | 16 ++-- src/page/flush.rs | 24 +++--- src/page/mapper.rs | 102 ++++++++++++++++++-------- src/page/mod.rs | 8 +- src/page/table.rs | 22 +++--- 18 files changed, 306 insertions(+), 293 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 7aad111abf..564004303a 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -1,16 +1,7 @@ -use core::{ - marker::PhantomData, - mem, -}; +use core::{marker::PhantomData, mem}; use crate::{ - Arch, - BumpAllocator, - FrameAllocator, - FrameCount, - FrameUsage, - PhysicalAddress, - VirtualAddress, + Arch, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, PhysicalAddress, VirtualAddress, }; #[repr(transparent)] @@ -94,7 +85,7 @@ impl BuddyAllocator { // Allocate buddy table let table_phys = bump_allocator.allocate_one()?; let table_virt = A::phys_to_virt(table_phys); - for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { let virt = table_virt.add(i * mem::size_of::>()); A::write(virt, BuddyEntry::::empty()); } @@ -117,7 +108,7 @@ impl BuddyAllocator { area.size -= offset; offset = 0; } - for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { let virt = table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); let inserted = if area.base.add(area.size) == { entry.base } { @@ -147,7 +138,7 @@ impl BuddyAllocator { //TODO: sort areas? // Allocate buddy maps - for i in 0 .. Self::BUDDY_ENTRIES { + for i in 0..Self::BUDDY_ENTRIES { let virt = table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); @@ -186,13 +177,15 @@ impl FrameAllocator for BuddyAllocator { return None; } - for entry_i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(entry_i * mem::size_of::>()); + for entry_i in 0..Self::BUDDY_ENTRIES { + let virt = self + .table_virt + .add(entry_i * mem::size_of::>()); let mut entry = A::read::>(virt); let mut free_page = entry.skip; let mut free_count = 0; - for page in entry.skip .. entry.pages() { + for page in entry.skip..entry.pages() { let usage = entry.usage(page)?; if usage.0 == 0 { free_count += 1; @@ -207,7 +200,7 @@ impl FrameAllocator for BuddyAllocator { } if free_count == count.data() { - for page in free_page .. free_page + free_count { + for page in free_page..free_page + free_count { // Update usage let mut usage = entry.usage(page)?; usage.0 += 1; @@ -243,7 +236,7 @@ impl FrameAllocator for BuddyAllocator { } let size = count.data() * A::PAGE_SIZE; - for i in 0 .. Self::BUDDY_ENTRIES { + for i in 0..Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); @@ -269,7 +262,9 @@ impl FrameAllocator for BuddyAllocator { entry.used -= 1; } - entry.set_usage(page, usage).expect("failed to set usage during free"); + entry + .set_usage(page, usage) + .expect("failed to set usage during free"); } // Write updated entry @@ -283,7 +278,7 @@ impl FrameAllocator for BuddyAllocator { unsafe fn usage(&self) -> FrameUsage { let mut total = 0; let mut used = 0; - for i in 0 .. Self::BUDDY_ENTRIES { + for i in 0..Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::>()); let entry = A::read::>(virt); total += entry.size >> A::PAGE_SHIFT; diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index a911647159..51bee174ca 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -1,13 +1,6 @@ use core::marker::PhantomData; -use crate::{ - Arch, - FrameAllocator, - FrameCount, - FrameUsage, - MemoryArea, - PhysicalAddress, -}; +use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress}; pub struct BumpAllocator { areas: &'static [MemoryArea], @@ -17,7 +10,7 @@ pub struct BumpAllocator { } impl BumpAllocator { - pub fn new(areas: &'static [MemoryArea], offset: usize) -> Self { + pub fn new(areas: &'static [MemoryArea], _offset: usize) -> Self { Self { areas, offset: 0, diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index 99c9d3215c..626ff41714 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -1,7 +1,6 @@ use crate::PhysicalAddress; -pub use self::buddy::*; -pub use self::bump::*; +pub use self::{buddy::*, bump::*}; mod buddy; mod bump; @@ -30,7 +29,7 @@ impl FrameUsage { pub fn new(used: FrameCount, total: FrameCount) -> Self { Self { used, total } } - + pub fn used(&self) -> FrameCount { self.used } @@ -60,7 +59,10 @@ pub trait FrameAllocator { unsafe fn usage(&self) -> FrameUsage; } -impl FrameAllocator for &mut T where T: FrameAllocator { +impl FrameAllocator for &mut T +where + T: FrameAllocator, +{ unsafe fn allocate(&mut self, count: FrameCount) -> Option { T::allocate(self, count) } diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index d4ebf2185a..3a811a0184 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct AArch64Arch; @@ -18,12 +12,10 @@ impl Arch for AArch64Arch { //TODO const ENTRY_ADDRESS_SHIFT: usize = 52; - const ENTRY_FLAG_DEFAULT_PAGE: usize - = Self::ENTRY_FLAG_PRESENT + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Page flag | 1 << 10 // Access flag - | Self::ENTRY_FLAG_NO_GLOBAL - ; + | Self::ENTRY_FLAG_NO_GLOBAL; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Table flag @@ -58,12 +50,14 @@ impl Arch for AArch64Arch { #[inline(always)] unsafe fn invalidate_all() { - asm!(" + asm!( + " dsb ishst tlbi vmalle1is dsb ish isb - "); + " + ); } #[inline(always)] @@ -72,7 +66,7 @@ impl Arch for AArch64Arch { match table_kind { TableKind::User => { asm!("mrs {0}, ttbr0_el1", out(reg) address); - }, + } TableKind::Kernel => { asm!("mrs {0}, ttbr1_el1", out(reg) address); } @@ -85,7 +79,7 @@ impl Arch for AArch64Arch { match table_kind { TableKind::User => { asm!("msr ttbr0_el1, {0}", in(reg) address.data()); - }, + } TableKind::Kernel => { asm!("msr ttbr1_el1, {0}", in(reg) address.data()); } @@ -101,8 +95,8 @@ impl Arch for AArch64Arch { #[cfg(test)] mod tests { - use crate::Arch; use super::AArch64Arch; + use crate::Arch; #[test] fn constants() { diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index a73bb084c8..88a200769a 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -1,18 +1,9 @@ -use core::{ - marker::PhantomData, - mem, - ptr, -}; +use core::{marker::PhantomData, mem, ptr}; use std::collections::BTreeMap; use crate::{ - MEGABYTE, - Arch, - MemoryArea, - PageEntry, - PhysicalAddress, - VirtualAddress, - arch::x86_64::X8664Arch, + arch::x86_64::X8664Arch, page::PageFlags, Arch, MemoryArea, PageEntry, PhysicalAddress, + TableKind, VirtualAddress, MEGABYTE, }; #[derive(Clone, Copy)] @@ -35,6 +26,9 @@ impl Arch for EmulateArch { const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; + const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; + const ENTRY_FLAG_NO_GLOBAL: usize = X8664Arch::ENTRY_FLAG_NO_GLOBAL; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); @@ -43,7 +37,10 @@ impl Arch for EmulateArch { let pml4 = 0; let pdp = pml4 + Self::PAGE_SIZE; let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT; - machine.write_phys::(PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), pdp | flags); + machine.write_phys::( + PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), + pdp | flags, + ); // PDP link to PD let pd = pdp + Self::PAGE_SIZE; @@ -56,13 +53,16 @@ impl Arch for EmulateArch { // PT links to frames for i in 0..Self::PAGE_ENTRIES { let page = i * Self::PAGE_SIZE; - machine.write_phys::(PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), page | flags); + machine.write_phys::( + PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), + page | flags, + ); } MACHINE = Some(machine); // Set table to pml4 - EmulateArch::set_table(PhysicalAddress::new(pml4)); + EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4)); &MEMORY_AREAS } @@ -93,12 +93,12 @@ impl Arch for EmulateArch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { MACHINE.as_mut().unwrap().get_table() } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { MACHINE.as_mut().unwrap().set_table(address); } fn virt_is_valid(_address: VirtualAddress) -> bool { @@ -111,12 +111,12 @@ const MEMORY_SIZE: usize = 64 * MEGABYTE; static MEMORY_AREAS: [MemoryArea; 2] = [ MemoryArea { base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted - size: MEMORY_SIZE/2 - EmulateArch::PAGE_SIZE * 4, + size: MEMORY_SIZE / 2 - EmulateArch::PAGE_SIZE * 4, }, // Second area for debugging MemoryArea { - base: PhysicalAddress::new(MEMORY_SIZE/2), - size: MEMORY_SIZE/2, + base: PhysicalAddress::new(MEMORY_SIZE / 2), + size: MEMORY_SIZE / 2, }, ]; @@ -142,11 +142,13 @@ impl Machine { fn read_phys(&self, phys: PhysicalAddress) -> T { let size = mem::size_of::(); if phys.add(size).data() <= self.memory.len() { - unsafe { - ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) - } + unsafe { ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) } } else { - panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys.data(), size); + panic!( + "read_phys: 0x{:X} size 0x{:X} outside of memory", + phys.data(), + size + ); } } @@ -157,29 +159,34 @@ impl Machine { ptr::write(self.memory.as_mut_ptr().add(phys.data()) as *mut T, value); } } else { - panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys.data(), size); + panic!( + "write_phys: 0x{:X} size 0x{:X} outside of memory", + phys.data(), + size + ); } } fn write_phys_bytes(&mut self, phys: PhysicalAddress, value: u8, count: usize) { if phys.add(count).data() <= self.memory.len() { unsafe { - ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()) as *mut u8, value, count); + ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()), value, count); } } else { - panic!("write_phys_bytes: 0x{:X} count 0x{:X} outside of memory", phys.data(), count); + panic!( + "write_phys_bytes: 0x{:X} count 0x{:X} outside of memory", + phys.data(), + count + ); } } - fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, usize)> { + fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags)> { let virt_data = virt.data(); let page = virt_data & A::PAGE_ADDRESS_MASK; let offset = virt_data & A::PAGE_OFFSET_MASK; let entry = self.map.get(&VirtualAddress::new(page))?; - Some(( - entry.address().add(offset), - entry.flags(), - )) + Some((entry.address().ok()?.add(offset), entry.flags())) } fn read(&self, virt: VirtualAddress) -> T { @@ -187,7 +194,10 @@ impl Machine { let virt_data = virt.data(); let size = mem::size_of::(); if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt_data, size); + panic!( + "read: 0x{:X} size 0x{:X} passes page boundary", + virt_data, size + ); } if let Some((phys, _flags)) = self.translate(virt) { @@ -202,11 +212,14 @@ impl Machine { let virt_data = virt.data(); let size = mem::size_of::(); if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt_data, size); + panic!( + "write: 0x{:X} size 0x{:X} passes page boundary", + virt_data, size + ); } if let Some((phys, flags)) = self.translate(virt) { - if flags & A::ENTRY_FLAG_READWRITE != 0 { + if flags.has_write() { self.write_phys(phys, value); } else { panic!("write: 0x{:X} size 0x{:X} not writable", virt_data, size); @@ -219,18 +232,28 @@ impl Machine { fn write_bytes(&mut self, virt: VirtualAddress, value: u8, count: usize) { //TODO: allow writing past page boundaries let virt_data = virt.data(); - if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("write_bytes: 0x{:X} count 0x{:X} passes page boundary", virt_data, count); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK) + { + panic!( + "write_bytes: 0x{:X} count 0x{:X} passes page boundary", + virt_data, count + ); } if let Some((phys, flags)) = self.translate(virt) { - if flags & A::ENTRY_FLAG_READWRITE != 0 { + if flags.has_write() { self.write_phys_bytes(phys, value, count); } else { - panic!("write_bytes: 0x{:X} count 0x{:X} not writable", virt_data, count); + panic!( + "write_bytes: 0x{:X} count 0x{:X} not writable", + virt_data, count + ); } } else { - panic!("write_bytes: 0x{:X} count 0x{:X} not present", virt_data, count); + panic!( + "write_bytes: 0x{:X} count 0x{:X} not present", + virt_data, count + ); } } @@ -247,37 +270,45 @@ impl Machine { for i4 in 0..A::PAGE_ENTRIES { let e3 = self.read_phys::(PhysicalAddress::new(a4 + i4 * A::PAGE_ENTRY_SIZE)); let f3 = e3 & A::ENTRY_FLAGS_MASK; - if f3 & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f3 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page directory pointer let a3 = e3 & A::ENTRY_ADDRESS_MASK; for i3 in 0..A::PAGE_ENTRIES { - let e2 = self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); + let e2 = + self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); let f2 = e2 & A::ENTRY_FLAGS_MASK; - if f2 & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f2 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page directory let a2 = e2 & A::ENTRY_ADDRESS_MASK; for i2 in 0..A::PAGE_ENTRIES { - let e1 = self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); + let e1 = + self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); let f1 = e1 & A::ENTRY_FLAGS_MASK; - if f1 & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f1 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page table let a1 = e1 & A::ENTRY_ADDRESS_MASK; for i1 in 0..A::PAGE_ENTRIES { - let e = self.read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); + let e = self + .read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); let f = e & A::ENTRY_FLAGS_MASK; - if f & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page - let page = - (i4 << 39) | - (i3 << 30) | - (i2 << 21) | - (i1 << 12); + let page = (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12); //println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); - self.map.insert(VirtualAddress::new(page), PageEntry::new(e)); + self.map + .insert(VirtualAddress::new(page), PageEntry::new(e)); } } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f22eaae406..f6f5ec9c53 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,35 +1,25 @@ use core::ptr; -use crate::{ - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; //TODO: Support having all page tables compile on all architectures +#[cfg(all(feature = "std", target_pointer_width = "64"))] +pub use self::emulate::EmulateArch; #[cfg(target_pointer_width = "32")] -pub use self::{ - x86::X86Arch, -}; +pub use self::x86::X86Arch; #[cfg(target_pointer_width = "64")] pub use self::{ aarch64::AArch64Arch, - riscv64::{ - RiscV64Sv39Arch, - RiscV64Sv48Arch - }, + riscv64::{RiscV64Sv39Arch, RiscV64Sv48Arch}, x86_64::X8664Arch, }; -#[cfg(all(feature = "std", target_pointer_width = "64"))] -pub use self::emulate::EmulateArch; #[cfg(target_pointer_width = "64")] mod aarch64; -#[cfg(target_pointer_width = "64")] -mod riscv64; #[cfg(all(feature = "std", target_pointer_width = "64"))] mod emulate; +#[cfg(target_pointer_width = "64")] +mod riscv64; #[cfg(target_pointer_width = "32")] mod x86; #[cfg(target_pointer_width = "64")] @@ -62,11 +52,12 @@ pub trait Arch: Clone + Copy { const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; - const PAGE_NEGATIVE_MASK: usize = !((Self::PAGE_ADDRESS_SIZE as u64) - 1) as usize; + const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1) as usize; const ENTRY_ADDRESS_SIZE: u64 = 1 << Self::ENTRY_ADDRESS_SHIFT; - const ENTRY_ADDRESS_MASK: usize = (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; - const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK as usize; + const ENTRY_ADDRESS_MASK: usize = + (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; + const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; unsafe fn init() -> &'static [MemoryArea]; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index ced66efa3b..0980428754 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct RiscV64Sv39Arch; @@ -22,9 +16,7 @@ impl Arch for RiscV64Sv39Arch { = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag ; - const ENTRY_FLAG_DEFAULT_TABLE: usize - = Self::ENTRY_FLAG_PRESENT - ; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; @@ -51,14 +43,13 @@ impl Arch for RiscV64Sv39Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT // Convert from PPN + (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN ) } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - let satp = - (8 << 60) | // Sv39 MODE + let satp = (8 << 60) | // Sv39 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); } @@ -73,8 +64,8 @@ impl Arch for RiscV64Sv39Arch { #[cfg(test)] mod tests { - use crate::Arch; use super::RiscV64Sv39Arch; + use crate::Arch; #[test] fn constants() { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 1246842bfd..b1142317f0 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct RiscV64Sv48Arch; @@ -22,9 +16,7 @@ impl Arch for RiscV64Sv48Arch { = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag ; - const ENTRY_FLAG_DEFAULT_TABLE: usize - = Self::ENTRY_FLAG_PRESENT - ; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; @@ -51,14 +43,13 @@ impl Arch for RiscV64Sv48Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT // Convert from PPN + (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN ) } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - let satp = - (9 << 60) | // Sv48 MODE + let satp = (9 << 60) | // Sv48 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); } @@ -67,15 +58,14 @@ impl Arch for RiscV64Sv48Arch { let mask = 0xFFFF_8000_0000_0000; let masked = address.data() & mask; - masked == mask - || masked == 0 + masked == mask || masked == 0 } } #[cfg(test)] mod tests { - use crate::Arch; use super::RiscV64Sv48Arch; + use crate::Arch; #[test] fn constants() { @@ -104,7 +94,9 @@ mod tests { assert!(RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); } fn no(address: usize) { - assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); + assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new( + address + ))); } yes(0xFFFF_8000_1337_1337); diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 424a1332b0..8dcd238f3e 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -1,13 +1,7 @@ //TODO: USE PAE use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct X86Arch; @@ -61,8 +55,8 @@ impl Arch for X86Arch { #[cfg(test)] mod tests { - use crate::Arch; use super::{VirtualAddress, X86Arch}; + use crate::Arch; #[test] fn constants() { diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 29e8d14002..113f66f036 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy, Debug)] pub struct X8664Arch; @@ -65,15 +59,14 @@ impl VirtualAddress { pub fn is_canonical(self) -> bool { let masked = self.data() & 0xFFFF_8000_0000_0000; // TODO: 5-level paging - masked == 0xFFFF_8000_0000_0000 - || masked == 0 + masked == 0xFFFF_8000_0000_0000 || masked == 0 } } #[cfg(test)] mod tests { - use crate::Arch; use super::{VirtualAddress, X8664Arch}; + use crate::Arch; #[test] fn constants() { diff --git a/src/lib.rs b/src/lib.rs index 259c4eefbe..b0615774d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,7 @@ #![cfg_attr(not(feature = "std"), no_std)] #![feature(doc_cfg)] -pub use crate::{ - allocator::*, - arch::*, - page::*, -}; +pub use crate::{allocator::*, arch::*, page::*}; mod allocator; mod arch; diff --git a/src/main.rs b/src/main.rs index a501837db4..86f5c67b60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,24 +1,11 @@ #![cfg(target_pointer_width = "64")] use rmm::{ - KILOBYTE, - MEGABYTE, - GIGABYTE, - TERABYTE, - Arch, - BuddyAllocator, - BumpAllocator, - EmulateArch, - FrameAllocator, - FrameCount, - MemoryArea, - PageFlags, - PageFlushAll, - PageMapper, - PageTable, - PhysicalAddress, - VirtualAddress, + Arch, BuddyAllocator, BumpAllocator, EmulateArch, Flusher, FrameAllocator, FrameCount, + MemoryArea, PageFlags, PageFlushAll, PageMapper, PageTable, PhysicalAddress, TableKind, + VirtualAddress, GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE, }; +use std::marker::PhantomData; pub fn format_size(size: usize) -> String { if size >= 2 * TERABYTE { @@ -34,6 +21,7 @@ pub fn format_size(size: usize) -> String { } } +#[allow(dead_code)] unsafe fn dump_tables(table: PageTable) { let level = table.level(); for i in 0..A::PAGE_ENTRIES { @@ -41,7 +29,11 @@ unsafe fn dump_tables(table: PageTable) { if let Some(entry) = table.entry(i) { if entry.present() { let base = table.entry_base(i).unwrap(); - println!("0x{:X}: 0x{:X}", base.data(), entry.address().data()); + println!( + "0x{:X}: 0x{:X}", + base.data(), + entry.address().unwrap().data() + ); } } } else { @@ -179,18 +171,15 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { { // Map all physical areas at PHYS_OFFSET - let mut mapper = PageMapper::::create( - &mut bump_allocator - ).expect("failed to create Mapper"); + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); for area in areas.iter() { for i in 0..area.size / A::PAGE_SIZE { let phys = area.base.add(i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); - let flush = mapper.map_phys( - virt, - phys, - PageFlags::::new().write(true) - ).expect("failed to map page to frame"); + let flush = mapper + .map_phys(virt, phys, PageFlags::::new().write(true)) + .expect("failed to map page to frame"); flush.ignore(); // Not the active table } } @@ -229,35 +218,39 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { } } - let mut mapper = PageMapper::::current( - &mut allocator - ); - let flush_all = PageFlushAll::new(); + let mut mapper = PageMapper::::current(TableKind::Kernel, &mut allocator); + let mut flush_all = PageFlushAll::new(); for i in 0..16 { let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); - let flush = mapper.map( - virt, - PageFlags::::new().user(true).write(true) - ).expect("failed to map page"); + let flush = mapper + .map(virt, PageFlags::::new().user(true).write(true)) + .expect("failed to map page"); flush_all.consume(flush); } flush_all.flush(); - let flush_all = PageFlushAll::new(); + let mut flush_all = PageFlushAll::new(); for i in 0..16 { let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); - let flush = mapper.unmap( - virt, - ).expect("failed to unmap page"); + let flush = mapper.unmap(virt, false).expect("failed to unmap page"); flush_all.consume(flush); } flush_all.flush(); let usage = allocator.usage(); println!("Allocator usage:"); - println!(" Used: {}", format_size(usage.used().data() * A::PAGE_SIZE)); - println!(" Free: {}", format_size(usage.free().data() * A::PAGE_SIZE)); - println!(" Total: {}", format_size(usage.total().data() * A::PAGE_SIZE)); + println!( + " Used: {}", + format_size(usage.used().data() * A::PAGE_SIZE) + ); + println!( + " Free: {}", + format_size(usage.free().data() * A::PAGE_SIZE) + ); + println!( + " Total: {}", + format_size(usage.total().data() * A::PAGE_SIZE) + ); } unsafe fn inner() { @@ -270,19 +263,28 @@ unsafe fn inner() { //dump_tables(PageTable::::top()); - for i in &[1, 2, 4, 8, 16, 32] { let phys = PhysicalAddress::new(i * MEGABYTE); let virt = A::phys_to_virt(phys); // Test read - println!("0x{:X} (0x{:X}) = 0x{:X}", virt.data(), phys.data(), A::read::(virt)); + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); // Test write A::write::(virt, 0x5A); // Test read - println!("0x{:X} (0x{:X}) = 0x{:X}", virt.data(), phys.data(), A::read::(virt)); + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); } } diff --git a/src/page/entry.rs b/src/page/entry.rs index 800482cc2f..34a83d448f 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -1,10 +1,6 @@ use core::marker::PhantomData; -use crate::{ - Arch, - PageFlags, - PhysicalAddress, -}; +use crate::{Arch, PageFlags, PhysicalAddress}; #[derive(Clone, Copy, Debug)] pub struct PageEntry { @@ -15,7 +11,10 @@ pub struct PageEntry { impl PageEntry { #[inline(always)] pub fn new(data: usize) -> Self { - Self { data, phantom: PhantomData } + Self { + data, + phantom: PhantomData, + } } #[inline(always)] diff --git a/src/page/flags.rs b/src/page/flags.rs index b65d31344e..bffe12643e 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -14,10 +14,10 @@ impl PageFlags { unsafe { Self::from_data( // Flags set to present, kernel space, read-only, no-execute by default - A::ENTRY_FLAG_DEFAULT_PAGE | - A::ENTRY_FLAG_READONLY | - A::ENTRY_FLAG_NO_EXEC | - A::ENTRY_FLAG_NO_GLOBAL, + A::ENTRY_FLAG_DEFAULT_PAGE + | A::ENTRY_FLAG_READONLY + | A::ENTRY_FLAG_NO_EXEC + | A::ENTRY_FLAG_NO_GLOBAL, ) } } @@ -27,10 +27,10 @@ impl PageFlags { unsafe { Self::from_data( // Flags set to present, kernel space, read-only, no-execute by default - A::ENTRY_FLAG_DEFAULT_TABLE | - A::ENTRY_FLAG_READONLY | - A::ENTRY_FLAG_NO_EXEC | - A::ENTRY_FLAG_NO_GLOBAL, + A::ENTRY_FLAG_DEFAULT_TABLE + | A::ENTRY_FLAG_READONLY + | A::ENTRY_FLAG_NO_EXEC + | A::ENTRY_FLAG_NO_GLOBAL, ) } } diff --git a/src/page/flush.rs b/src/page/flush.rs index 1114c33bc8..371301b901 100644 --- a/src/page/flush.rs +++ b/src/page/flush.rs @@ -1,12 +1,6 @@ -use core::{ - marker::PhantomData, - mem, -}; +use core::{marker::PhantomData, mem}; -use crate::{ - Arch, - VirtualAddress, -}; +use crate::{Arch, VirtualAddress}; pub trait Flusher { fn consume(&mut self, flush: PageFlush); @@ -27,7 +21,9 @@ impl PageFlush { } pub fn flush(self) { - unsafe { A::invalidate(self.virt); } + unsafe { + A::invalidate(self.virt); + } } pub unsafe fn ignore(self) { @@ -41,7 +37,7 @@ pub struct PageFlushAll { phantom: PhantomData A>, } -impl PageFlushAll { +impl PageFlushAll { pub fn new() -> Self { Self { phantom: PhantomData, @@ -56,12 +52,16 @@ impl PageFlushAll { } impl Drop for PageFlushAll { fn drop(&mut self) { - unsafe { A::invalidate_all(); } + unsafe { + A::invalidate_all(); + } } } impl Flusher for PageFlushAll { fn consume(&mut self, flush: PageFlush) { - unsafe { flush.ignore(); } + unsafe { + flush.ignore(); + } } } impl + ?Sized> Flusher for &mut T { diff --git a/src/page/mapper.rs b/src/page/mapper.rs index f75c01a9af..28bea52791 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -1,14 +1,7 @@ use core::marker::PhantomData; use crate::{ - Arch, - FrameAllocator, - PageEntry, - PageFlags, - PageFlush, - PageTable, - PhysicalAddress, - TableKind, + Arch, FrameAllocator, PageEntry, PageFlags, PageFlush, PageTable, PhysicalAddress, TableKind, VirtualAddress, }; @@ -50,13 +43,7 @@ impl PageMapper { pub fn table(&self) -> PageTable { // SAFETY: The only way to initialize a PageMapper is via new(), and we assume it upholds // all necessary invariants for this to be safe. - unsafe { - PageTable::new( - VirtualAddress::new(0), - self.table_addr, - A::PAGE_LEVELS - 1 - ) - } + unsafe { PageTable::new(VirtualAddress::new(0), self.table_addr, A::PAGE_LEVELS - 1) } } pub fn allocator(&self) -> &F { @@ -67,7 +54,11 @@ impl PageMapper { &mut self.allocator } - pub unsafe fn remap_with_full(&mut self, virt: VirtualAddress, f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags)) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + pub unsafe fn remap_with_full( + &mut self, + virt: VirtualAddress, + f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags), + ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { self.visit(virt, |p1, i| { let old_entry = p1.entry(i)?; let old_phys = old_entry.address().ok()?; @@ -77,21 +68,41 @@ impl PageMapper { let new_entry = PageEntry::new(new_phys.data() | new_flags.data()); p1.set_entry(i, new_entry); Some((old_flags, old_phys, PageFlush::new(virt))) - }).flatten() + }) + .flatten() } - pub unsafe fn remap_with(&mut self, virt: VirtualAddress, map_flags: impl FnOnce(PageFlags) -> PageFlags) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { - self.remap_with_full(virt, |same_phys, old_flags| (same_phys, map_flags(old_flags))) + pub unsafe fn remap_with( + &mut self, + virt: VirtualAddress, + map_flags: impl FnOnce(PageFlags) -> PageFlags, + ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + self.remap_with_full(virt, |same_phys, old_flags| { + (same_phys, map_flags(old_flags)) + }) } - pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + pub unsafe fn remap( + &mut self, + virt: VirtualAddress, + flags: PageFlags, + ) -> Option> { self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) } - pub unsafe fn map(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + pub unsafe fn map( + &mut self, + virt: VirtualAddress, + flags: PageFlags, + ) -> Option> { let phys = self.allocator.allocate_one()?; self.map_phys(virt, phys, flags) } - pub unsafe fn map_phys(&mut self, virt: VirtualAddress, phys: PhysicalAddress, flags: PageFlags) -> Option> { + pub unsafe fn map_phys( + &mut self, + virt: VirtualAddress, + phys: PhysicalAddress, + flags: PageFlags, + ) -> Option> { //TODO: verify virt and phys are aligned //TODO: verify flags have correct bits let entry = PageEntry::new(phys.data() | flags.data()); @@ -109,7 +120,13 @@ impl PageMapper { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { A::ENTRY_FLAG_USER } else { 0 }; + let flags = A::ENTRY_FLAG_READWRITE + | A::ENTRY_FLAG_DEFAULT_TABLE + | if virt.kind() == TableKind::User { + A::ENTRY_FLAG_USER + } else { + 0 + }; table.set_entry(i, PageEntry::new(next_phys.data() | flags)); table.next(i)? } @@ -118,11 +135,19 @@ impl PageMapper { } } } - pub unsafe fn map_linearly(&mut self, phys: PhysicalAddress, flags: PageFlags) -> Option<(VirtualAddress, PageFlush)> { + pub unsafe fn map_linearly( + &mut self, + phys: PhysicalAddress, + flags: PageFlags, + ) -> Option<(VirtualAddress, PageFlush)> { let virt = A::phys_to_virt(phys); self.map_phys(virt, phys, flags).map(|flush| (virt, flush)) } - fn visit(&self, virt: VirtualAddress, f: impl FnOnce(&mut PageTable, usize) -> T) -> Option { + fn visit( + &self, + virt: VirtualAddress, + f: impl FnOnce(&mut PageTable, usize) -> T, + ) -> Option { let mut table = self.table(); unsafe { loop { @@ -140,20 +165,35 @@ impl PageMapper { Some((entry.address().ok()?, entry.flags())) } - pub unsafe fn unmap(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option> { + pub unsafe fn unmap( + &mut self, + virt: VirtualAddress, + unmap_parents: bool, + ) -> Option> { let (old, _, flush) = self.unmap_phys(virt, unmap_parents)?; self.allocator.free_one(old); Some(flush) } - pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { + pub unsafe fn unmap_phys( + &mut self, + virt: VirtualAddress, + unmap_parents: bool, + ) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { //TODO: verify virt is aligned let mut table = self.table(); let level = table.level(); - unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator) + .map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) } } -unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable, initial_level: usize, unmap_parents: bool, allocator: &mut impl FrameAllocator) -> Option<(PhysicalAddress, PageFlags)> { +unsafe fn unmap_phys_inner( + virt: VirtualAddress, + table: &mut PageTable, + initial_level: usize, + unmap_parents: bool, + allocator: &mut impl FrameAllocator, +) -> Option<(PhysicalAddress, PageFlags)> { let i = table.index_of(virt)?; if table.level() == 0 { @@ -172,7 +212,9 @@ unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable< if unmap_parents { // TODO: Use a counter? This would reduce the remaining number of available bits, but could be // faster (benchmark is needed). - let is_still_populated = (0..A::PAGE_ENTRIES).map(|j| subtable.entry(j).expect("must be within bounds")).any(|e| e.present()); + let is_still_populated = (0..A::PAGE_ENTRIES) + .map(|j| subtable.entry(j).expect("must be within bounds")) + .any(|e| e.present()); if !is_still_populated { allocator.free_one(subtable.phys()); diff --git a/src/page/mod.rs b/src/page/mod.rs index bcf360b322..c4c4f00241 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -1,10 +1,4 @@ -pub use self::{ - entry::*, - flags::*, - flush::*, - mapper::*, - table::*, -}; +pub use self::{entry::*, flags::*, flush::*, mapper::*, table::*}; mod entry; mod flags; diff --git a/src/page/table.rs b/src/page/table.rs index 3c003cb7ec..323a808c00 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -1,12 +1,7 @@ use core::marker::PhantomData; -use crate::{ - Arch, - PhysicalAddress, - TableKind, - VirtualAddress, -}; use super::PageEntry; +use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress}; pub struct PageTable { base: VirtualAddress, @@ -17,14 +12,19 @@ pub struct PageTable { impl PageTable { pub unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self { - Self { base, phys, level, phantom: PhantomData } + Self { + base, + phys, + level, + phantom: PhantomData, + } } pub unsafe fn top(table_kind: TableKind) -> Self { Self::new( VirtualAddress::new(0), A::table(table_kind), - A::PAGE_LEVELS - 1 + A::PAGE_LEVELS - 1, ) } @@ -85,7 +85,11 @@ impl PageTable { // Canonicalize address first let address = VirtualAddress::new(address.data() & A::PAGE_ADDRESS_MASK); let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; - let level_mask = (A::PAGE_ENTRIES << level_shift) - 1; + // Intentionally wraps around at last-level table to get all-ones mask on architectures + // where addressable physical address space covers entire usized space (e.g. x86) + let level_mask = A::PAGE_ENTRIES + .wrapping_shl(level_shift as u32) + .wrapping_sub(1); if address >= self.base && address <= self.base.add(level_mask) { Some((address.data() >> level_shift) & A::PAGE_ENTRY_MASK) } else {