From e7ae8e68124446fc91ab494467f059ce6a2085d2 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 15:22:35 -0600 Subject: [PATCH 001/120] init --- .gitignore | 1 + Cargo.lock | 5 + Cargo.toml | 9 ++ src/main.rs | 274 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 289 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..ea8c4bf7f3 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..e919422bb9 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "rmm" +version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..95b68f8ee7 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "rmm" +version = "0.1.0" +authors = ["Jeremy Soller "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000000..33e0b4fc37 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,274 @@ +use core::{ + mem, + ptr, +}; +use std::collections::BTreeMap; + +//TODO: should this be a constant? +pub const PAGE_SHIFT: usize = 12; +pub const PAGE_SIZE: usize = 1 << PAGE_SHIFT; +pub const PAGE_OFFSET_MASK: usize = PAGE_SIZE - 1; +pub const PAGE_ADDRESS_MASK: usize = !PAGE_OFFSET_MASK; +pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); +pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; + +// Physical memory address +#[repr(transparent)] +pub struct PhysicalAddress(usize); + +impl PhysicalAddress { + pub const fn new(address: usize) -> Self { + Self(address) + } + + pub const fn aligned(&self) -> bool { + self.0 & PAGE_OFFSET_MASK == 0 + } +} + +// Physical memory frame +#[repr(transparent)] +pub struct Frame(PhysicalAddress); + +impl Frame { + pub fn new(address: PhysicalAddress) -> Option { + if address.aligned() { + Some(Frame(address)) + } else { + None + } + } +} + +// Virtual memory address +#[repr(transparent)] +pub struct VirtualAddress(usize); + +impl VirtualAddress { + pub const fn new(address: usize) -> Self { + Self(address) + } + + pub const fn aligned(&self) -> bool { + self.0 & PAGE_OFFSET_MASK == 0 + } +} + +// Virtual memory page +#[repr(transparent)] +pub struct Page(VirtualAddress); + +impl Page { + pub fn new(address: VirtualAddress) -> Option { + if address.aligned() { + Some(Self(address)) + } else { + None + } + } +} + +const ENTRY_PRESENT: usize = 1 << 0; +const ENTRY_WRITABLE: usize = 1 << 1; +const ENTRY_ADDRESS_MASK: usize = PAGE_ADDRESS_MASK; +const ENTRY_FLAGS_MASK: usize = !ENTRY_ADDRESS_MASK; + +static mut MACHINE: Option = None; + +#[inline(always)] +pub unsafe fn machine_read_u8(address: usize) -> u8 { + MACHINE.as_ref().unwrap().read_u8(address) +} + +#[inline(always)] +pub unsafe fn machine_write_u8(address: usize, value: u8) { + MACHINE.as_mut().unwrap().write_u8(address, value) +} + +#[inline(always)] +pub unsafe fn machine_invalidate(address: usize) { + MACHINE.as_mut().unwrap().invalidate(address); +} + +#[inline(always)] +pub unsafe fn machine_invalidate_all() { + MACHINE.as_mut().unwrap().invalidate_all(); +} + +#[inline(always)] +pub unsafe fn machine_set_table(address: usize) { + MACHINE.as_mut().unwrap().set_table(address); +} + +pub struct Machine { + pub memory: Box<[u8]>, + pub map: BTreeMap, + pub table_addr: usize, +} + +impl Machine { + pub fn new(memory_size: usize) -> Self { + Self { + memory: vec![0; memory_size].into_boxed_slice(), + map: BTreeMap::new(), + table_addr: 0, + } + } + + pub fn read_phys_usize(&self, phys: usize) -> usize { + if phys + mem::size_of::() <= self.memory.len() { + unsafe { + ptr::read(self.memory.as_ptr().add(phys) as *const usize) + } + } else { + panic!("read_phys_usize {:X} outside of memory", phys); + } + } + + pub fn write_phys_usize(&mut self, phys: usize, value: usize) { + if phys + mem::size_of::() <= self.memory.len() { + unsafe { + ptr::write(self.memory.as_mut_ptr().add(phys) as *mut usize, value); + } + } else { + panic!("write_phys_usize {:X} outside of memory", phys); + } + } + + pub fn translate(&self, virt: usize) -> Option<(usize, usize)> { + let page = virt & PAGE_ADDRESS_MASK; + let offset = virt & PAGE_OFFSET_MASK; + let phys = self.map.get(&page)?; + Some(( + (phys & ENTRY_ADDRESS_MASK) + offset, + phys & ENTRY_FLAGS_MASK, + )) + } + + pub fn read_u8(&self, virt: usize) -> u8 { + if let Some((phys, _flags)) = self.translate(virt) { + self.memory[phys] + } else { + panic!("read_u8: {:X} not present", virt); + } + } + + pub fn write_u8(&mut self, virt: usize, value: u8) { + if let Some((phys, flags)) = self.translate(virt) { + if flags & ENTRY_WRITABLE != 0 { + self.memory[phys] = value; + } else { + panic!("write_u8: {:X} not writable", virt); + } + } else { + panic!("write_u8: {:X} not present", virt); + } + } + + pub fn invalidate(&mut self, _address: usize) { + unimplemented!(); + } + + pub fn invalidate_all(&mut self) { + self.map.clear(); + + // PML4 + let a4 = self.table_addr; + for i4 in 0..PAGE_ENTRIES { + let e3 = self.read_phys_usize(a4 + i4 * PAGE_ENTRY_SIZE); + let f3 = e3 & ENTRY_FLAGS_MASK; + if f3 & ENTRY_PRESENT == 0 { continue; } + + // Page directory pointer + let a3 = e3 & ENTRY_ADDRESS_MASK; + for i3 in 0..PAGE_ENTRIES { + let e2 = self.read_phys_usize(a3 + i3 * PAGE_ENTRY_SIZE); + let f2 = e2 & ENTRY_FLAGS_MASK; + if f2 & ENTRY_PRESENT == 0 { continue; } + + // Page directory + let a2 = e2 & ENTRY_ADDRESS_MASK; + for i2 in 0..PAGE_ENTRIES { + let e1 = self.read_phys_usize(a2 + i2 * PAGE_ENTRY_SIZE); + let f1 = e1 & ENTRY_FLAGS_MASK; + if f1 & ENTRY_PRESENT == 0 { continue; } + + // Page table + let a1 = e1 & ENTRY_ADDRESS_MASK; + for i1 in 0..PAGE_ENTRIES { + let e = self.read_phys_usize(a1 + i1 * PAGE_ENTRY_SIZE); + let f = e & ENTRY_FLAGS_MASK; + if f & ENTRY_PRESENT == 0 { continue; } + + // Page + let a = e & ENTRY_ADDRESS_MASK; + let page = + (i4 << 39) | + (i3 << 30) | + (i2 << 21) | + (i1 << 12); + println!("map {:X} to {:X}, {:X}", page, a, f); + self.map.insert(page, e); + } + } + } + } + } + + pub fn set_table(&mut self, address: usize) { + self.table_addr = address; + self.invalidate_all(); + } +} + +fn main() { + let memory_size = 64 * 1024 * 1024; + + unsafe { + let megabyte = 0x100000; + + // Create machine with PAGE_ENTRIES pages identity mapped (2 MiB on x86_64) + // Pages over 1 MiB will be mapped writable + { + let mut machine = Machine::new(memory_size); + + // PML4 link to PDP + let pml4 = 0; + let pdp = pml4 + PAGE_SIZE; + machine.write_phys_usize(pml4, pdp | ENTRY_PRESENT); + + // PDP link to PD + let pd = pdp + PAGE_SIZE; + machine.write_phys_usize(pdp, pd | ENTRY_PRESENT); + + // PD link to PT + let pt = pd + PAGE_SIZE; + machine.write_phys_usize(pd, pt | ENTRY_PRESENT); + + // PT links to frames + for i in 0..PAGE_ENTRIES { + let page = i * PAGE_SIZE; + let flags = if page >= megabyte { + ENTRY_WRITABLE | ENTRY_PRESENT + } else { + ENTRY_PRESENT + }; + machine.write_phys_usize(pt + i * PAGE_ENTRY_SIZE, page | flags); + } + + MACHINE = Some(machine); + + // Set table to pml4 + machine_set_table(pml4); + } + + // Test read + println!("{:X} = {:X}", megabyte, machine_read_u8(megabyte)); + + // Test write + machine_write_u8(megabyte, 0x5A); + + // Test read + println!("{:X} = {:X}", megabyte, machine_read_u8(megabyte)); + } +} From 21dc9624d72c3be9b602b7e4139374d2f99ca9fe Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 21:23:23 +0000 Subject: [PATCH 002/120] Add LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..c7a7f3c3fe --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jeremy Soller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 048298d65baf5d4df85cb35f465957c776d9bafa Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 21:24:43 +0000 Subject: [PATCH 003/120] Add README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000..f475fba520 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# Redox Memory Management + +This is a Rust crate to provide abstractions for hardware memory management. It +also contains a mechanism for testing memory management with software emulation. \ No newline at end of file From a70bc0026457b6c9f2827c5237ba7fdadf1e78a1 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 15:42:42 -0600 Subject: [PATCH 004/120] Improve machine read/write functions --- src/main.rs | 76 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/src/main.rs b/src/main.rs index 33e0b4fc37..94a9b482d0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,13 +76,13 @@ const ENTRY_FLAGS_MASK: usize = !ENTRY_ADDRESS_MASK; static mut MACHINE: Option = None; #[inline(always)] -pub unsafe fn machine_read_u8(address: usize) -> u8 { - MACHINE.as_ref().unwrap().read_u8(address) +pub unsafe fn machine_read(address: usize) -> T { + MACHINE.as_ref().unwrap().read(address) } #[inline(always)] -pub unsafe fn machine_write_u8(address: usize, value: u8) { - MACHINE.as_mut().unwrap().write_u8(address, value) +pub unsafe fn machine_write(address: usize, value: T) { + MACHINE.as_mut().unwrap().write(address, value) } #[inline(always)] @@ -115,23 +115,25 @@ impl Machine { } } - pub fn read_phys_usize(&self, phys: usize) -> usize { - if phys + mem::size_of::() <= self.memory.len() { + pub fn read_phys(&self, phys: usize) -> T { + let size = mem::size_of::(); + if phys + size <= self.memory.len() { unsafe { - ptr::read(self.memory.as_ptr().add(phys) as *const usize) + ptr::read(self.memory.as_ptr().add(phys) as *const T) } } else { - panic!("read_phys_usize {:X} outside of memory", phys); + panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); } } - pub fn write_phys_usize(&mut self, phys: usize, value: usize) { - if phys + mem::size_of::() <= self.memory.len() { + pub fn write_phys(&mut self, phys: usize, value: T) { + let size = mem::size_of::(); + if phys + size <= self.memory.len() { unsafe { - ptr::write(self.memory.as_mut_ptr().add(phys) as *mut usize, value); + ptr::write(self.memory.as_mut_ptr().add(phys) as *mut T, value); } } else { - panic!("write_phys_usize {:X} outside of memory", phys); + panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); } } @@ -145,23 +147,35 @@ impl Machine { )) } - pub fn read_u8(&self, virt: usize) -> u8 { + pub fn read(&self, virt: usize) -> T { + //TODO: allow reading past page boundaries + let size = mem::size_of::(); + if (virt & PAGE_ADDRESS_MASK) != ((virt + size) & PAGE_ADDRESS_MASK) { + panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); + } + if let Some((phys, _flags)) = self.translate(virt) { - self.memory[phys] + self.read_phys(phys) } else { - panic!("read_u8: {:X} not present", virt); + panic!("read: 0x{:X} size 0x{:X} not present", virt, size); } } - pub fn write_u8(&mut self, virt: usize, value: u8) { + pub fn write(&mut self, virt: usize, value: T) { + //TODO: allow writing past page boundaries + let size = mem::size_of::(); + if (virt & PAGE_ADDRESS_MASK) != ((virt + size) & PAGE_ADDRESS_MASK) { + panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); + } + if let Some((phys, flags)) = self.translate(virt) { if flags & ENTRY_WRITABLE != 0 { - self.memory[phys] = value; + self.write_phys(phys, value); } else { - panic!("write_u8: {:X} not writable", virt); + panic!("write: 0x{:X} size 0x{:X} not writable", virt, size); } } else { - panic!("write_u8: {:X} not present", virt); + panic!("write: 0x{:X} size 0x{:X} not present", virt, size); } } @@ -175,28 +189,28 @@ impl Machine { // PML4 let a4 = self.table_addr; for i4 in 0..PAGE_ENTRIES { - let e3 = self.read_phys_usize(a4 + i4 * PAGE_ENTRY_SIZE); + let e3 = self.read_phys::(a4 + i4 * PAGE_ENTRY_SIZE); let f3 = e3 & ENTRY_FLAGS_MASK; if f3 & ENTRY_PRESENT == 0 { continue; } // Page directory pointer let a3 = e3 & ENTRY_ADDRESS_MASK; for i3 in 0..PAGE_ENTRIES { - let e2 = self.read_phys_usize(a3 + i3 * PAGE_ENTRY_SIZE); + let e2 = self.read_phys::(a3 + i3 * PAGE_ENTRY_SIZE); let f2 = e2 & ENTRY_FLAGS_MASK; if f2 & ENTRY_PRESENT == 0 { continue; } // Page directory let a2 = e2 & ENTRY_ADDRESS_MASK; for i2 in 0..PAGE_ENTRIES { - let e1 = self.read_phys_usize(a2 + i2 * PAGE_ENTRY_SIZE); + let e1 = self.read_phys::(a2 + i2 * PAGE_ENTRY_SIZE); let f1 = e1 & ENTRY_FLAGS_MASK; if f1 & ENTRY_PRESENT == 0 { continue; } // Page table let a1 = e1 & ENTRY_ADDRESS_MASK; for i1 in 0..PAGE_ENTRIES { - let e = self.read_phys_usize(a1 + i1 * PAGE_ENTRY_SIZE); + let e = self.read_phys::(a1 + i1 * PAGE_ENTRY_SIZE); let f = e & ENTRY_FLAGS_MASK; if f & ENTRY_PRESENT == 0 { continue; } @@ -207,7 +221,7 @@ impl Machine { (i3 << 30) | (i2 << 21) | (i1 << 12); - println!("map {:X} to {:X}, {:X}", page, a, f); + println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); self.map.insert(page, e); } } @@ -235,15 +249,15 @@ fn main() { // PML4 link to PDP let pml4 = 0; let pdp = pml4 + PAGE_SIZE; - machine.write_phys_usize(pml4, pdp | ENTRY_PRESENT); + machine.write_phys::(pml4, pdp | ENTRY_PRESENT); // PDP link to PD let pd = pdp + PAGE_SIZE; - machine.write_phys_usize(pdp, pd | ENTRY_PRESENT); + machine.write_phys::(pdp, pd | ENTRY_PRESENT); // PD link to PT let pt = pd + PAGE_SIZE; - machine.write_phys_usize(pd, pt | ENTRY_PRESENT); + machine.write_phys::(pd, pt | ENTRY_PRESENT); // PT links to frames for i in 0..PAGE_ENTRIES { @@ -253,7 +267,7 @@ fn main() { } else { ENTRY_PRESENT }; - machine.write_phys_usize(pt + i * PAGE_ENTRY_SIZE, page | flags); + machine.write_phys::(pt + i * PAGE_ENTRY_SIZE, page | flags); } MACHINE = Some(machine); @@ -263,12 +277,12 @@ fn main() { } // Test read - println!("{:X} = {:X}", megabyte, machine_read_u8(megabyte)); + println!("0x{:X} = 0x{:X}", megabyte, machine_read::(megabyte)); // Test write - machine_write_u8(megabyte, 0x5A); + machine_write::(megabyte, 0x5A); // Test read - println!("{:X} = {:X}", megabyte, machine_read_u8(megabyte)); + println!("0x{:X} = 0x{:X}", megabyte, machine_read::(megabyte)); } } From a55e19868eddbee4cac2f0e0d97444524bff88be Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 20:56:48 -0600 Subject: [PATCH 005/120] Add abstraction for page table --- src/main.rs | 156 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 16 deletions(-) diff --git a/src/main.rs b/src/main.rs index 94a9b482d0..02ae2a3840 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,27 +76,32 @@ const ENTRY_FLAGS_MASK: usize = !ENTRY_ADDRESS_MASK; static mut MACHINE: Option = None; #[inline(always)] -pub unsafe fn machine_read(address: usize) -> T { +pub unsafe fn arch_read(address: usize) -> T { MACHINE.as_ref().unwrap().read(address) } #[inline(always)] -pub unsafe fn machine_write(address: usize, value: T) { +pub unsafe fn arch_write(address: usize, value: T) { MACHINE.as_mut().unwrap().write(address, value) } #[inline(always)] -pub unsafe fn machine_invalidate(address: usize) { +pub unsafe fn arch_invalidate(address: usize) { MACHINE.as_mut().unwrap().invalidate(address); } #[inline(always)] -pub unsafe fn machine_invalidate_all() { +pub unsafe fn arch_invalidate_all() { MACHINE.as_mut().unwrap().invalidate_all(); } #[inline(always)] -pub unsafe fn machine_set_table(address: usize) { +pub unsafe fn arch_get_table() -> usize { + MACHINE.as_mut().unwrap().get_table() +} + +#[inline(always)] +pub unsafe fn arch_set_table(address: usize) { MACHINE.as_mut().unwrap().set_table(address); } @@ -150,7 +155,7 @@ impl Machine { pub fn read(&self, virt: usize) -> T { //TODO: allow reading past page boundaries let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + size) & PAGE_ADDRESS_MASK) { + if (virt & PAGE_ADDRESS_MASK) != ((virt + size - 1) & PAGE_ADDRESS_MASK) { panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); } @@ -164,7 +169,7 @@ impl Machine { pub fn write(&mut self, virt: usize, value: T) { //TODO: allow writing past page boundaries let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + size) & PAGE_ADDRESS_MASK) { + if (virt & PAGE_ADDRESS_MASK) != ((virt + size - 1) & PAGE_ADDRESS_MASK) { panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); } @@ -229,12 +234,126 @@ impl Machine { } } + pub fn get_table(&self) -> usize { + self.table_addr + } + pub fn set_table(&mut self, address: usize) { self.table_addr = address; self.invalidate_all(); } } +#[repr(transparent)] +pub struct PageEntry(usize); + +impl PageEntry { + pub fn address(&self) -> PhysicalAddress { + PhysicalAddress(self.0 & ENTRY_ADDRESS_MASK) + } + + pub fn present(&self) -> bool { + self.0 & ENTRY_PRESENT != 0 + } +} + +pub struct PageTable { + base: VirtualAddress, + phys: PhysicalAddress, + level: usize +} + +impl PageTable { + pub unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self { + Self { base, phys, level } + } + + pub unsafe fn top() -> Self { + Self::new( + VirtualAddress::new(0), + PhysicalAddress::new(arch_get_table()), + 3 + ) + } + + pub fn level(&self) -> usize { + self.level + } + + pub unsafe fn virt(&self) -> VirtualAddress { + //TODO: something other than identity mapping + VirtualAddress(self.phys.0) + } + + pub fn entry_base(&self, i: usize) -> Option { + if i < PAGE_ENTRIES { + Some(VirtualAddress( + self.base.0 + i << (9 * self.level + PAGE_SHIFT) + )) + } else { + None + } + } + + pub unsafe fn entry_virt(&self, i: usize) -> Option { + if i < PAGE_ENTRIES { + Some(VirtualAddress( + self.virt().0 + i * PAGE_ENTRY_SIZE + )) + } else { + None + } + } + + pub unsafe fn entry(&self, i: usize) -> Option { + let addr = self.entry_virt(i)?; + Some(PageEntry(arch_read::(addr.0))) + } + + pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { + let addr = self.entry_virt(i)?; + arch_write::(addr.0, entry.0); + Some(()) + } + + pub unsafe fn next(&self, i: usize) -> Option { + if self.level > 0 { + let entry = self.entry(i)?; + if entry.present() { + return Some(PageTable::new( + self.entry_base(i)?, + entry.address(), + self.level - 1 + )); + } + } + None + } +} + +unsafe fn dump_tables(table: PageTable) { + let level = table.level(); + for i in 0..PAGE_ENTRIES { + if level == 0 { + if let Some(entry) = table.entry(i) { + if entry.present() { + let base = table.entry_base(i).unwrap(); + println!("0x{:X}: 0x{:X}", base.0, entry.address().0); + } + } + } else { + if let Some(next) = table.next(i) { + dump_tables(next); + } + } + } +} + +pub struct MemoryArea { + base: PhysicalAddress, + size: usize, +} + fn main() { let memory_size = 64 * 1024 * 1024; @@ -262,27 +381,32 @@ fn main() { // PT links to frames for i in 0..PAGE_ENTRIES { let page = i * PAGE_SIZE; - let flags = if page >= megabyte { - ENTRY_WRITABLE | ENTRY_PRESENT - } else { - ENTRY_PRESENT - }; + let flags = ENTRY_WRITABLE | ENTRY_PRESENT; machine.write_phys::(pt + i * PAGE_ENTRY_SIZE, page | flags); } MACHINE = Some(machine); // Set table to pml4 - machine_set_table(pml4); + arch_set_table(pml4); } + // Debug table + dump_tables(PageTable::top()); + // Test read - println!("0x{:X} = 0x{:X}", megabyte, machine_read::(megabyte)); + println!("0x{:X} = 0x{:X}", megabyte, arch_read::(megabyte)); // Test write - machine_write::(megabyte, 0x5A); + arch_write::(megabyte, 0x5A); // Test read - println!("0x{:X} = 0x{:X}", megabyte, machine_read::(megabyte)); + println!("0x{:X} = 0x{:X}", megabyte, arch_read::(megabyte)); + + // Initialize memory allocator + let areas = [MemoryArea { + base: PhysicalAddress::new(megabyte), + size: megabyte, + }]; } } From adad6bbe65513787d6581b3c91cb1e27a962e457 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 21:07:31 -0600 Subject: [PATCH 006/120] Add recursive mapping --- src/main.rs | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 02ae2a3840..2c2abedb2f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; // Physical memory address +#[derive(Clone, Copy, Debug)] #[repr(transparent)] pub struct PhysicalAddress(usize); @@ -27,6 +28,7 @@ impl PhysicalAddress { } // Physical memory frame +#[derive(Clone, Copy, Debug)] #[repr(transparent)] pub struct Frame(PhysicalAddress); @@ -41,6 +43,7 @@ impl Frame { } // Virtual memory address +#[derive(Clone, Copy, Debug)] #[repr(transparent)] pub struct VirtualAddress(usize); @@ -55,6 +58,7 @@ impl VirtualAddress { } // Virtual memory page +#[derive(Clone, Copy, Debug)] #[repr(transparent)] pub struct Page(VirtualAddress); @@ -244,6 +248,7 @@ impl Machine { } } +#[derive(Clone, Copy, Debug)] #[repr(transparent)] pub struct PageEntry(usize); @@ -276,6 +281,14 @@ impl PageTable { ) } + pub fn base(&self) -> VirtualAddress { + self.base + } + + pub fn phys(&self) -> PhysicalAddress { + self.phys + } + pub fn level(&self) -> usize { self.level } @@ -288,7 +301,7 @@ impl PageTable { pub fn entry_base(&self, i: usize) -> Option { if i < PAGE_ENTRIES { Some(VirtualAddress( - self.base.0 + i << (9 * self.level + PAGE_SHIFT) + self.base.0 + (i << (self.level * 9 + PAGE_SHIFT)) )) } else { None @@ -368,20 +381,20 @@ fn main() { // PML4 link to PDP let pml4 = 0; let pdp = pml4 + PAGE_SIZE; - machine.write_phys::(pml4, pdp | ENTRY_PRESENT); + let flags = ENTRY_WRITABLE | ENTRY_PRESENT; + machine.write_phys::(pml4, pdp | flags); // PDP link to PD let pd = pdp + PAGE_SIZE; - machine.write_phys::(pdp, pd | ENTRY_PRESENT); + machine.write_phys::(pdp, pd | flags); // PD link to PT let pt = pd + PAGE_SIZE; - machine.write_phys::(pd, pt | ENTRY_PRESENT); + machine.write_phys::(pd, pt | flags); // PT links to frames for i in 0..PAGE_ENTRIES { let page = i * PAGE_SIZE; - let flags = ENTRY_WRITABLE | ENTRY_PRESENT; machine.write_phys::(pt + i * PAGE_ENTRY_SIZE, page | flags); } @@ -403,6 +416,16 @@ fn main() { // Test read println!("0x{:X} = 0x{:X}", megabyte, arch_read::(megabyte)); + // Set recursive mapping + { + let mut top = PageTable::top(); + top.set_entry(PAGE_ENTRIES - 1, PageEntry(top.phys().0 | ENTRY_WRITABLE | ENTRY_PRESENT)); + arch_invalidate_all(); + } + + // Debug table + dump_tables(PageTable::top()); + // Initialize memory allocator let areas = [MemoryArea { base: PhysicalAddress::new(megabyte), From 556b777386b4cf716935312bad2e8658d031aab4 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 4 Sep 2020 21:19:57 -0600 Subject: [PATCH 007/120] Define for page levels, enable recursive mapping by default --- src/main.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2c2abedb2f..b81a912789 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,7 @@ pub const PAGE_OFFSET_MASK: usize = PAGE_SIZE - 1; pub const PAGE_ADDRESS_MASK: usize = !PAGE_OFFSET_MASK; pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; +pub const PAGE_LEVELS: usize = 4; // Physical memory address #[derive(Clone, Copy, Debug)] @@ -277,7 +278,7 @@ impl PageTable { Self::new( VirtualAddress::new(0), PhysicalAddress::new(arch_get_table()), - 3 + PAGE_LEVELS - 1 ) } @@ -294,7 +295,9 @@ impl PageTable { } pub unsafe fn virt(&self) -> VirtualAddress { - //TODO: something other than identity mapping + //TODO: Recursive mapping + + // Identity mapping VirtualAddress(self.phys.0) } @@ -384,6 +387,9 @@ fn main() { let flags = ENTRY_WRITABLE | ENTRY_PRESENT; machine.write_phys::(pml4, pdp | flags); + // Recursive mapping + machine.write_phys::(pml4 + (PAGE_ENTRIES - 1) * PAGE_ENTRY_SIZE, pml4 | flags); + // PDP link to PD let pd = pdp + PAGE_SIZE; machine.write_phys::(pdp, pd | flags); @@ -416,16 +422,6 @@ fn main() { // Test read println!("0x{:X} = 0x{:X}", megabyte, arch_read::(megabyte)); - // Set recursive mapping - { - let mut top = PageTable::top(); - top.set_entry(PAGE_ENTRIES - 1, PageEntry(top.phys().0 | ENTRY_WRITABLE | ENTRY_PRESENT)); - arch_invalidate_all(); - } - - // Debug table - dump_tables(PageTable::top()); - // Initialize memory allocator let areas = [MemoryArea { base: PhysicalAddress::new(megabyte), From 58323aadd07cbb34479a9202cb50353fbe66a70d Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sat, 5 Sep 2020 19:00:32 -0600 Subject: [PATCH 008/120] Implement recursive mapping --- src/main.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index b81a912789..0006621204 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,8 @@ pub const PAGE_OFFSET_MASK: usize = PAGE_SIZE - 1; pub const PAGE_ADDRESS_MASK: usize = !PAGE_OFFSET_MASK; pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; +pub const PAGE_ENTRY_SHIFT: usize = 9; +pub const PAGE_ENTRY_MASK: usize = (1 << PAGE_ENTRY_SHIFT) - 1; pub const PAGE_LEVELS: usize = 4; // Physical memory address @@ -160,7 +162,7 @@ impl Machine { pub fn read(&self, virt: usize) -> T { //TODO: allow reading past page boundaries let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + size - 1) & PAGE_ADDRESS_MASK) { + if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); } @@ -174,7 +176,7 @@ impl Machine { pub fn write(&mut self, virt: usize, value: T) { //TODO: allow writing past page boundaries let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + size - 1) & PAGE_ADDRESS_MASK) { + if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); } @@ -227,6 +229,7 @@ impl Machine { // Page let a = e & ENTRY_ADDRESS_MASK; let page = + if i4 >= 256 { 0xFFFF_0000_0000_0000 } else { 0 } | (i4 << 39) | (i3 << 30) | (i2 << 21) | @@ -295,16 +298,23 @@ impl PageTable { } pub unsafe fn virt(&self) -> VirtualAddress { - //TODO: Recursive mapping + // Recursive mapping + let mut addr = 0xFFFF_FFFF_FFFF_F000; + for level in (self.level + 1 .. PAGE_LEVELS).rev() { + let index = (self.base.0 >> (level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) & PAGE_ENTRY_MASK; + addr <<= PAGE_ENTRY_SHIFT; + addr |= index << PAGE_SHIFT; + } + VirtualAddress(addr) // Identity mapping - VirtualAddress(self.phys.0) + //VirtualAddress(self.phys.0) } pub fn entry_base(&self, i: usize) -> Option { if i < PAGE_ENTRIES { Some(VirtualAddress( - self.base.0 + (i << (self.level * 9 + PAGE_SHIFT)) + self.base.0 + (i << (self.level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) )) } else { None From 8c8e09d23ef27b7cfa1029203fdc92f411b4d1c9 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sat, 5 Sep 2020 21:02:27 -0600 Subject: [PATCH 009/120] Refactor, use trait for arch differences --- src/arch/emulate.rs | 236 ++++++++++++++++++++++++ src/arch/mod.rs | 28 +++ src/arch/x86_64.rs | 21 +++ src/lib.rs | 54 ++++++ src/main.rs | 438 ++------------------------------------------ src/page/entry.rs | 27 +++ src/page/mod.rs | 7 + src/page/table.rs | 106 +++++++++++ 8 files changed, 499 insertions(+), 418 deletions(-) create mode 100644 src/arch/emulate.rs create mode 100644 src/arch/mod.rs create mode 100644 src/arch/x86_64.rs create mode 100644 src/lib.rs create mode 100644 src/page/entry.rs create mode 100644 src/page/mod.rs create mode 100644 src/page/table.rs diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs new file mode 100644 index 0000000000..9332b77a47 --- /dev/null +++ b/src/arch/emulate.rs @@ -0,0 +1,236 @@ +// Import values from x86_64 +pub use crate::arch::x86_64::*; + +use core::{ + mem, + ptr, +}; +use std::collections::BTreeMap; + +use crate::{ + Arch, + PhysicalAddress, + MemoryArea, +}; + + +pub struct EmulateArch; + +impl Arch for EmulateArch { + unsafe fn init() -> &'static [MemoryArea] { + // Create machine with PAGE_ENTRIES pages identity mapped (2 MiB on x86_64) + // Pages over 1 MiB will be mapped writable + let mut machine = Machine::new(MEMORY_SIZE); + + // PML4 link to PDP + let pml4 = 0; + let pdp = pml4 + PAGE_SIZE; + let flags = ENTRY_FLAG_WRITABLE | ENTRY_FLAG_PRESENT; + machine.write_phys::(pml4, pdp | flags); + + // Recursive mapping + machine.write_phys::(pml4 + (PAGE_ENTRIES - 1) * PAGE_ENTRY_SIZE, pml4 | flags); + + // PDP link to PD + let pd = pdp + PAGE_SIZE; + machine.write_phys::(pdp, pd | flags); + + // PD link to PT + let pt = pd + PAGE_SIZE; + machine.write_phys::(pd, pt | flags); + + // PT links to frames + for i in 0..PAGE_ENTRIES { + let page = i * PAGE_SIZE; + machine.write_phys::(pt + i * PAGE_ENTRY_SIZE, page | flags); + } + + MACHINE = Some(machine); + + // Set table to pml4 + EmulateArch::set_table(pml4); + + &MEMORY_AREAS + } + + #[inline(always)] + unsafe fn read(address: usize) -> T { + MACHINE.as_ref().unwrap().read(address) + } + + #[inline(always)] + unsafe fn write(address: usize, value: T) { + MACHINE.as_mut().unwrap().write(address, value) + } + + #[inline(always)] + unsafe fn invalidate(address: usize) { + MACHINE.as_mut().unwrap().invalidate(address); + } + + #[inline(always)] + unsafe fn invalidate_all() { + MACHINE.as_mut().unwrap().invalidate_all(); + } + + #[inline(always)] + unsafe fn table() -> usize { + MACHINE.as_mut().unwrap().get_table() + } + + #[inline(always)] + unsafe fn set_table(address: usize) { + MACHINE.as_mut().unwrap().set_table(address); + } +} + +const MEGABYTE: usize = 1024 * 1024; +const MEMORY_SIZE: usize = 64 * MEGABYTE; +static MEMORY_AREAS: [MemoryArea; 1] = [ + MemoryArea { + base: PhysicalAddress::new(MEGABYTE), + size: MEMORY_SIZE, + } +]; + +static mut MACHINE: Option = None; + +struct Machine { + memory: Box<[u8]>, + map: BTreeMap, + table_addr: usize, +} + +impl Machine { + fn new(memory_size: usize) -> Self { + Self { + memory: vec![0; memory_size].into_boxed_slice(), + map: BTreeMap::new(), + table_addr: 0, + } + } + + fn read_phys(&self, phys: usize) -> T { + let size = mem::size_of::(); + if phys + size <= self.memory.len() { + unsafe { + ptr::read(self.memory.as_ptr().add(phys) as *const T) + } + } else { + panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); + } + } + + fn write_phys(&mut self, phys: usize, value: T) { + let size = mem::size_of::(); + if phys + size <= self.memory.len() { + unsafe { + ptr::write(self.memory.as_mut_ptr().add(phys) as *mut T, value); + } + } else { + panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); + } + } + + fn translate(&self, virt: usize) -> Option<(usize, usize)> { + let page = virt & PAGE_ADDRESS_MASK; + let offset = virt & PAGE_OFFSET_MASK; + let phys = self.map.get(&page)?; + Some(( + (phys & ENTRY_ADDRESS_MASK) + offset, + phys & ENTRY_FLAGS_MASK, + )) + } + + fn read(&self, virt: usize) -> T { + //TODO: allow reading past page boundaries + let size = mem::size_of::(); + if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { + panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); + } + + if let Some((phys, _flags)) = self.translate(virt) { + self.read_phys(phys) + } else { + panic!("read: 0x{:X} size 0x{:X} not present", virt, size); + } + } + + fn write(&mut self, virt: usize, value: T) { + //TODO: allow writing past page boundaries + let size = mem::size_of::(); + if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { + panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); + } + + if let Some((phys, flags)) = self.translate(virt) { + if flags & ENTRY_FLAG_WRITABLE != 0 { + self.write_phys(phys, value); + } else { + panic!("write: 0x{:X} size 0x{:X} not writable", virt, size); + } + } else { + panic!("write: 0x{:X} size 0x{:X} not present", virt, size); + } + } + + fn invalidate(&mut self, _address: usize) { + unimplemented!(); + } + + fn invalidate_all(&mut self) { + self.map.clear(); + + // PML4 + let a4 = self.table_addr; + for i4 in 0..PAGE_ENTRIES { + let e3 = self.read_phys::(a4 + i4 * PAGE_ENTRY_SIZE); + let f3 = e3 & ENTRY_FLAGS_MASK; + if f3 & ENTRY_FLAG_PRESENT == 0 { continue; } + + // Page directory pointer + let a3 = e3 & ENTRY_ADDRESS_MASK; + for i3 in 0..PAGE_ENTRIES { + let e2 = self.read_phys::(a3 + i3 * PAGE_ENTRY_SIZE); + let f2 = e2 & ENTRY_FLAGS_MASK; + if f2 & ENTRY_FLAG_PRESENT == 0 { continue; } + + // Page directory + let a2 = e2 & ENTRY_ADDRESS_MASK; + for i2 in 0..PAGE_ENTRIES { + let e1 = self.read_phys::(a2 + i2 * PAGE_ENTRY_SIZE); + let f1 = e1 & ENTRY_FLAGS_MASK; + if f1 & ENTRY_FLAG_PRESENT == 0 { continue; } + + // Page table + let a1 = e1 & ENTRY_ADDRESS_MASK; + for i1 in 0..PAGE_ENTRIES { + let e = self.read_phys::(a1 + i1 * PAGE_ENTRY_SIZE); + let f = e & ENTRY_FLAGS_MASK; + if f & ENTRY_FLAG_PRESENT == 0 { continue; } + + // Page + let a = e & ENTRY_ADDRESS_MASK; + let page = + if i4 >= 256 { 0xFFFF_0000_0000_0000 } else { 0 } | + (i4 << 39) | + (i3 << 30) | + (i2 << 21) | + (i1 << 12); + println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); + self.map.insert(page, e); + } + } + } + } + } + + fn get_table(&self) -> usize { + self.table_addr + } + + fn set_table(&mut self, address: usize) { + self.table_addr = address; + self.invalidate_all(); + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs new file mode 100644 index 0000000000..d066d1b010 --- /dev/null +++ b/src/arch/mod.rs @@ -0,0 +1,28 @@ +use core::ptr; + +use crate::MemoryArea; + +pub mod emulate; +pub mod x86_64; + +pub trait Arch { + unsafe fn init() -> &'static [MemoryArea]; + + #[inline(always)] + unsafe fn read(address: usize) -> T { + ptr::read(address as *const T) + } + + #[inline(always)] + unsafe fn write(address: usize, value: T) { + ptr::write(address as *mut T, value) + } + + unsafe fn invalidate(address: usize); + + unsafe fn invalidate_all(); + + unsafe fn table() -> usize; + + unsafe fn set_table(address: usize); +} diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs new file mode 100644 index 0000000000..7c743ee44a --- /dev/null +++ b/src/arch/x86_64.rs @@ -0,0 +1,21 @@ +use core::mem; + +//TODO: should these be constants? +pub const PAGE_SHIFT: usize = 12; // 4096 bytes +pub const PAGE_SIZE: usize = 1 << PAGE_SHIFT; +pub const PAGE_OFFSET_MASK: usize = PAGE_SIZE - 1; +pub const PAGE_ADDRESS_MASK: usize = !PAGE_OFFSET_MASK; +pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); +pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; +pub const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries +pub const PAGE_ENTRY_MASK: usize = (1 << PAGE_ENTRY_SHIFT) - 1; +pub const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + +pub const ENTRY_FLAG_PRESENT: usize = 1 << 0; +pub const ENTRY_FLAG_WRITABLE: usize = 1 << 1; +pub const ENTRY_FLAG_USER: usize = 1 << 2; +pub const ENTRY_FLAG_HUGE: usize = 1 << 7; +pub const ENTRY_FLAG_GLOBAL: usize = 1 << 8; +pub const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; +pub const ENTRY_ADDRESS_MASK: usize = PAGE_ADDRESS_MASK; +pub const ENTRY_FLAGS_MASK: usize = !ENTRY_ADDRESS_MASK; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000000..2253006639 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,54 @@ +pub use crate::{ + arch::Arch, + arch::emulate::*, + page::{ + PageEntry, + PageTable + }, +}; + +mod arch; +mod page; + +// Physical memory address +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct PhysicalAddress(usize); + +impl PhysicalAddress { + pub const fn new(address: usize) -> Self { + Self(address) + } + + pub unsafe fn data(&self) -> usize { + self.0 + } + + pub const fn aligned(&self) -> bool { + self.0 & PAGE_OFFSET_MASK == 0 + } +} + +// Virtual memory address +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct VirtualAddress(usize); + +impl VirtualAddress { + pub const fn new(address: usize) -> Self { + Self(address) + } + + pub unsafe fn data(&self) -> usize { + self.0 + } + + pub const fn aligned(&self) -> bool { + self.0 & PAGE_OFFSET_MASK == 0 + } +} + +pub struct MemoryArea { + base: PhysicalAddress, + size: usize, +} diff --git a/src/main.rs b/src/main.rs index 0006621204..31a11fd698 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,370 +1,13 @@ -use core::{ - mem, - ptr, -}; -use std::collections::BTreeMap; +use rmm::*; -//TODO: should this be a constant? -pub const PAGE_SHIFT: usize = 12; -pub const PAGE_SIZE: usize = 1 << PAGE_SHIFT; -pub const PAGE_OFFSET_MASK: usize = PAGE_SIZE - 1; -pub const PAGE_ADDRESS_MASK: usize = !PAGE_OFFSET_MASK; -pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); -pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; -pub const PAGE_ENTRY_SHIFT: usize = 9; -pub const PAGE_ENTRY_MASK: usize = (1 << PAGE_ENTRY_SHIFT) - 1; -pub const PAGE_LEVELS: usize = 4; - -// Physical memory address -#[derive(Clone, Copy, Debug)] -#[repr(transparent)] -pub struct PhysicalAddress(usize); - -impl PhysicalAddress { - pub const fn new(address: usize) -> Self { - Self(address) - } - - pub const fn aligned(&self) -> bool { - self.0 & PAGE_OFFSET_MASK == 0 - } -} - -// Physical memory frame -#[derive(Clone, Copy, Debug)] -#[repr(transparent)] -pub struct Frame(PhysicalAddress); - -impl Frame { - pub fn new(address: PhysicalAddress) -> Option { - if address.aligned() { - Some(Frame(address)) - } else { - None - } - } -} - -// Virtual memory address -#[derive(Clone, Copy, Debug)] -#[repr(transparent)] -pub struct VirtualAddress(usize); - -impl VirtualAddress { - pub const fn new(address: usize) -> Self { - Self(address) - } - - pub const fn aligned(&self) -> bool { - self.0 & PAGE_OFFSET_MASK == 0 - } -} - -// Virtual memory page -#[derive(Clone, Copy, Debug)] -#[repr(transparent)] -pub struct Page(VirtualAddress); - -impl Page { - pub fn new(address: VirtualAddress) -> Option { - if address.aligned() { - Some(Self(address)) - } else { - None - } - } -} - -const ENTRY_PRESENT: usize = 1 << 0; -const ENTRY_WRITABLE: usize = 1 << 1; -const ENTRY_ADDRESS_MASK: usize = PAGE_ADDRESS_MASK; -const ENTRY_FLAGS_MASK: usize = !ENTRY_ADDRESS_MASK; - -static mut MACHINE: Option = None; - -#[inline(always)] -pub unsafe fn arch_read(address: usize) -> T { - MACHINE.as_ref().unwrap().read(address) -} - -#[inline(always)] -pub unsafe fn arch_write(address: usize, value: T) { - MACHINE.as_mut().unwrap().write(address, value) -} - -#[inline(always)] -pub unsafe fn arch_invalidate(address: usize) { - MACHINE.as_mut().unwrap().invalidate(address); -} - -#[inline(always)] -pub unsafe fn arch_invalidate_all() { - MACHINE.as_mut().unwrap().invalidate_all(); -} - -#[inline(always)] -pub unsafe fn arch_get_table() -> usize { - MACHINE.as_mut().unwrap().get_table() -} - -#[inline(always)] -pub unsafe fn arch_set_table(address: usize) { - MACHINE.as_mut().unwrap().set_table(address); -} - -pub struct Machine { - pub memory: Box<[u8]>, - pub map: BTreeMap, - pub table_addr: usize, -} - -impl Machine { - pub fn new(memory_size: usize) -> Self { - Self { - memory: vec![0; memory_size].into_boxed_slice(), - map: BTreeMap::new(), - table_addr: 0, - } - } - - pub fn read_phys(&self, phys: usize) -> T { - let size = mem::size_of::(); - if phys + size <= self.memory.len() { - unsafe { - ptr::read(self.memory.as_ptr().add(phys) as *const T) - } - } else { - panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); - } - } - - pub fn write_phys(&mut self, phys: usize, value: T) { - let size = mem::size_of::(); - if phys + size <= self.memory.len() { - unsafe { - ptr::write(self.memory.as_mut_ptr().add(phys) as *mut T, value); - } - } else { - panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); - } - } - - pub fn translate(&self, virt: usize) -> Option<(usize, usize)> { - let page = virt & PAGE_ADDRESS_MASK; - let offset = virt & PAGE_OFFSET_MASK; - let phys = self.map.get(&page)?; - Some(( - (phys & ENTRY_ADDRESS_MASK) + offset, - phys & ENTRY_FLAGS_MASK, - )) - } - - pub fn read(&self, virt: usize) -> T { - //TODO: allow reading past page boundaries - let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { - panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); - } - - if let Some((phys, _flags)) = self.translate(virt) { - self.read_phys(phys) - } else { - panic!("read: 0x{:X} size 0x{:X} not present", virt, size); - } - } - - pub fn write(&mut self, virt: usize, value: T) { - //TODO: allow writing past page boundaries - let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { - panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); - } - - if let Some((phys, flags)) = self.translate(virt) { - if flags & ENTRY_WRITABLE != 0 { - self.write_phys(phys, value); - } else { - panic!("write: 0x{:X} size 0x{:X} not writable", virt, size); - } - } else { - panic!("write: 0x{:X} size 0x{:X} not present", virt, size); - } - } - - pub fn invalidate(&mut self, _address: usize) { - unimplemented!(); - } - - pub fn invalidate_all(&mut self) { - self.map.clear(); - - // PML4 - let a4 = self.table_addr; - for i4 in 0..PAGE_ENTRIES { - let e3 = self.read_phys::(a4 + i4 * PAGE_ENTRY_SIZE); - let f3 = e3 & ENTRY_FLAGS_MASK; - if f3 & ENTRY_PRESENT == 0 { continue; } - - // Page directory pointer - let a3 = e3 & ENTRY_ADDRESS_MASK; - for i3 in 0..PAGE_ENTRIES { - let e2 = self.read_phys::(a3 + i3 * PAGE_ENTRY_SIZE); - let f2 = e2 & ENTRY_FLAGS_MASK; - if f2 & ENTRY_PRESENT == 0 { continue; } - - // Page directory - let a2 = e2 & ENTRY_ADDRESS_MASK; - for i2 in 0..PAGE_ENTRIES { - let e1 = self.read_phys::(a2 + i2 * PAGE_ENTRY_SIZE); - let f1 = e1 & ENTRY_FLAGS_MASK; - if f1 & ENTRY_PRESENT == 0 { continue; } - - // Page table - let a1 = e1 & ENTRY_ADDRESS_MASK; - for i1 in 0..PAGE_ENTRIES { - let e = self.read_phys::(a1 + i1 * PAGE_ENTRY_SIZE); - let f = e & ENTRY_FLAGS_MASK; - if f & ENTRY_PRESENT == 0 { continue; } - - // Page - let a = e & ENTRY_ADDRESS_MASK; - let page = - if i4 >= 256 { 0xFFFF_0000_0000_0000 } else { 0 } | - (i4 << 39) | - (i3 << 30) | - (i2 << 21) | - (i1 << 12); - println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); - self.map.insert(page, e); - } - } - } - } - } - - pub fn get_table(&self) -> usize { - self.table_addr - } - - pub fn set_table(&mut self, address: usize) { - self.table_addr = address; - self.invalidate_all(); - } -} - -#[derive(Clone, Copy, Debug)] -#[repr(transparent)] -pub struct PageEntry(usize); - -impl PageEntry { - pub fn address(&self) -> PhysicalAddress { - PhysicalAddress(self.0 & ENTRY_ADDRESS_MASK) - } - - pub fn present(&self) -> bool { - self.0 & ENTRY_PRESENT != 0 - } -} - -pub struct PageTable { - base: VirtualAddress, - phys: PhysicalAddress, - level: usize -} - -impl PageTable { - pub unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self { - Self { base, phys, level } - } - - pub unsafe fn top() -> Self { - Self::new( - VirtualAddress::new(0), - PhysicalAddress::new(arch_get_table()), - PAGE_LEVELS - 1 - ) - } - - pub fn base(&self) -> VirtualAddress { - self.base - } - - pub fn phys(&self) -> PhysicalAddress { - self.phys - } - - pub fn level(&self) -> usize { - self.level - } - - pub unsafe fn virt(&self) -> VirtualAddress { - // Recursive mapping - let mut addr = 0xFFFF_FFFF_FFFF_F000; - for level in (self.level + 1 .. PAGE_LEVELS).rev() { - let index = (self.base.0 >> (level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) & PAGE_ENTRY_MASK; - addr <<= PAGE_ENTRY_SHIFT; - addr |= index << PAGE_SHIFT; - } - VirtualAddress(addr) - - // Identity mapping - //VirtualAddress(self.phys.0) - } - - pub fn entry_base(&self, i: usize) -> Option { - if i < PAGE_ENTRIES { - Some(VirtualAddress( - self.base.0 + (i << (self.level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) - )) - } else { - None - } - } - - pub unsafe fn entry_virt(&self, i: usize) -> Option { - if i < PAGE_ENTRIES { - Some(VirtualAddress( - self.virt().0 + i * PAGE_ENTRY_SIZE - )) - } else { - None - } - } - - pub unsafe fn entry(&self, i: usize) -> Option { - let addr = self.entry_virt(i)?; - Some(PageEntry(arch_read::(addr.0))) - } - - pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { - let addr = self.entry_virt(i)?; - arch_write::(addr.0, entry.0); - Some(()) - } - - pub unsafe fn next(&self, i: usize) -> Option { - if self.level > 0 { - let entry = self.entry(i)?; - if entry.present() { - return Some(PageTable::new( - self.entry_base(i)?, - entry.address(), - self.level - 1 - )); - } - } - None - } -} - -unsafe fn dump_tables(table: PageTable) { +unsafe fn dump_tables(table: PageTable) { let level = table.level(); for i in 0..PAGE_ENTRIES { if level == 0 { if let Some(entry) = table.entry(i) { if entry.present() { let base = table.entry_base(i).unwrap(); - println!("0x{:X}: 0x{:X}", base.0, entry.address().0); + println!("0x{:X}: 0x{:X}", base.data(), entry.address().data()); } } } else { @@ -375,67 +18,26 @@ unsafe fn dump_tables(table: PageTable) { } } -pub struct MemoryArea { - base: PhysicalAddress, - size: usize, +unsafe fn inner() { + let areas = A::init(); + + // Debug table + dump_tables(PageTable::::top()); + + let megabyte = 0x100000; + + // Test read + println!("0x{:X} = 0x{:X}", megabyte, A::read::(megabyte)); + + // Test write + A::write::(megabyte, 0x5A); + + // Test read + println!("0x{:X} = 0x{:X}", megabyte, A::read::(megabyte)); } fn main() { - let memory_size = 64 * 1024 * 1024; - unsafe { - let megabyte = 0x100000; - - // Create machine with PAGE_ENTRIES pages identity mapped (2 MiB on x86_64) - // Pages over 1 MiB will be mapped writable - { - let mut machine = Machine::new(memory_size); - - // PML4 link to PDP - let pml4 = 0; - let pdp = pml4 + PAGE_SIZE; - let flags = ENTRY_WRITABLE | ENTRY_PRESENT; - machine.write_phys::(pml4, pdp | flags); - - // Recursive mapping - machine.write_phys::(pml4 + (PAGE_ENTRIES - 1) * PAGE_ENTRY_SIZE, pml4 | flags); - - // PDP link to PD - let pd = pdp + PAGE_SIZE; - machine.write_phys::(pdp, pd | flags); - - // PD link to PT - let pt = pd + PAGE_SIZE; - machine.write_phys::(pd, pt | flags); - - // PT links to frames - for i in 0..PAGE_ENTRIES { - let page = i * PAGE_SIZE; - machine.write_phys::(pt + i * PAGE_ENTRY_SIZE, page | flags); - } - - MACHINE = Some(machine); - - // Set table to pml4 - arch_set_table(pml4); - } - - // Debug table - dump_tables(PageTable::top()); - - // Test read - println!("0x{:X} = 0x{:X}", megabyte, arch_read::(megabyte)); - - // Test write - arch_write::(megabyte, 0x5A); - - // Test read - println!("0x{:X} = 0x{:X}", megabyte, arch_read::(megabyte)); - - // Initialize memory allocator - let areas = [MemoryArea { - base: PhysicalAddress::new(megabyte), - size: megabyte, - }]; + inner::(); } } diff --git a/src/page/entry.rs b/src/page/entry.rs new file mode 100644 index 0000000000..7b8c7e8148 --- /dev/null +++ b/src/page/entry.rs @@ -0,0 +1,27 @@ +use crate::{ + PhysicalAddress, + ENTRY_ADDRESS_MASK, + ENTRY_FLAG_PRESENT, +}; + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct PageEntry(usize); + +impl PageEntry { + pub unsafe fn new(data: usize) -> Self { + Self(data) + } + + pub fn data(&self) -> usize { + self.0 + } + + pub fn address(&self) -> PhysicalAddress { + PhysicalAddress(self.0 & ENTRY_ADDRESS_MASK) + } + + pub fn present(&self) -> bool { + self.0 & ENTRY_FLAG_PRESENT != 0 + } +} diff --git a/src/page/mod.rs b/src/page/mod.rs new file mode 100644 index 0000000000..bef67a6fd6 --- /dev/null +++ b/src/page/mod.rs @@ -0,0 +1,7 @@ +pub use self::{ + entry::PageEntry, + table::PageTable, +}; + +mod entry; +mod table; diff --git a/src/page/table.rs b/src/page/table.rs new file mode 100644 index 0000000000..478077d83d --- /dev/null +++ b/src/page/table.rs @@ -0,0 +1,106 @@ +use core::marker::PhantomData; + +use crate::{ + Arch, + PhysicalAddress, + VirtualAddress, + PAGE_ENTRIES, + PAGE_ENTRY_MASK, + PAGE_ENTRY_SHIFT, + PAGE_ENTRY_SIZE, + PAGE_LEVELS, + PAGE_SHIFT, +}; +use super::PageEntry; + +pub struct PageTable { + base: VirtualAddress, + phys: PhysicalAddress, + level: usize, + phantom: PhantomData, +} + +impl PageTable { + pub unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self { + Self { base, phys, level, phantom: PhantomData } + } + + pub unsafe fn top() -> Self { + Self::new( + VirtualAddress::new(0), + PhysicalAddress::new(A::table()), + PAGE_LEVELS - 1 + ) + } + + pub fn base(&self) -> VirtualAddress { + self.base + } + + pub fn phys(&self) -> PhysicalAddress { + self.phys + } + + pub fn level(&self) -> usize { + self.level + } + + pub unsafe fn virt(&self) -> VirtualAddress { + // Recursive mapping + let mut addr = 0xFFFF_FFFF_FFFF_F000; + for level in (self.level + 1 .. PAGE_LEVELS).rev() { + let index = (self.base.0 >> (level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) & PAGE_ENTRY_MASK; + addr <<= PAGE_ENTRY_SHIFT; + addr |= index << PAGE_SHIFT; + } + VirtualAddress(addr) + + // Identity mapping + //VirtualAddress(self.phys.0) + } + + pub fn entry_base(&self, i: usize) -> Option { + if i < PAGE_ENTRIES { + Some(VirtualAddress( + self.base.0 + (i << (self.level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) + )) + } else { + None + } + } + + pub unsafe fn entry_virt(&self, i: usize) -> Option { + if i < PAGE_ENTRIES { + Some(VirtualAddress( + self.virt().0 + i * PAGE_ENTRY_SIZE + )) + } else { + None + } + } + + pub unsafe fn entry(&self, i: usize) -> Option { + let addr = self.entry_virt(i)?; + Some(PageEntry::new(A::read::(addr.0))) + } + + pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { + let addr = self.entry_virt(i)?; + A::write::(addr.0, entry.data()); + Some(()) + } + + pub unsafe fn next(&self, i: usize) -> Option { + if self.level > 0 { + let entry = self.entry(i)?; + if entry.present() { + return Some(PageTable::new( + self.entry_base(i)?, + entry.address(), + self.level - 1 + )); + } + } + None + } +} From f5e8c031ca629c75359547b6e5b5356ee555b6ab Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 08:30:56 -0600 Subject: [PATCH 010/120] Move constants into Arch trait --- src/arch/emulate.rs | 95 +++++++++++++++++++++++++-------------------- src/arch/mod.rs | 21 ++++++++++ src/arch/x86_64.rs | 64 +++++++++++++++++++++--------- src/lib.rs | 8 ---- src/main.rs | 8 +++- src/page/entry.rs | 21 +++++----- src/page/table.rs | 28 ++++++------- 7 files changed, 148 insertions(+), 97 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 9332b77a47..2e290b6810 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -1,7 +1,5 @@ -// Import values from x86_64 -pub use crate::arch::x86_64::*; - use core::{ + marker::PhantomData, mem, ptr, }; @@ -11,12 +9,23 @@ use crate::{ Arch, PhysicalAddress, MemoryArea, + arch::x86_64::X8664Arch, }; - pub struct EmulateArch; impl Arch for EmulateArch { + const PAGE_SHIFT: usize = X8664Arch::PAGE_SHIFT; + const PAGE_ENTRY_SHIFT: usize = X8664Arch::PAGE_ENTRY_SHIFT; + const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS; + + const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_WRITABLE: usize = X8664Arch::ENTRY_FLAG_WRITABLE; + const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; + const ENTRY_FLAG_HUGE: usize = X8664Arch::ENTRY_FLAG_HUGE; + const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; + const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages identity mapped (2 MiB on x86_64) // Pages over 1 MiB will be mapped writable @@ -24,25 +33,25 @@ impl Arch for EmulateArch { // PML4 link to PDP let pml4 = 0; - let pdp = pml4 + PAGE_SIZE; - let flags = ENTRY_FLAG_WRITABLE | ENTRY_FLAG_PRESENT; + let pdp = pml4 + Self::PAGE_SIZE; + let flags = Self::ENTRY_FLAG_WRITABLE | Self::ENTRY_FLAG_PRESENT; machine.write_phys::(pml4, pdp | flags); // Recursive mapping - machine.write_phys::(pml4 + (PAGE_ENTRIES - 1) * PAGE_ENTRY_SIZE, pml4 | flags); + machine.write_phys::(pml4 + (Self::PAGE_ENTRIES - 1) * Self::PAGE_ENTRY_SIZE, pml4 | flags); // PDP link to PD - let pd = pdp + PAGE_SIZE; + let pd = pdp + Self::PAGE_SIZE; machine.write_phys::(pdp, pd | flags); // PD link to PT - let pt = pd + PAGE_SIZE; + let pt = pd + Self::PAGE_SIZE; machine.write_phys::(pd, pt | flags); // PT links to frames - for i in 0..PAGE_ENTRIES { - let page = i * PAGE_SIZE; - machine.write_phys::(pt + i * PAGE_ENTRY_SIZE, page | flags); + for i in 0..Self::PAGE_ENTRIES { + let page = i * Self::PAGE_SIZE; + machine.write_phys::(pt + i * Self::PAGE_ENTRY_SIZE, page | flags); } MACHINE = Some(machine); @@ -93,20 +102,22 @@ static MEMORY_AREAS: [MemoryArea; 1] = [ } ]; -static mut MACHINE: Option = None; +static mut MACHINE: Option> = None; -struct Machine { +struct Machine { memory: Box<[u8]>, map: BTreeMap, table_addr: usize, + phantom: PhantomData, } -impl Machine { +impl Machine { fn new(memory_size: usize) -> Self { Self { memory: vec![0; memory_size].into_boxed_slice(), map: BTreeMap::new(), table_addr: 0, + phantom: PhantomData, } } @@ -133,19 +144,19 @@ impl Machine { } fn translate(&self, virt: usize) -> Option<(usize, usize)> { - let page = virt & PAGE_ADDRESS_MASK; - let offset = virt & PAGE_OFFSET_MASK; + let page = virt & A::PAGE_ADDRESS_MASK; + let offset = virt & A::PAGE_OFFSET_MASK; let phys = self.map.get(&page)?; Some(( - (phys & ENTRY_ADDRESS_MASK) + offset, - phys & ENTRY_FLAGS_MASK, + (phys & A::ENTRY_ADDRESS_MASK) + offset, + phys & A::ENTRY_FLAGS_MASK, )) } fn read(&self, virt: usize) -> T { //TODO: allow reading past page boundaries let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { + if (virt & A::PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & A::PAGE_ADDRESS_MASK) { panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); } @@ -159,12 +170,12 @@ impl Machine { fn write(&mut self, virt: usize, value: T) { //TODO: allow writing past page boundaries let size = mem::size_of::(); - if (virt & PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & PAGE_ADDRESS_MASK) { + if (virt & A::PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & A::PAGE_ADDRESS_MASK) { panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); } if let Some((phys, flags)) = self.translate(virt) { - if flags & ENTRY_FLAG_WRITABLE != 0 { + if flags & A::ENTRY_FLAG_WRITABLE != 0 { self.write_phys(phys, value); } else { panic!("write: 0x{:X} size 0x{:X} not writable", virt, size); @@ -183,34 +194,34 @@ impl Machine { // PML4 let a4 = self.table_addr; - for i4 in 0..PAGE_ENTRIES { - let e3 = self.read_phys::(a4 + i4 * PAGE_ENTRY_SIZE); - let f3 = e3 & ENTRY_FLAGS_MASK; - if f3 & ENTRY_FLAG_PRESENT == 0 { continue; } + for i4 in 0..A::PAGE_ENTRIES { + let e3 = self.read_phys::(a4 + i4 * A::PAGE_ENTRY_SIZE); + let f3 = e3 & A::ENTRY_FLAGS_MASK; + if f3 & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page directory pointer - let a3 = e3 & ENTRY_ADDRESS_MASK; - for i3 in 0..PAGE_ENTRIES { - let e2 = self.read_phys::(a3 + i3 * PAGE_ENTRY_SIZE); - let f2 = e2 & ENTRY_FLAGS_MASK; - if f2 & ENTRY_FLAG_PRESENT == 0 { continue; } + let a3 = e3 & A::ENTRY_ADDRESS_MASK; + for i3 in 0..A::PAGE_ENTRIES { + let e2 = self.read_phys::(a3 + i3 * A::PAGE_ENTRY_SIZE); + let f2 = e2 & A::ENTRY_FLAGS_MASK; + if f2 & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page directory - let a2 = e2 & ENTRY_ADDRESS_MASK; - for i2 in 0..PAGE_ENTRIES { - let e1 = self.read_phys::(a2 + i2 * PAGE_ENTRY_SIZE); - let f1 = e1 & ENTRY_FLAGS_MASK; - if f1 & ENTRY_FLAG_PRESENT == 0 { continue; } + let a2 = e2 & A::ENTRY_ADDRESS_MASK; + for i2 in 0..A::PAGE_ENTRIES { + let e1 = self.read_phys::(a2 + i2 * A::PAGE_ENTRY_SIZE); + let f1 = e1 & A::ENTRY_FLAGS_MASK; + if f1 & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page table - let a1 = e1 & ENTRY_ADDRESS_MASK; - for i1 in 0..PAGE_ENTRIES { - let e = self.read_phys::(a1 + i1 * PAGE_ENTRY_SIZE); - let f = e & ENTRY_FLAGS_MASK; - if f & ENTRY_FLAG_PRESENT == 0 { continue; } + let a1 = e1 & A::ENTRY_ADDRESS_MASK; + for i1 in 0..A::PAGE_ENTRIES { + let e = self.read_phys::(a1 + i1 * A::PAGE_ENTRY_SIZE); + let f = e & A::ENTRY_FLAGS_MASK; + if f & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page - let a = e & ENTRY_ADDRESS_MASK; + let a = e & A::ENTRY_ADDRESS_MASK; let page = if i4 >= 256 { 0xFFFF_0000_0000_0000 } else { 0 } | (i4 << 39) | diff --git a/src/arch/mod.rs b/src/arch/mod.rs index d066d1b010..c7a8bf0bb9 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -6,6 +6,27 @@ pub mod emulate; pub mod x86_64; pub trait Arch { + const PAGE_SHIFT: usize; + const PAGE_ENTRY_SHIFT: usize; + const PAGE_LEVELS: usize; + + const ENTRY_FLAG_PRESENT: usize; + const ENTRY_FLAG_WRITABLE: usize; + const ENTRY_FLAG_USER: usize; + const ENTRY_FLAG_HUGE: usize; + const ENTRY_FLAG_GLOBAL: usize; + const ENTRY_FLAG_NO_EXEC: usize; + + const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT; + const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1; + const PAGE_ADDRESS_MASK: usize = !Self::PAGE_OFFSET_MASK; + const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); + const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; + const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; + + const ENTRY_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_MASK; + const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; + unsafe fn init() -> &'static [MemoryArea]; #[inline(always)] diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 7c743ee44a..3554304759 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -1,21 +1,47 @@ -use core::mem; +use crate::{ + Arch, + MemoryArea +}; -//TODO: should these be constants? -pub const PAGE_SHIFT: usize = 12; // 4096 bytes -pub const PAGE_SIZE: usize = 1 << PAGE_SHIFT; -pub const PAGE_OFFSET_MASK: usize = PAGE_SIZE - 1; -pub const PAGE_ADDRESS_MASK: usize = !PAGE_OFFSET_MASK; -pub const PAGE_ENTRY_SIZE: usize = mem::size_of::(); -pub const PAGE_ENTRIES: usize = PAGE_SIZE / PAGE_ENTRY_SIZE; -pub const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries -pub const PAGE_ENTRY_MASK: usize = (1 << PAGE_ENTRY_SHIFT) - 1; -pub const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT +pub struct X8664Arch; -pub const ENTRY_FLAG_PRESENT: usize = 1 << 0; -pub const ENTRY_FLAG_WRITABLE: usize = 1 << 1; -pub const ENTRY_FLAG_USER: usize = 1 << 2; -pub const ENTRY_FLAG_HUGE: usize = 1 << 7; -pub const ENTRY_FLAG_GLOBAL: usize = 1 << 8; -pub const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; -pub const ENTRY_ADDRESS_MASK: usize = PAGE_ADDRESS_MASK; -pub const ENTRY_FLAGS_MASK: usize = !ENTRY_ADDRESS_MASK; +impl Arch for X8664Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_WRITABLE: usize = 1 << 1; + const ENTRY_FLAG_USER: usize = 1 << 2; + const ENTRY_FLAG_HUGE: usize = 1 << 7; + const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!() + } + + #[inline(always)] + unsafe fn invalidate(address: usize) { + //TODO: invlpg address + unimplemented!(); + } + + #[inline(always)] + unsafe fn invalidate_all() { + //TODO: mov cr3, cr3 + unimplemented!(); + } + + #[inline(always)] + unsafe fn table() -> usize { + //TODO: return cr3 + unimplemented!(); + } + + #[inline(always)] + unsafe fn set_table(address: usize) { + //TODO: mov cr3, address + unimplemented!(); + } +} diff --git a/src/lib.rs b/src/lib.rs index 2253006639..72417fe1cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,10 +23,6 @@ impl PhysicalAddress { pub unsafe fn data(&self) -> usize { self.0 } - - pub const fn aligned(&self) -> bool { - self.0 & PAGE_OFFSET_MASK == 0 - } } // Virtual memory address @@ -42,10 +38,6 @@ impl VirtualAddress { pub unsafe fn data(&self) -> usize { self.0 } - - pub const fn aligned(&self) -> bool { - self.0 & PAGE_OFFSET_MASK == 0 - } } pub struct MemoryArea { diff --git a/src/main.rs b/src/main.rs index 31a11fd698..b43058d287 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,12 @@ -use rmm::*; +use rmm::{ + Arch, + EmulateArch, + PageTable +}; unsafe fn dump_tables(table: PageTable) { let level = table.level(); - for i in 0..PAGE_ENTRIES { + for i in 0..A::PAGE_ENTRIES { if level == 0 { if let Some(entry) = table.entry(i) { if entry.present() { diff --git a/src/page/entry.rs b/src/page/entry.rs index 7b8c7e8148..ba219bde81 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -1,27 +1,30 @@ +use core::marker::PhantomData; + use crate::{ + Arch, PhysicalAddress, - ENTRY_ADDRESS_MASK, - ENTRY_FLAG_PRESENT, }; #[derive(Clone, Copy, Debug)] -#[repr(transparent)] -pub struct PageEntry(usize); +pub struct PageEntry { + data: usize, + phantom: PhantomData, +} -impl PageEntry { +impl PageEntry { pub unsafe fn new(data: usize) -> Self { - Self(data) + Self { data, phantom: PhantomData } } pub fn data(&self) -> usize { - self.0 + self.data } pub fn address(&self) -> PhysicalAddress { - PhysicalAddress(self.0 & ENTRY_ADDRESS_MASK) + PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK) } pub fn present(&self) -> bool { - self.0 & ENTRY_FLAG_PRESENT != 0 + self.data & A::ENTRY_FLAG_PRESENT != 0 } } diff --git a/src/page/table.rs b/src/page/table.rs index 478077d83d..ab77a39c2d 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -4,12 +4,6 @@ use crate::{ Arch, PhysicalAddress, VirtualAddress, - PAGE_ENTRIES, - PAGE_ENTRY_MASK, - PAGE_ENTRY_SHIFT, - PAGE_ENTRY_SIZE, - PAGE_LEVELS, - PAGE_SHIFT, }; use super::PageEntry; @@ -29,7 +23,7 @@ impl PageTable { Self::new( VirtualAddress::new(0), PhysicalAddress::new(A::table()), - PAGE_LEVELS - 1 + A::PAGE_LEVELS - 1 ) } @@ -48,10 +42,10 @@ impl PageTable { pub unsafe fn virt(&self) -> VirtualAddress { // Recursive mapping let mut addr = 0xFFFF_FFFF_FFFF_F000; - for level in (self.level + 1 .. PAGE_LEVELS).rev() { - let index = (self.base.0 >> (level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) & PAGE_ENTRY_MASK; - addr <<= PAGE_ENTRY_SHIFT; - addr |= index << PAGE_SHIFT; + for level in (self.level + 1 .. A::PAGE_LEVELS).rev() { + let index = (self.base.0 >> (level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) & A::PAGE_ENTRY_MASK; + addr <<= A::PAGE_ENTRY_SHIFT; + addr |= index << A::PAGE_SHIFT; } VirtualAddress(addr) @@ -60,9 +54,9 @@ impl PageTable { } pub fn entry_base(&self, i: usize) -> Option { - if i < PAGE_ENTRIES { + if i < A::PAGE_ENTRIES { Some(VirtualAddress( - self.base.0 + (i << (self.level * PAGE_ENTRY_SHIFT + PAGE_SHIFT)) + self.base.0 + (i << (self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) )) } else { None @@ -70,21 +64,21 @@ impl PageTable { } pub unsafe fn entry_virt(&self, i: usize) -> Option { - if i < PAGE_ENTRIES { + if i < A::PAGE_ENTRIES { Some(VirtualAddress( - self.virt().0 + i * PAGE_ENTRY_SIZE + self.virt().0 + i * A::PAGE_ENTRY_SIZE )) } else { None } } - pub unsafe fn entry(&self, i: usize) -> Option { + pub unsafe fn entry(&self, i: usize) -> Option> { let addr = self.entry_virt(i)?; Some(PageEntry::new(A::read::(addr.0))) } - pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { + pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { let addr = self.entry_virt(i)?; A::write::(addr.0, entry.data()); Some(()) From 2df212f48d027f52f36200aa2dd0fd69316fcdcc Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 13:00:01 -0600 Subject: [PATCH 011/120] Add tests to ensure correct constant calculation --- src/arch/mod.rs | 5 ++++- src/arch/x86_64.rs | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index c7a8bf0bb9..3b9075fb39 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -19,11 +19,14 @@ pub trait Arch { const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT; const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1; - const PAGE_ADDRESS_MASK: usize = !Self::PAGE_OFFSET_MASK; + const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT; + const PAGE_ADDRESS_SIZE: usize = 1 << Self::PAGE_ADDRESS_SHIFT; + const PAGE_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_SIZE - 1 - Self::PAGE_OFFSET_MASK; const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; + //TODO: allow this to differ from page address mask const ENTRY_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_MASK; const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 3554304759..41d674d949 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -45,3 +45,24 @@ impl Arch for X8664Arch { unimplemented!(); } } + +#[cfg(test)] +mod tests { + use crate::Arch; + use super::X8664Arch; + + #[test] + fn constants() { + assert_eq!(X8664Arch::PAGE_SIZE, 4096); + assert_eq!(X8664Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(X8664Arch::PAGE_ADDRESS_SHIFT, 48); + assert_eq!(X8664Arch::PAGE_ADDRESS_SIZE, 0x0001_0000_0000_0000); + assert_eq!(X8664Arch::PAGE_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(X8664Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(X8664Arch::PAGE_ENTRIES, 512); + assert_eq!(X8664Arch::PAGE_ENTRY_MASK, 0x1FF); + + assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFFF_0000_0000_0FFF); + } +} From 023bb5811a38126d37dff00542e5594ebc64a53a Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 13:25:21 -0600 Subject: [PATCH 012/120] Add definition of physical address size --- src/arch/emulate.rs | 1 + src/arch/mod.rs | 7 ++++--- src/arch/x86_64.rs | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 2e290b6810..c4210d631e 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -19,6 +19,7 @@ impl Arch for EmulateArch { const PAGE_ENTRY_SHIFT: usize = X8664Arch::PAGE_ENTRY_SHIFT; const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS; + const ENTRY_ADDRESS_SHIFT: usize = X8664Arch::ENTRY_ADDRESS_SHIFT; const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_WRITABLE: usize = X8664Arch::ENTRY_FLAG_WRITABLE; const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 3b9075fb39..1dbd31a096 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -10,6 +10,7 @@ pub trait Arch { const PAGE_ENTRY_SHIFT: usize; const PAGE_LEVELS: usize; + const ENTRY_ADDRESS_SHIFT: usize; const ENTRY_FLAG_PRESENT: usize; const ENTRY_FLAG_WRITABLE: usize; const ENTRY_FLAG_USER: usize; @@ -21,13 +22,13 @@ pub trait Arch { const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1; const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT; const PAGE_ADDRESS_SIZE: usize = 1 << Self::PAGE_ADDRESS_SHIFT; - const PAGE_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_SIZE - 1 - Self::PAGE_OFFSET_MASK; + const PAGE_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_SIZE - Self::PAGE_SIZE; const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; - //TODO: allow this to differ from page address mask - const ENTRY_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_MASK; + const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_SHIFT; + const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - Self::PAGE_SIZE; const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; unsafe fn init() -> &'static [MemoryArea]; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 41d674d949..6197e7cf54 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -10,6 +10,7 @@ impl Arch for X8664Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + const ENTRY_ADDRESS_SHIFT: usize = 52; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_WRITABLE: usize = 1 << 1; const ENTRY_FLAG_USER: usize = 1 << 2; @@ -62,7 +63,8 @@ mod tests { assert_eq!(X8664Arch::PAGE_ENTRIES, 512); assert_eq!(X8664Arch::PAGE_ENTRY_MASK, 0x1FF); - assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); - assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFFF_0000_0000_0FFF); + assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); + assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); } } From 8efc425b07102c5391b55cce3497c6a4d5fe34e5 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 13:54:58 -0600 Subject: [PATCH 013/120] Add phys_to_virt function --- src/arch/emulate.rs | 1 + src/arch/mod.rs | 12 +++++++++++- src/arch/x86_64.rs | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index c4210d631e..efdde5a80b 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -18,6 +18,7 @@ impl Arch for EmulateArch { const PAGE_SHIFT: usize = X8664Arch::PAGE_SHIFT; const PAGE_ENTRY_SHIFT: usize = X8664Arch::PAGE_ENTRY_SHIFT; const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS; + const PAGE_OFFSET: usize = X8664Arch::PAGE_OFFSET; const ENTRY_ADDRESS_SHIFT: usize = X8664Arch::ENTRY_ADDRESS_SHIFT; const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 1dbd31a096..a404f99733 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,6 +1,10 @@ use core::ptr; -use crate::MemoryArea; +use crate::{ + MemoryArea, + PhysicalAddress, + VirtualAddress, +}; pub mod emulate; pub mod x86_64; @@ -9,6 +13,7 @@ pub trait Arch { const PAGE_SHIFT: usize; const PAGE_ENTRY_SHIFT: usize; const PAGE_LEVELS: usize; + const PAGE_OFFSET: usize; const ENTRY_ADDRESS_SHIFT: usize; const ENTRY_FLAG_PRESENT: usize; @@ -26,6 +31,7 @@ pub trait Arch { const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; + const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1); const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_SHIFT; const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - Self::PAGE_SIZE; @@ -50,4 +56,8 @@ pub trait Arch { unsafe fn table() -> usize; unsafe fn set_table(address: usize); + + unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { + VirtualAddress::new(phys.data() + Self::PAGE_OFFSET) + } } diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 6197e7cf54..bfba675a1e 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -9,6 +9,7 @@ impl Arch for X8664Arch { const PAGE_SHIFT: usize = 12; // 4096 bytes const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + const PAGE_OFFSET: usize = Self::PAGE_NEGATIVE_MASK; // PML4 slot 256 and onwards const ENTRY_ADDRESS_SHIFT: usize = 52; const ENTRY_FLAG_PRESENT: usize = 1 << 0; @@ -62,6 +63,7 @@ mod tests { assert_eq!(X8664Arch::PAGE_ENTRY_SIZE, 8); assert_eq!(X8664Arch::PAGE_ENTRIES, 512); assert_eq!(X8664Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(X8664Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); From ab07997bbe15c5b9027b1d7fc1012281c32d7e28 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 13:55:38 -0600 Subject: [PATCH 014/120] Add derives for physical and virtual addresses --- src/lib.rs | 8 ++++---- src/page/entry.rs | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 72417fe1cc..2c43d12c0e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ mod arch; mod page; // Physical memory address -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct PhysicalAddress(usize); @@ -20,13 +20,13 @@ impl PhysicalAddress { Self(address) } - pub unsafe fn data(&self) -> usize { + pub fn data(&self) -> usize { self.0 } } // Virtual memory address -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct VirtualAddress(usize); @@ -35,7 +35,7 @@ impl VirtualAddress { Self(address) } - pub unsafe fn data(&self) -> usize { + pub fn data(&self) -> usize { self.0 } } diff --git a/src/page/entry.rs b/src/page/entry.rs index ba219bde81..f935ba287d 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -12,7 +12,7 @@ pub struct PageEntry { } impl PageEntry { - pub unsafe fn new(data: usize) -> Self { + pub fn new(data: usize) -> Self { Self { data, phantom: PhantomData } } @@ -24,6 +24,10 @@ impl PageEntry { PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK) } + pub fn flags(&self) -> usize { + self.data & A::ENTRY_FLAGS_MASK + } + pub fn present(&self) -> bool { self.data & A::ENTRY_FLAG_PRESENT != 0 } From a482e506c5a33eab94acba680bd713c0df5f80e8 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 13:58:40 -0600 Subject: [PATCH 015/120] Use physical and virtual address abstraction for arch functions --- src/arch/emulate.rs | 91 ++++++++++++++++++++++++--------------------- src/arch/mod.rs | 14 +++---- src/arch/x86_64.rs | 10 +++-- src/main.rs | 9 +++-- src/page/table.rs | 16 ++++---- 5 files changed, 75 insertions(+), 65 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index efdde5a80b..331c81fe8e 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -7,8 +7,10 @@ use std::collections::BTreeMap; use crate::{ Arch, - PhysicalAddress, MemoryArea, + PageEntry, + PhysicalAddress, + VirtualAddress, arch::x86_64::X8664Arch, }; @@ -37,45 +39,45 @@ impl Arch for EmulateArch { let pml4 = 0; let pdp = pml4 + Self::PAGE_SIZE; let flags = Self::ENTRY_FLAG_WRITABLE | Self::ENTRY_FLAG_PRESENT; - machine.write_phys::(pml4, pdp | flags); + machine.write_phys::(PhysicalAddress::new(pml4), pdp | flags); // Recursive mapping - machine.write_phys::(pml4 + (Self::PAGE_ENTRIES - 1) * Self::PAGE_ENTRY_SIZE, pml4 | flags); + machine.write_phys::(PhysicalAddress::new(pml4 + (Self::PAGE_ENTRIES - 1) * Self::PAGE_ENTRY_SIZE), pml4 | flags); // PDP link to PD let pd = pdp + Self::PAGE_SIZE; - machine.write_phys::(pdp, pd | flags); + machine.write_phys::(PhysicalAddress::new(pdp), pd | flags); // PD link to PT let pt = pd + Self::PAGE_SIZE; - machine.write_phys::(pd, pt | flags); + machine.write_phys::(PhysicalAddress::new(pd), pt | flags); // PT links to frames for i in 0..Self::PAGE_ENTRIES { let page = i * Self::PAGE_SIZE; - machine.write_phys::(pt + i * Self::PAGE_ENTRY_SIZE, page | flags); + machine.write_phys::(PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), page | flags); } MACHINE = Some(machine); // Set table to pml4 - EmulateArch::set_table(pml4); + EmulateArch::set_table(PhysicalAddress::new(pml4)); &MEMORY_AREAS } #[inline(always)] - unsafe fn read(address: usize) -> T { + unsafe fn read(address: VirtualAddress) -> T { MACHINE.as_ref().unwrap().read(address) } #[inline(always)] - unsafe fn write(address: usize, value: T) { + unsafe fn write(address: VirtualAddress, value: T) { MACHINE.as_mut().unwrap().write(address, value) } #[inline(always)] - unsafe fn invalidate(address: usize) { + unsafe fn invalidate(address: VirtualAddress) { MACHINE.as_mut().unwrap().invalidate(address); } @@ -85,12 +87,12 @@ impl Arch for EmulateArch { } #[inline(always)] - unsafe fn table() -> usize { + unsafe fn table() -> PhysicalAddress { MACHINE.as_mut().unwrap().get_table() } #[inline(always)] - unsafe fn set_table(address: usize) { + unsafe fn set_table(address: PhysicalAddress) { MACHINE.as_mut().unwrap().set_table(address); } } @@ -108,8 +110,8 @@ static mut MACHINE: Option> = None; struct Machine { memory: Box<[u8]>, - map: BTreeMap, - table_addr: usize, + map: BTreeMap>, + table_addr: PhysicalAddress, phantom: PhantomData, } @@ -118,12 +120,13 @@ impl Machine { Self { memory: vec![0; memory_size].into_boxed_slice(), map: BTreeMap::new(), - table_addr: 0, + table_addr: PhysicalAddress::new(0), phantom: PhantomData, } } - fn read_phys(&self, phys: usize) -> T { + fn read_phys(&self, phys: PhysicalAddress) -> T { + let phys = phys.data(); let size = mem::size_of::(); if phys + size <= self.memory.len() { unsafe { @@ -134,7 +137,8 @@ impl Machine { } } - fn write_phys(&mut self, phys: usize, value: T) { + fn write_phys(&mut self, phys: PhysicalAddress, value: T) { + let phys = phys.data(); let size = mem::size_of::(); if phys + size <= self.memory.len() { unsafe { @@ -145,104 +149,107 @@ impl Machine { } } - fn translate(&self, virt: usize) -> Option<(usize, usize)> { - let page = virt & A::PAGE_ADDRESS_MASK; - let offset = virt & A::PAGE_OFFSET_MASK; - let phys = self.map.get(&page)?; + fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, usize)> { + let virt_data = virt.data(); + let page = virt_data & A::PAGE_ADDRESS_MASK; + let offset = virt_data & A::PAGE_OFFSET_MASK; + let entry = self.map.get(&VirtualAddress::new(page))?; Some(( - (phys & A::ENTRY_ADDRESS_MASK) + offset, - phys & A::ENTRY_FLAGS_MASK, + PhysicalAddress::new(entry.address().data() + offset), + entry.flags(), )) } - fn read(&self, virt: usize) -> T { + fn read(&self, virt: VirtualAddress) -> T { //TODO: allow reading past page boundaries + let virt_data = virt.data(); let size = mem::size_of::(); - if (virt & A::PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt, size); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { + panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt_data, size); } if let Some((phys, _flags)) = self.translate(virt) { self.read_phys(phys) } else { - panic!("read: 0x{:X} size 0x{:X} not present", virt, size); + panic!("read: 0x{:X} size 0x{:X} not present", virt_data, size); } } - fn write(&mut self, virt: usize, value: T) { + fn write(&mut self, virt: VirtualAddress, value: T) { //TODO: allow writing past page boundaries + let virt_data = virt.data(); let size = mem::size_of::(); - if (virt & A::PAGE_ADDRESS_MASK) != ((virt + (size - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt, size); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { + panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt_data, size); } if let Some((phys, flags)) = self.translate(virt) { if flags & A::ENTRY_FLAG_WRITABLE != 0 { self.write_phys(phys, value); } else { - panic!("write: 0x{:X} size 0x{:X} not writable", virt, size); + panic!("write: 0x{:X} size 0x{:X} not writable", virt_data, size); } } else { - panic!("write: 0x{:X} size 0x{:X} not present", virt, size); + panic!("write: 0x{:X} size 0x{:X} not present", virt_data, size); } } - fn invalidate(&mut self, _address: usize) { + fn invalidate(&mut self, _address: VirtualAddress) { unimplemented!(); } + //TODO: cleanup fn invalidate_all(&mut self) { self.map.clear(); // PML4 - let a4 = self.table_addr; + let a4 = self.table_addr.data(); for i4 in 0..A::PAGE_ENTRIES { - let e3 = self.read_phys::(a4 + i4 * A::PAGE_ENTRY_SIZE); + let e3 = self.read_phys::(PhysicalAddress::new(a4 + i4 * A::PAGE_ENTRY_SIZE)); let f3 = e3 & A::ENTRY_FLAGS_MASK; if f3 & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page directory pointer let a3 = e3 & A::ENTRY_ADDRESS_MASK; for i3 in 0..A::PAGE_ENTRIES { - let e2 = self.read_phys::(a3 + i3 * A::PAGE_ENTRY_SIZE); + let e2 = self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); let f2 = e2 & A::ENTRY_FLAGS_MASK; if f2 & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page directory let a2 = e2 & A::ENTRY_ADDRESS_MASK; for i2 in 0..A::PAGE_ENTRIES { - let e1 = self.read_phys::(a2 + i2 * A::PAGE_ENTRY_SIZE); + let e1 = self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); let f1 = e1 & A::ENTRY_FLAGS_MASK; if f1 & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page table let a1 = e1 & A::ENTRY_ADDRESS_MASK; for i1 in 0..A::PAGE_ENTRIES { - let e = self.read_phys::(a1 + i1 * A::PAGE_ENTRY_SIZE); + let e = self.read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); let f = e & A::ENTRY_FLAGS_MASK; if f & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page let a = e & A::ENTRY_ADDRESS_MASK; let page = - if i4 >= 256 { 0xFFFF_0000_0000_0000 } else { 0 } | (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12); println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); - self.map.insert(page, e); + self.map.insert(VirtualAddress::new(page), PageEntry::new(e)); } } } } } - fn get_table(&self) -> usize { + fn get_table(&self) -> PhysicalAddress { self.table_addr } - fn set_table(&mut self, address: usize) { + fn set_table(&mut self, address: PhysicalAddress) { self.table_addr = address; self.invalidate_all(); } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index a404f99733..32f6b2a797 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -40,22 +40,22 @@ pub trait Arch { unsafe fn init() -> &'static [MemoryArea]; #[inline(always)] - unsafe fn read(address: usize) -> T { - ptr::read(address as *const T) + unsafe fn read(address: VirtualAddress) -> T { + ptr::read(address.data() as *const T) } #[inline(always)] - unsafe fn write(address: usize, value: T) { - ptr::write(address as *mut T, value) + unsafe fn write(address: VirtualAddress, value: T) { + ptr::write(address.data() as *mut T, value) } - unsafe fn invalidate(address: usize); + unsafe fn invalidate(address: VirtualAddress); unsafe fn invalidate_all(); - unsafe fn table() -> usize; + unsafe fn table() -> PhysicalAddress; - unsafe fn set_table(address: usize); + unsafe fn set_table(address: PhysicalAddress); unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { VirtualAddress::new(phys.data() + Self::PAGE_OFFSET) diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index bfba675a1e..5fbac9f49d 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -1,6 +1,8 @@ use crate::{ Arch, - MemoryArea + MemoryArea, + PhysicalAddress, + VirtualAddress, }; pub struct X8664Arch; @@ -24,7 +26,7 @@ impl Arch for X8664Arch { } #[inline(always)] - unsafe fn invalidate(address: usize) { + unsafe fn invalidate(address: VirtualAddress) { //TODO: invlpg address unimplemented!(); } @@ -36,13 +38,13 @@ impl Arch for X8664Arch { } #[inline(always)] - unsafe fn table() -> usize { + unsafe fn table() -> PhysicalAddress { //TODO: return cr3 unimplemented!(); } #[inline(always)] - unsafe fn set_table(address: usize) { + unsafe fn set_table(address: PhysicalAddress) { //TODO: mov cr3, address unimplemented!(); } diff --git a/src/main.rs b/src/main.rs index b43058d287..1fd59273a1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ use rmm::{ Arch, EmulateArch, - PageTable + PageTable, + VirtualAddress, }; unsafe fn dump_tables(table: PageTable) { @@ -28,16 +29,16 @@ unsafe fn inner() { // Debug table dump_tables(PageTable::::top()); - let megabyte = 0x100000; + let megabyte = VirtualAddress::new(0x100000); // Test read - println!("0x{:X} = 0x{:X}", megabyte, A::read::(megabyte)); + println!("0x{:X} = 0x{:X}", megabyte.data(), A::read::(megabyte)); // Test write A::write::(megabyte, 0x5A); // Test read - println!("0x{:X} = 0x{:X}", megabyte, A::read::(megabyte)); + println!("0x{:X} = 0x{:X}", megabyte.data(), A::read::(megabyte)); } fn main() { diff --git a/src/page/table.rs b/src/page/table.rs index ab77a39c2d..44113a9d26 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -22,7 +22,7 @@ impl PageTable { pub unsafe fn top() -> Self { Self::new( VirtualAddress::new(0), - PhysicalAddress::new(A::table()), + A::table(), A::PAGE_LEVELS - 1 ) } @@ -47,7 +47,7 @@ impl PageTable { addr <<= A::PAGE_ENTRY_SHIFT; addr |= index << A::PAGE_SHIFT; } - VirtualAddress(addr) + VirtualAddress::new(addr) // Identity mapping //VirtualAddress(self.phys.0) @@ -55,8 +55,8 @@ impl PageTable { pub fn entry_base(&self, i: usize) -> Option { if i < A::PAGE_ENTRIES { - Some(VirtualAddress( - self.base.0 + (i << (self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) + Some(VirtualAddress::new( + self.base.data() + (i << (self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) )) } else { None @@ -65,8 +65,8 @@ impl PageTable { pub unsafe fn entry_virt(&self, i: usize) -> Option { if i < A::PAGE_ENTRIES { - Some(VirtualAddress( - self.virt().0 + i * A::PAGE_ENTRY_SIZE + Some(VirtualAddress::new( + self.virt().data() + i * A::PAGE_ENTRY_SIZE )) } else { None @@ -75,12 +75,12 @@ impl PageTable { pub unsafe fn entry(&self, i: usize) -> Option> { let addr = self.entry_virt(i)?; - Some(PageEntry::new(A::read::(addr.0))) + Some(PageEntry::new(A::read::(addr))) } pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { let addr = self.entry_virt(i)?; - A::write::(addr.0, entry.data()); + A::write::(addr, entry.data()); Some(()) } From 5fc0a080ad4307a789688559f962a8813a3649d3 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 14:17:26 -0600 Subject: [PATCH 016/120] Update nightly --- rust-toolchain | 1 + 1 file changed, 1 insertion(+) create mode 100644 rust-toolchain diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 0000000000..40973dae63 --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +nightly-2020-07-27 From f801a40908471b123e4d0314858805f0794746bb Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 14:17:49 -0600 Subject: [PATCH 017/120] Add function for addresses --- src/arch/emulate.rs | 16 +++++++--------- src/lib.rs | 18 ++++++++++++++++-- src/page/entry.rs | 5 +++++ src/page/table.rs | 8 ++------ 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 331c81fe8e..40274c6633 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -126,26 +126,24 @@ impl Machine { } fn read_phys(&self, phys: PhysicalAddress) -> T { - let phys = phys.data(); let size = mem::size_of::(); - if phys + size <= self.memory.len() { + if phys.add(size).data() <= self.memory.len() { unsafe { - ptr::read(self.memory.as_ptr().add(phys) as *const T) + ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) } } else { - panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); + panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys.data(), size); } } fn write_phys(&mut self, phys: PhysicalAddress, value: T) { - let phys = phys.data(); let size = mem::size_of::(); - if phys + size <= self.memory.len() { + if phys.add(size).data() <= self.memory.len() { unsafe { - ptr::write(self.memory.as_mut_ptr().add(phys) as *mut T, value); + ptr::write(self.memory.as_mut_ptr().add(phys.data()) as *mut T, value); } } else { - panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys, size); + panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys.data(), size); } } @@ -155,7 +153,7 @@ impl Machine { let offset = virt_data & A::PAGE_OFFSET_MASK; let entry = self.map.get(&VirtualAddress::new(page))?; Some(( - PhysicalAddress::new(entry.address().data() + offset), + entry.address().add(offset), entry.flags(), )) } diff --git a/src/lib.rs b/src/lib.rs index 2c43d12c0e..b174223b35 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,13 +16,20 @@ mod page; pub struct PhysicalAddress(usize); impl PhysicalAddress { + #[inline(always)] pub const fn new(address: usize) -> Self { Self(address) } + #[inline(always)] pub fn data(&self) -> usize { self.0 } + + #[inline(always)] + pub fn add(self, offset: usize) -> Self { + Self(self.0 + offset) + } } // Virtual memory address @@ -31,16 +38,23 @@ impl PhysicalAddress { pub struct VirtualAddress(usize); impl VirtualAddress { + #[inline(always)] pub const fn new(address: usize) -> Self { Self(address) } + #[inline(always)] pub fn data(&self) -> usize { self.0 } + + #[inline(always)] + pub fn add(self, offset: usize) -> Self { + Self(self.0 + offset) + } } pub struct MemoryArea { - base: PhysicalAddress, - size: usize, + pub base: PhysicalAddress, + pub size: usize, } diff --git a/src/page/entry.rs b/src/page/entry.rs index f935ba287d..6732da4970 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -12,22 +12,27 @@ pub struct PageEntry { } impl PageEntry { + #[inline(always)] pub fn new(data: usize) -> Self { Self { data, phantom: PhantomData } } + #[inline(always)] pub fn data(&self) -> usize { self.data } + #[inline(always)] pub fn address(&self) -> PhysicalAddress { PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK) } + #[inline(always)] pub fn flags(&self) -> usize { self.data & A::ENTRY_FLAGS_MASK } + #[inline(always)] pub fn present(&self) -> bool { self.data & A::ENTRY_FLAG_PRESENT != 0 } diff --git a/src/page/table.rs b/src/page/table.rs index 44113a9d26..15a13f34d7 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -55,9 +55,7 @@ impl PageTable { pub fn entry_base(&self, i: usize) -> Option { if i < A::PAGE_ENTRIES { - Some(VirtualAddress::new( - self.base.data() + (i << (self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) - )) + Some(self.base.add(i << (self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT))) } else { None } @@ -65,9 +63,7 @@ impl PageTable { pub unsafe fn entry_virt(&self, i: usize) -> Option { if i < A::PAGE_ENTRIES { - Some(VirtualAddress::new( - self.virt().data() + i * A::PAGE_ENTRY_SIZE - )) + Some(self.virt().add(i * A::PAGE_ENTRY_SIZE)) } else { None } From 379294b59f6d628b8552edb9f74ae6e8507c1620 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 14:18:01 -0600 Subject: [PATCH 018/120] Implement x86_64 arch functions --- src/arch/x86_64.rs | 14 ++++++-------- src/lib.rs | 2 ++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 5fbac9f49d..d4e1ec6e3e 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -27,26 +27,24 @@ impl Arch for X8664Arch { #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - //TODO: invlpg address - unimplemented!(); + asm!("invlpg [{0}]", in(reg) address.data()); } #[inline(always)] unsafe fn invalidate_all() { - //TODO: mov cr3, cr3 - unimplemented!(); + Self::set_table(Self::table()); } #[inline(always)] unsafe fn table() -> PhysicalAddress { - //TODO: return cr3 - unimplemented!(); + let address: usize; + asm!("mov rax, cr3", out("rax") address); + PhysicalAddress::new(address) } #[inline(always)] unsafe fn set_table(address: PhysicalAddress) { - //TODO: mov cr3, address - unimplemented!(); + asm!("mov cr3, rax", in("rax") address.data()); } } diff --git a/src/lib.rs b/src/lib.rs index b174223b35..9cf7b30597 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(asm)] + pub use crate::{ arch::Arch, arch::emulate::*, From a40b26293ba4efec7b3960d97815907649a9d162 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 14:20:29 -0600 Subject: [PATCH 019/120] Do not use explicit register fo x86_64 table operations --- src/arch/mod.rs | 5 ++++- src/arch/x86_64.rs | 9 ++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 32f6b2a797..a53d7e8a92 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -51,7 +51,10 @@ pub trait Arch { unsafe fn invalidate(address: VirtualAddress); - unsafe fn invalidate_all(); + #[inline(always)] + unsafe fn invalidate_all() { + Self::set_table(Self::table()); + } unsafe fn table() -> PhysicalAddress; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index d4e1ec6e3e..a718b273e8 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -30,21 +30,16 @@ impl Arch for X8664Arch { asm!("invlpg [{0}]", in(reg) address.data()); } - #[inline(always)] - unsafe fn invalidate_all() { - Self::set_table(Self::table()); - } - #[inline(always)] unsafe fn table() -> PhysicalAddress { let address: usize; - asm!("mov rax, cr3", out("rax") address); + asm!("mov {0}, cr3", out(reg) address); PhysicalAddress::new(address) } #[inline(always)] unsafe fn set_table(address: PhysicalAddress) { - asm!("mov cr3, rax", in("rax") address.data()); + asm!("mov cr3, {0}", in(reg) address.data()); } } From d5111d429e751392e79d6944d1cc173945f25e4c Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 14:30:50 -0600 Subject: [PATCH 020/120] Implement PHYS_OFFSET mapping --- src/arch/emulate.rs | 7 ++++--- src/arch/mod.rs | 5 +++-- src/arch/x86_64.rs | 5 ++++- src/page/table.rs | 19 +++++++++---------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 40274c6633..6cf7c9d606 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -20,7 +20,6 @@ impl Arch for EmulateArch { const PAGE_SHIFT: usize = X8664Arch::PAGE_SHIFT; const PAGE_ENTRY_SHIFT: usize = X8664Arch::PAGE_ENTRY_SHIFT; const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS; - const PAGE_OFFSET: usize = X8664Arch::PAGE_OFFSET; const ENTRY_ADDRESS_SHIFT: usize = X8664Arch::ENTRY_ADDRESS_SHIFT; const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; @@ -30,6 +29,8 @@ impl Arch for EmulateArch { const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; + const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages identity mapped (2 MiB on x86_64) // Pages over 1 MiB will be mapped writable @@ -41,8 +42,8 @@ impl Arch for EmulateArch { let flags = Self::ENTRY_FLAG_WRITABLE | Self::ENTRY_FLAG_PRESENT; machine.write_phys::(PhysicalAddress::new(pml4), pdp | flags); - // Recursive mapping - machine.write_phys::(PhysicalAddress::new(pml4 + (Self::PAGE_ENTRIES - 1) * Self::PAGE_ENTRY_SIZE), pml4 | flags); + // PML4 index 256, set to PDP again for PHYS_OFFSET mapping + machine.write_phys::(PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), pdp | flags); // PDP link to PD let pd = pdp + Self::PAGE_SIZE; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index a53d7e8a92..21a88f6368 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -13,7 +13,6 @@ pub trait Arch { const PAGE_SHIFT: usize; const PAGE_ENTRY_SHIFT: usize; const PAGE_LEVELS: usize; - const PAGE_OFFSET: usize; const ENTRY_ADDRESS_SHIFT: usize; const ENTRY_FLAG_PRESENT: usize; @@ -23,6 +22,8 @@ pub trait Arch { const ENTRY_FLAG_GLOBAL: usize; const ENTRY_FLAG_NO_EXEC: usize; + const PHYS_OFFSET: usize; + const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT; const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1; const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT; @@ -61,6 +62,6 @@ pub trait Arch { unsafe fn set_table(address: PhysicalAddress); unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { - VirtualAddress::new(phys.data() + Self::PAGE_OFFSET) + VirtualAddress::new(phys.data() + Self::PHYS_OFFSET) } } diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index a718b273e8..f5d3bf084b 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -11,7 +11,6 @@ impl Arch for X8664Arch { const PAGE_SHIFT: usize = 12; // 4096 bytes const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT - const PAGE_OFFSET: usize = Self::PAGE_NEGATIVE_MASK; // PML4 slot 256 and onwards const ENTRY_ADDRESS_SHIFT: usize = 52; const ENTRY_FLAG_PRESENT: usize = 1 << 0; @@ -21,6 +20,8 @@ impl Arch for X8664Arch { const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; + const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards + unsafe fn init() -> &'static [MemoryArea] { unimplemented!() } @@ -63,5 +64,7 @@ mod tests { assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + + assert_eq!(X8664Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } } diff --git a/src/page/table.rs b/src/page/table.rs index 15a13f34d7..38691e0e3f 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -40,17 +40,16 @@ impl PageTable { } pub unsafe fn virt(&self) -> VirtualAddress { - // Recursive mapping - let mut addr = 0xFFFF_FFFF_FFFF_F000; - for level in (self.level + 1 .. A::PAGE_LEVELS).rev() { - let index = (self.base.0 >> (level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) & A::PAGE_ENTRY_MASK; - addr <<= A::PAGE_ENTRY_SHIFT; - addr |= index << A::PAGE_SHIFT; - } - VirtualAddress::new(addr) + A::phys_to_virt(self.phys) - // Identity mapping - //VirtualAddress(self.phys.0) + // Recursive mapping + // let mut addr = 0xFFFF_FFFF_FFFF_F000; + // for level in (self.level + 1 .. A::PAGE_LEVELS).rev() { + // let index = (self.base.0 >> (level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) & A::PAGE_ENTRY_MASK; + // addr <<= A::PAGE_ENTRY_SHIFT; + // addr |= index << A::PAGE_SHIFT; + // } + // VirtualAddress::new(addr) } pub fn entry_base(&self, i: usize) -> Option { From 4bf43652c32dbde7e39e3fba75df5bb73736daae Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 14:38:31 -0600 Subject: [PATCH 021/120] Only use offset mapping --- src/arch/emulate.rs | 10 +++------- src/main.rs | 12 +++++++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 6cf7c9d606..642b23228c 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -32,17 +32,13 @@ impl Arch for EmulateArch { const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; unsafe fn init() -> &'static [MemoryArea] { - // Create machine with PAGE_ENTRIES pages identity mapped (2 MiB on x86_64) - // Pages over 1 MiB will be mapped writable + // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); - // PML4 link to PDP + // PML4 index 256 (PHYS_OFFSET) link to PDP let pml4 = 0; let pdp = pml4 + Self::PAGE_SIZE; let flags = Self::ENTRY_FLAG_WRITABLE | Self::ENTRY_FLAG_PRESENT; - machine.write_phys::(PhysicalAddress::new(pml4), pdp | flags); - - // PML4 index 256, set to PDP again for PHYS_OFFSET mapping machine.write_phys::(PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), pdp | flags); // PDP link to PD @@ -102,7 +98,7 @@ const MEGABYTE: usize = 1024 * 1024; const MEMORY_SIZE: usize = 64 * MEGABYTE; static MEMORY_AREAS: [MemoryArea; 1] = [ MemoryArea { - base: PhysicalAddress::new(MEGABYTE), + base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted size: MEMORY_SIZE, } ]; diff --git a/src/main.rs b/src/main.rs index 1fd59273a1..bb41b46c7f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,9 @@ use rmm::{ Arch, EmulateArch, + MemoryArea, PageTable, + PhysicalAddress, VirtualAddress, }; @@ -23,13 +25,21 @@ unsafe fn dump_tables(table: PageTable) { } } +unsafe fn new_tables(areas: &[MemoryArea]) { + +} + unsafe fn inner() { let areas = A::init(); // Debug table dump_tables(PageTable::::top()); - let megabyte = VirtualAddress::new(0x100000); + new_tables::(areas); + + dump_tables(PageTable::::top()); + + let megabyte = A::phys_to_virt(PhysicalAddress::new(0x100000)); // Test read println!("0x{:X} = 0x{:X}", megabyte.data(), A::read::(megabyte)); From 2dc085390326d62a7a596b75e2b764c24bef6e4c Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 21:06:12 -0600 Subject: [PATCH 022/120] Add constants for sizes --- src/arch/emulate.rs | 2 +- src/lib.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 642b23228c..db4b8c2f96 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -6,6 +6,7 @@ use core::{ use std::collections::BTreeMap; use crate::{ + MEGABYTE, Arch, MemoryArea, PageEntry, @@ -94,7 +95,6 @@ impl Arch for EmulateArch { } } -const MEGABYTE: usize = 1024 * 1024; const MEMORY_SIZE: usize = 64 * MEGABYTE; static MEMORY_AREAS: [MemoryArea; 1] = [ MemoryArea { diff --git a/src/lib.rs b/src/lib.rs index 9cf7b30597..b34ed7e15b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,11 @@ pub use crate::{ mod arch; mod page; +pub const KILOBYTE: usize = 1024; +pub const MEGABYTE: usize = KILOBYTE * KILOBYTE; +pub const GIGABYTE: usize = KILOBYTE * MEGABYTE; +pub const TERABYTE: usize = KILOBYTE * GIGABYTE; + // Physical memory address #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] #[repr(transparent)] From d9500dc24e578a431d2fffdf5442cefb0fe8b32b Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 21:06:41 -0600 Subject: [PATCH 023/120] Fix excess size of emulated memory, do not print emulated mapping --- src/arch/emulate.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index db4b8c2f96..478a8e7c1a 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -99,7 +99,7 @@ const MEMORY_SIZE: usize = 64 * MEGABYTE; static MEMORY_AREAS: [MemoryArea; 1] = [ MemoryArea { base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted - size: MEMORY_SIZE, + size: MEMORY_SIZE - EmulateArch::PAGE_SIZE * 4, } ]; @@ -232,7 +232,7 @@ impl Machine { (i3 << 30) | (i2 << 21) | (i1 << 12); - println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); + //println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); self.map.insert(VirtualAddress::new(page), PageEntry::new(e)); } } From 03d326408550d2beacd1d7c380ac07f2a6a1fccb Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 21:06:58 -0600 Subject: [PATCH 024/120] Add index_of function to PageTable --- src/page/table.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/page/table.rs b/src/page/table.rs index 38691e0e3f..23a8fa1293 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -54,7 +54,8 @@ impl PageTable { pub fn entry_base(&self, i: usize) -> Option { if i < A::PAGE_ENTRIES { - Some(self.base.add(i << (self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT))) + let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + Some(self.base.add(i << level_shift)) } else { None } @@ -79,6 +80,18 @@ impl PageTable { Some(()) } + pub unsafe fn index_of(&self, address: VirtualAddress) -> Option { + // Canonicalize address first + let address = VirtualAddress::new(address.data() & A::PAGE_ADDRESS_MASK); + let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_mask = (A::PAGE_ENTRIES << level_shift) - 1; + if address >= self.base && address <= self.base.add(level_mask) { + Some((address.data() >> level_shift) & A::PAGE_ENTRY_MASK) + } else { + None + } + } + pub unsafe fn next(&self, i: usize) -> Option { if self.level > 0 { let entry = self.entry(i)?; From d450cafad61a5bbc048c97f5f89aaceef5a1f511 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 6 Sep 2020 21:07:14 -0600 Subject: [PATCH 025/120] Add mapping functionality --- src/main.rs | 143 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index bb41b46c7f..7c8a2287b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,33 @@ use rmm::{ + KILOBYTE, + MEGABYTE, + GIGABYTE, + TERABYTE, Arch, EmulateArch, MemoryArea, + PageEntry, PageTable, PhysicalAddress, VirtualAddress, }; +use core::marker::PhantomData; + +pub fn format_size(size: usize) -> String { + if size >= 2 * TERABYTE { + format!("{} TB", size / TERABYTE) + } else if size >= 2 * GIGABYTE { + format!("{} GB", size / GIGABYTE) + } else if size >= 2 * MEGABYTE { + format!("{} MB", size / MEGABYTE) + } else if size >= 2 * KILOBYTE { + format!("{} KB", size / KILOBYTE) + } else { + format!("{} B", size) + } +} + unsafe fn dump_tables(table: PageTable) { let level = table.level(); for i in 0..A::PAGE_ENTRIES { @@ -25,30 +46,132 @@ unsafe fn dump_tables(table: PageTable) { } } -unsafe fn new_tables(areas: &[MemoryArea]) { +pub struct BumpAllocator { + areas: &'static [MemoryArea], + offset: usize, + phantom: PhantomData, +} +impl BumpAllocator { + pub fn new(areas: &'static [MemoryArea]) -> Self { + Self { + areas, + offset: 0, + phantom: PhantomData, + } + } + + pub fn allocate(&mut self) -> Option { + let mut offset = self.offset; + for area in self.areas.iter() { + if offset < area.size { + self.offset += A::PAGE_SIZE; + return Some(area.base.add(offset)); + } + offset -= area.size; + } + None + } +} + +pub struct Mapper { + table_addr: PhysicalAddress, + allocator: BumpAllocator, +} + +impl Mapper { + pub unsafe fn new(mut allocator: BumpAllocator) -> Option { + let table_addr = allocator.allocate()?; + Some(Self { + table_addr, + allocator, + }) + } + + pub unsafe fn map(&mut self, virt: VirtualAddress, entry: PageEntry) -> Option<()> { + let mut table = PageTable::new( + VirtualAddress::new(0), + self.table_addr, + A::PAGE_LEVELS - 1 + ); + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + //TODO: check for overwriting entry + table.set_entry(i, entry); + return Some(()); + } else { + let next_opt = table.next(i); + let next = match next_opt { + Some(some) => some, + None => { + let phys = self.allocator.allocate()?; + //TODO: correct flags? + table.set_entry(i, PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); + table.next(i)? + } + }; + table = next; + } + } + } +} + +unsafe fn new_tables(areas: &'static [MemoryArea]) { + // First, calculate how much memory we have + let mut size = 0; + for area in areas.iter() { + size += area.size; + } + + println!("Memory: {}", format_size(size)); + + // Create a basic allocator for the first pages + let allocator = BumpAllocator::::new(areas); + + // Map all physical areas at PHYS_OFFSET + let mut mapper = Mapper::new(allocator).expect("failed to create Mapper"); + for area in areas.iter() { + for i in 0..area.size / A::PAGE_SIZE { + let phys = area.base.add(i * A::PAGE_SIZE); + let virt = A::phys_to_virt(phys); + mapper.map( + virt, + PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT) + ).expect("failed to map frame"); + } + } + + A::set_table(mapper.table_addr); + + let used = mapper.allocator.offset; + println!("Used: {}", format_size(used)); } unsafe fn inner() { let areas = A::init(); // Debug table - dump_tables(PageTable::::top()); + //dump_tables(PageTable::::top()); new_tables::(areas); - dump_tables(PageTable::::top()); + //dump_tables(PageTable::::top()); - let megabyte = A::phys_to_virt(PhysicalAddress::new(0x100000)); - // Test read - println!("0x{:X} = 0x{:X}", megabyte.data(), A::read::(megabyte)); + for i in &[1, 2, 4, 8, 16, 32] { + let phys = PhysicalAddress::new(i * MEGABYTE); + let virt = A::phys_to_virt(phys); - // Test write - A::write::(megabyte, 0x5A); + // Test read + println!("0x{:X} (0x{:X}) = 0x{:X}", virt.data(), phys.data(), A::read::(virt)); - // Test read - println!("0x{:X} = 0x{:X}", megabyte.data(), A::read::(megabyte)); + // Test write + A::write::(virt, 0x5A); + + // Test read + println!("0x{:X} (0x{:X}) = 0x{:X}", virt.data(), phys.data(), A::read::(virt)); + } } fn main() { From 011212905d6dc0b643b84989c4c7bca230037c77 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 7 Sep 2020 20:01:40 -0600 Subject: [PATCH 026/120] Add WIP slab allocator --- src/arch/emulate.rs | 1 - src/main.rs | 147 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 6 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 478a8e7c1a..5106c4561d 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -226,7 +226,6 @@ impl Machine { if f & A::ENTRY_FLAG_PRESENT == 0 { continue; } // Page - let a = e & A::ENTRY_ADDRESS_MASK; let page = (i4 << 39) | (i3 << 30) | diff --git a/src/main.rs b/src/main.rs index 7c8a2287b1..a331c3dfd5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,10 +53,10 @@ pub struct BumpAllocator { } impl BumpAllocator { - pub fn new(areas: &'static [MemoryArea]) -> Self { + pub fn new(areas: &'static [MemoryArea], offset: usize) -> Self { Self { areas, - offset: 0, + offset, phantom: PhantomData, } } @@ -74,6 +74,119 @@ impl BumpAllocator { } } +pub struct SlabNode { + next: PhysicalAddress, + count: usize, + phantom: PhantomData, +} + +impl SlabNode { + pub fn new(next: PhysicalAddress, count: usize) -> Self { + Self { + next, + count, + phantom: PhantomData, + } + } + + pub fn empty() -> Self { + Self::new(PhysicalAddress::new(0), 0) + } + + pub unsafe fn insert(&mut self, phys: PhysicalAddress) { + let virt = A::phys_to_virt(phys); + A::write(virt, self.next); + self.next = phys; + self.count += 1; + } + + pub unsafe fn remove(&mut self) -> Option { + if self.count > 0 { + let phys = self.next; + let virt = A::phys_to_virt(phys); + self.next = A::read(virt); + self.count -= 1; + Some(phys) + } else { + None + } + } +} + +pub struct SlabAllocator { + //TODO: Allow allocations up to maximum pageable size + nodes: [SlabNode; 4], + phantom: PhantomData, +} + +impl SlabAllocator { + pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Self { + let mut allocator = Self { + nodes: [ + SlabNode::empty(), + SlabNode::empty(), + SlabNode::empty(), + SlabNode::empty(), + ], + phantom: PhantomData, + }; + + // Add unused areas to free lists + let mut area_offset = offset; + for area in areas.iter() { + if area_offset < area.size { + area_offset = 0; + let area_base = area.base.add(offset); + let area_size = area.size - offset; + allocator.free(area_base, area_size); + } else { + area_offset -= area.size; + } + } + + allocator + } + + pub unsafe fn allocate(&mut self, size: usize) -> Option { + for level in 0..A::PAGE_LEVELS - 1 { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + if size <= level_size { + if let Some(base) = self.nodes[level].remove() { + self.free(base.add(size), level_size - size); + return Some(base); + } + } + } + None + } + + //TODO: This causes fragmentation, since neighbors are not identified + //TODO: remainders less than PAGE_SIZE will be lost + pub unsafe fn free(&mut self, mut base: PhysicalAddress, mut size: usize) { + for level in (0..A::PAGE_LEVELS - 1).rev() { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + while size >= level_size { + println!("Add {:X} {}", base.data(), format_size(level_size)); + self.nodes[level].insert(base); + base = base.add(level_size); + size -= level_size; + } + } + } + + pub unsafe fn remaining(&mut self) -> usize { + let mut remaining = 0; + for level in (0..A::PAGE_LEVELS - 1).rev() { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + remaining += self.nodes[level].count * level_size; + } + remaining + } +} + pub struct Mapper { table_addr: PhysicalAddress, allocator: BumpAllocator, @@ -127,7 +240,7 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { println!("Memory: {}", format_size(size)); // Create a basic allocator for the first pages - let allocator = BumpAllocator::::new(areas); + let allocator = BumpAllocator::::new(areas, 0); // Map all physical areas at PHYS_OFFSET let mut mapper = Mapper::new(allocator).expect("failed to create Mapper"); @@ -142,10 +255,34 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { } } + // Use the new table A::set_table(mapper.table_addr); - let used = mapper.allocator.offset; - println!("Used: {}", format_size(used)); + // Create the physical memory map + let offset = mapper.allocator.offset; + println!("Permanently used: {}", format_size(offset)); + + let mut allocator = SlabAllocator::::new(areas, offset); + for i in 0..16 { + let phys_opt = allocator.allocate(4 * KILOBYTE); + println!("4 KB page {}: {:X?}", i, phys_opt); + if i % 2 == 0 { + if let Some(phys) = phys_opt { + allocator.free(phys, 4 * KILOBYTE); + } + } + } + for i in 0..16 { + let phys_opt = allocator.allocate(2 * MEGABYTE); + println!("2 MB page {}: {:X?}", i, phys_opt); + if i % 2 == 0 { + if let Some(phys) = phys_opt { + allocator.free(phys, 2 * MEGABYTE); + } + } + } + + println!("Remaining: {}", format_size(allocator.remaining())); } unsafe fn inner() { From d752c5c91e66c80e0d96b1a29fd0c3f3e651e6ff Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 7 Sep 2020 21:05:04 -0600 Subject: [PATCH 027/120] WIP: buddy allocator --- src/arch/emulate.rs | 11 +++- src/lib.rs | 1 + src/main.rs | 145 +++++++++++++++++++++++++++++++++++++------- 3 files changed, 132 insertions(+), 25 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 5106c4561d..e4ed0ce220 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -96,11 +96,16 @@ impl Arch for EmulateArch { } const MEMORY_SIZE: usize = 64 * MEGABYTE; -static MEMORY_AREAS: [MemoryArea; 1] = [ +static MEMORY_AREAS: [MemoryArea; 2] = [ MemoryArea { base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted - size: MEMORY_SIZE - EmulateArch::PAGE_SIZE * 4, - } + size: MEMORY_SIZE/2 - EmulateArch::PAGE_SIZE * 4, + }, + // Second area for debugging + MemoryArea { + base: PhysicalAddress::new(MEMORY_SIZE/2), + size: MEMORY_SIZE/2, + }, ]; static mut MACHINE: Option> = None; diff --git a/src/lib.rs b/src/lib.rs index b34ed7e15b..cbd5f79462 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,6 +61,7 @@ impl VirtualAddress { } } +#[derive(Debug)] pub struct MemoryArea { pub base: PhysicalAddress, pub size: usize, diff --git a/src/main.rs b/src/main.rs index a331c3dfd5..1a3a6ca68c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,10 @@ use rmm::{ VirtualAddress, }; -use core::marker::PhantomData; +use core::{ + marker::PhantomData, + mem, +}; pub fn format_size(size: usize) -> String { if size >= 2 * TERABYTE { @@ -187,6 +190,104 @@ impl SlabAllocator { } } +#[derive(Clone, Copy, Debug)] +#[repr(packed)] +pub struct BuddyEntry { + pub base: PhysicalAddress, + pub size: usize, + pub map_phys: PhysicalAddress, + pub map_virt: VirtualAddress, +} + +impl BuddyEntry { + pub fn empty() -> Self { + Self { + base: PhysicalAddress::new(0), + size: 0, + map_phys: PhysicalAddress::new(0), + map_virt: VirtualAddress::new(0), + } + } +} + +pub struct BuddyAllocator { + table_phys: PhysicalAddress, + table_virt: VirtualAddress, + phantom: PhantomData, +} + +impl BuddyAllocator { + pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Option { + // First, we need an allocator, so we can allocate the buddy tables + // Since the tables are static, we can use the bump allocator + let mut bump_allocator = BumpAllocator::::new(areas, offset); + + // Allocate buddy table + let table_phys = bump_allocator.allocate()?; + let table_virt = A::phys_to_virt(table_phys); + for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + let virt = table_virt.add(i * mem::size_of::()); + A::write(virt, BuddyEntry::empty()); + } + + let mut allocator = Self { + table_phys, + table_virt, + phantom: PhantomData, + }; + + // Add areas to buddy table, combining areas when possible + for area in areas.iter() { + for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + let virt = table_virt.add(i * mem::size_of::()); + let mut entry = A::read::(virt); + let inserted = if area.base.add(area.size) == entry.base { + // Combine entry at start + entry.base = area.base; + entry.size += area.size; + true + } else if area.base == entry.base.add(entry.size) { + // Combine entry at end + entry.size += area.size; + true + } else if entry.size == 0 { + // Create new entry + entry.base = area.base; + entry.size = area.size; + true + } else { + false + }; + if inserted { + A::write(virt, entry); + break; + } + } + } + + //TODO: sort areas? + + //TODO: Allocate buddy maps + for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + let virt = table_virt.add(i * mem::size_of::()); + let mut entry = A::read::(virt); + if entry.size > 0 { + println!("{}: {:X?}", i, entry); + + let pages = entry.size / A::PAGE_SIZE; + println!(" pages: {}", pages); + let map_size = (pages + 7) / 8; + println!(" map size: {}", format_size(map_size)); + } + } + + //TODO: mark areas used for static tables as used + + Some(allocator) + } + +} + pub struct Mapper { table_addr: PhysicalAddress, allocator: BumpAllocator, @@ -262,27 +363,27 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let offset = mapper.allocator.offset; println!("Permanently used: {}", format_size(offset)); - let mut allocator = SlabAllocator::::new(areas, offset); - for i in 0..16 { - let phys_opt = allocator.allocate(4 * KILOBYTE); - println!("4 KB page {}: {:X?}", i, phys_opt); - if i % 2 == 0 { - if let Some(phys) = phys_opt { - allocator.free(phys, 4 * KILOBYTE); - } - } - } - for i in 0..16 { - let phys_opt = allocator.allocate(2 * MEGABYTE); - println!("2 MB page {}: {:X?}", i, phys_opt); - if i % 2 == 0 { - if let Some(phys) = phys_opt { - allocator.free(phys, 2 * MEGABYTE); - } - } - } - - println!("Remaining: {}", format_size(allocator.remaining())); + let mut allocator = BuddyAllocator::::new(areas, offset); + // for i in 0..16 { + // let phys_opt = allocator.allocate(4 * KILOBYTE); + // println!("4 KB page {}: {:X?}", i, phys_opt); + // if i % 2 == 0 { + // if let Some(phys) = phys_opt { + // allocator.free(phys, 4 * KILOBYTE); + // } + // } + // } + // for i in 0..16 { + // let phys_opt = allocator.allocate(2 * MEGABYTE); + // println!("2 MB page {}: {:X?}", i, phys_opt); + // if i % 2 == 0 { + // if let Some(phys) = phys_opt { + // allocator.free(phys, 2 * MEGABYTE); + // } + // } + // } + // + // println!("Remaining: {}", format_size(allocator.remaining())); } unsafe fn inner() { From 21d7f28fdc7c8ce1b8a83029a76ff39ecfe1fd92 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 7 Sep 2020 21:44:23 -0600 Subject: [PATCH 028/120] Allocate and clear buddy maps --- src/main.rs | 58 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1a3a6ca68c..41a729ed9d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -195,8 +195,7 @@ impl SlabAllocator { pub struct BuddyEntry { pub base: PhysicalAddress, pub size: usize, - pub map_phys: PhysicalAddress, - pub map_virt: VirtualAddress, + pub map: PhysicalAddress, } impl BuddyEntry { @@ -204,12 +203,18 @@ impl BuddyEntry { Self { base: PhysicalAddress::new(0), size: 0, - map_phys: PhysicalAddress::new(0), - map_virt: VirtualAddress::new(0), + map: PhysicalAddress::new(0), } } } +#[derive(Clone, Copy, Debug)] +#[repr(packed)] +pub struct BuddyMapFooter { + pub next: PhysicalAddress, + //TODO: index of last known free bit +} + pub struct BuddyAllocator { table_phys: PhysicalAddress, table_virt: VirtualAddress, @@ -217,6 +222,9 @@ pub struct BuddyAllocator { } impl BuddyAllocator { + const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); + const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; + pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Option { // First, we need an allocator, so we can allocate the buddy tables // Since the tables are static, we can use the bump allocator @@ -267,7 +275,7 @@ impl BuddyAllocator { //TODO: sort areas? - //TODO: Allocate buddy maps + // Allocate buddy maps for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { let virt = table_virt.add(i * mem::size_of::()); let mut entry = A::read::(virt); @@ -276,16 +284,50 @@ impl BuddyAllocator { let pages = entry.size / A::PAGE_SIZE; println!(" pages: {}", pages); - let map_size = (pages + 7) / 8; - println!(" map size: {}", format_size(map_size)); + let map_pages = (pages + (Self::MAP_PAGE_BITS - 1)) / Self::MAP_PAGE_BITS; + println!(" map pages: {}", map_pages); + + for _ in 0 .. map_pages { + let map_phys = bump_allocator.allocate()?; + let map_virt = A::phys_to_virt(map_phys); + for i in 0..Self::MAP_PAGE_BYTES { + A::write(map_virt.add(i), 0); + } + A::write(map_virt.add(Self::MAP_PAGE_BYTES), BuddyMapFooter { + next: entry.map, + }); + entry.map = map_phys; + } } } - //TODO: mark areas used for static tables as used + // Mark unused areas as free + let mut area_offset = bump_allocator.offset; + for area in areas.iter() { + if area_offset < area.size { + area_offset = 0; + let area_base = area.base.add(offset); + let area_size = area.size - offset; + allocator.free(area_base, area_size); + } else { + area_offset -= area.size; + } + } Some(allocator) } + pub unsafe fn free(&mut self, base: PhysicalAddress, size: usize) { + for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + let virt = self.table_virt.add(i * mem::size_of::()); + let entry = A::read::(virt); + if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { + println!("{:X}:{:X} inside of {:X}:{:X}", base.data(), size, entry.base.data(), entry.size); + + //TODO + } + } + } } pub struct Mapper { From 4eb4f579dfadc3fd837d2a883cd86624cfdb45a8 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 7 Sep 2020 22:02:24 -0600 Subject: [PATCH 029/120] Set entry when performing allocation --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index 41a729ed9d..d6518aa137 100644 --- a/src/main.rs +++ b/src/main.rs @@ -298,6 +298,8 @@ impl BuddyAllocator { }); entry.map = map_phys; } + + A::write(virt, entry); } } From 57d66236b76a7a66ce8057bc8fe197273d8159bb Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 7 Sep 2020 22:02:52 -0600 Subject: [PATCH 030/120] WIP buddy allocator free pages using bitmap --- src/main.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index d6518aa137..a1254a20e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -326,7 +326,33 @@ impl BuddyAllocator { if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { println!("{:X}:{:X} inside of {:X}:{:X}", base.data(), size, entry.base.data(), entry.size); - //TODO + //TODO: Correct logic + let pages = size / A::PAGE_SIZE; + for page in 0 .. pages { + let page_base = base.add(page * A::PAGE_SIZE); + let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; + let mut map_page = index / Self::MAP_PAGE_BITS; + let map_bit = index % Self::MAP_PAGE_BITS; + println!("index {} => map_page {}, map_bit {}", index, map_page, map_bit); + + //TODO: improve performance + let mut map_phys = entry.map; + loop { + if map_phys.data() == 0 { unimplemented!() } + let map_virt = A::phys_to_virt(map_phys); + if map_page == 0 { + let map_byte_virt = map_virt.add(map_bit / 8); + let mut value: u8 = A::read(map_byte_virt); + value |= 1 << (map_bit % 8); + A::write(map_byte_virt, value); + return; + } else { + let footer = A::read::(map_virt); + map_phys = footer.next; + map_page -= 1; + } + } + } } } } From 52a08e70fd4c3b0478d7319a05f582dd38c01990 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 09:27:40 -0600 Subject: [PATCH 031/120] Add buddy allocation --- src/main.rs | 70 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index a1254a20e9..689a9378a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -222,6 +222,7 @@ pub struct BuddyAllocator { } impl BuddyAllocator { + const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::(); const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; @@ -276,7 +277,7 @@ impl BuddyAllocator { //TODO: sort areas? // Allocate buddy maps - for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + for i in 0 .. Self::BUDDY_ENTRIES { let virt = table_virt.add(i * mem::size_of::()); let mut entry = A::read::(virt); if entry.size > 0 { @@ -298,7 +299,7 @@ impl BuddyAllocator { }); entry.map = map_phys; } - + A::write(virt, entry); } } @@ -319,13 +320,49 @@ impl BuddyAllocator { Some(allocator) } + pub unsafe fn allocate(&mut self, size: usize) -> Option { + //TODO: support other sizes + if size != A::PAGE_SIZE { + return None; + } + + for i in 0 .. Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::()); + let entry = A::read::(virt); + + //TODO: improve performance + let mut map_phys = entry.map; + let mut offset = 0; + while map_phys.data() != 0 { + let map_virt = A::phys_to_virt(map_phys); + for i in 0 .. Self::MAP_PAGE_BYTES { + let map_byte_virt = map_virt.add(i); + let mut value: u8 = A::read(map_byte_virt); + if (value & u8::MAX) != 0 { + for bit in 0..8 { + if (value & (1 << bit)) != 0 { + value &= !(1 << bit); + A::write(map_byte_virt, value); + let base = entry.base.add(offset + bit * A::PAGE_SIZE); + return Some(base); + } + } + } + offset += A::PAGE_SIZE * 8; + } + + let footer = A::read::(map_virt); + map_phys = footer.next; + } + } + None + } + pub unsafe fn free(&mut self, base: PhysicalAddress, size: usize) { - for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + for i in 0 .. Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::()); let entry = A::read::(virt); if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { - println!("{:X}:{:X} inside of {:X}:{:X}", base.data(), size, entry.base.data(), entry.size); - //TODO: Correct logic let pages = size / A::PAGE_SIZE; for page in 0 .. pages { @@ -333,7 +370,6 @@ impl BuddyAllocator { let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; let mut map_page = index / Self::MAP_PAGE_BITS; let map_bit = index % Self::MAP_PAGE_BITS; - println!("index {} => map_page {}, map_bit {}", index, map_page, map_bit); //TODO: improve performance let mut map_phys = entry.map; @@ -345,7 +381,7 @@ impl BuddyAllocator { let mut value: u8 = A::read(map_byte_virt); value |= 1 << (map_bit % 8); A::write(map_byte_virt, value); - return; + break; } else { let footer = A::read::(map_virt); map_phys = footer.next; @@ -433,16 +469,16 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let offset = mapper.allocator.offset; println!("Permanently used: {}", format_size(offset)); - let mut allocator = BuddyAllocator::::new(areas, offset); - // for i in 0..16 { - // let phys_opt = allocator.allocate(4 * KILOBYTE); - // println!("4 KB page {}: {:X?}", i, phys_opt); - // if i % 2 == 0 { - // if let Some(phys) = phys_opt { - // allocator.free(phys, 4 * KILOBYTE); - // } - // } - // } + let mut allocator = BuddyAllocator::::new(areas, offset).unwrap(); + for i in 0..16 { + let phys_opt = allocator.allocate(4 * KILOBYTE); + println!("4 KB page {}: {:X?}", i, phys_opt); + if i % 2 == 0 { + if let Some(phys) = phys_opt { + allocator.free(phys, 4 * KILOBYTE); + } + } + } // for i in 0..16 { // let phys_opt = allocator.allocate(2 * MEGABYTE); // println!("2 MB page {}: {:X?}", i, phys_opt); From 62da3afef56305e24cde6bb0f1bfa844420b239e Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 09:35:34 -0600 Subject: [PATCH 032/120] Add clearing of freed pages in BuddyAllocator --- src/main.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 689a9378a4..f812228873 100644 --- a/src/main.rs +++ b/src/main.rs @@ -218,6 +218,7 @@ pub struct BuddyMapFooter { pub struct BuddyAllocator { table_phys: PhysicalAddress, table_virt: VirtualAddress, + clear_frees: bool, phantom: PhantomData, } @@ -226,7 +227,7 @@ impl BuddyAllocator { const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; - pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Option { + pub unsafe fn new(areas: &'static [MemoryArea], offset: usize, clear_frees: bool) -> Option { // First, we need an allocator, so we can allocate the buddy tables // Since the tables are static, we can use the bump allocator let mut bump_allocator = BumpAllocator::::new(areas, offset); @@ -242,6 +243,7 @@ impl BuddyAllocator { let mut allocator = Self { table_phys, table_virt, + clear_frees, phantom: PhantomData, }; @@ -343,8 +345,8 @@ impl BuddyAllocator { if (value & (1 << bit)) != 0 { value &= !(1 << bit); A::write(map_byte_virt, value); - let base = entry.base.add(offset + bit * A::PAGE_SIZE); - return Some(base); + let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); + return Some(page_phys); } } } @@ -359,6 +361,18 @@ impl BuddyAllocator { } pub unsafe fn free(&mut self, base: PhysicalAddress, size: usize) { + if self.clear_frees { + // Zero freed pages for security, also ensures all allocs are zerod + //TODO: improve performance + //TODO: assumes linear physical mapping + let mut zero_virt = A::phys_to_virt(base); + let zero_end = zero_virt.add(size); + while zero_virt < zero_end { + A::write(zero_virt, 0usize); + zero_virt = zero_virt.add(mem::size_of::()); + } + } + for i in 0 .. Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::()); let entry = A::read::(virt); @@ -469,7 +483,7 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let offset = mapper.allocator.offset; println!("Permanently used: {}", format_size(offset)); - let mut allocator = BuddyAllocator::::new(areas, offset).unwrap(); + let mut allocator = BuddyAllocator::::new(areas, offset, true).unwrap(); for i in 0..16 { let phys_opt = allocator.allocate(4 * KILOBYTE); println!("4 KB page {}: {:X?}", i, phys_opt); From d153af8b8367db097fe7512878bd98bc23ab4ed4 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 10:24:43 -0600 Subject: [PATCH 033/120] Fix calculation of free areas --- src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index f812228873..1c2f96b3c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -138,10 +138,10 @@ impl SlabAllocator { let mut area_offset = offset; for area in areas.iter() { if area_offset < area.size { - area_offset = 0; - let area_base = area.base.add(offset); - let area_size = area.size - offset; + let area_base = area.base.add(area_offset); + let area_size = area.size - area_offset; allocator.free(area_base, area_size); + area_offset = 0; } else { area_offset -= area.size; } @@ -310,10 +310,10 @@ impl BuddyAllocator { let mut area_offset = bump_allocator.offset; for area in areas.iter() { if area_offset < area.size { - area_offset = 0; - let area_base = area.base.add(offset); - let area_size = area.size - offset; + let area_base = area.base.add(area_offset); + let area_size = area.size - area_offset; allocator.free(area_base, area_size); + area_offset = 0; } else { area_offset -= area.size; } From 29945b84b1c1e4a51c58ecbc2d98da86f21058db Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 10:24:53 -0600 Subject: [PATCH 034/120] no_std building support --- Cargo.toml | 5 +++++ src/arch/mod.rs | 9 +++++++-- src/lib.rs | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 95b68f8ee7..5580d869da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,4 +6,9 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + [dependencies] + +[features] +default = ["std"] +std = [] diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 21a88f6368..e0559713d8 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -6,8 +6,13 @@ use crate::{ VirtualAddress, }; -pub mod emulate; -pub mod x86_64; +#[cfg(feature = "std")] +pub use self::emulate::EmulateArch; +pub use self::x86_64::X8664Arch; + +#[cfg(feature = "std")] +mod emulate; +mod x86_64; pub trait Arch { const PAGE_SHIFT: usize; diff --git a/src/lib.rs b/src/lib.rs index cbd5f79462..d72240bac5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,8 @@ +#![cfg_attr(not(feature = "std"), no_std)] #![feature(asm)] pub use crate::{ - arch::Arch, - arch::emulate::*, + arch::*, page::{ PageEntry, PageTable From fcb64422c4692f9836ff6c29a9973e32338e4d79 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 10:52:52 -0600 Subject: [PATCH 035/120] Add FrameAllocator trait, move frame allocator and mapper to library --- Cargo.toml | 3 - src/allocator/frame/buddy.rs | 229 ++++++++++++++++++++++++ src/allocator/frame/bump.rs | 55 ++++++ src/allocator/frame/mod.rs | 12 ++ src/allocator/mod.rs | 3 + src/lib.rs | 7 +- src/main.rs | 329 +++-------------------------------- src/page/mapper.rs | 59 +++++++ src/page/mod.rs | 2 + 9 files changed, 387 insertions(+), 312 deletions(-) create mode 100644 src/allocator/frame/buddy.rs create mode 100644 src/allocator/frame/bump.rs create mode 100644 src/allocator/frame/mod.rs create mode 100644 src/allocator/mod.rs create mode 100644 src/page/mapper.rs diff --git a/Cargo.toml b/Cargo.toml index 5580d869da..adafa2a7dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] - [features] default = ["std"] std = [] diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs new file mode 100644 index 0000000000..945b02be96 --- /dev/null +++ b/src/allocator/frame/buddy.rs @@ -0,0 +1,229 @@ +use core::{ + marker::PhantomData, + mem, +}; + +use crate::{ + Arch, + BumpAllocator, + FrameAllocator, + MemoryArea, + PhysicalAddress, + VirtualAddress, +}; + +#[derive(Clone, Copy, Debug)] +#[repr(packed)] +struct BuddyEntry { + base: PhysicalAddress, + size: usize, + map: PhysicalAddress, +} + +impl BuddyEntry { + pub fn empty() -> Self { + Self { + base: PhysicalAddress::new(0), + size: 0, + map: PhysicalAddress::new(0), + } + } +} + +#[derive(Clone, Copy, Debug)] +#[repr(packed)] +struct BuddyMapFooter { + next: PhysicalAddress, + //TODO: index of last known free bit +} + +pub struct BuddyAllocator { + table_phys: PhysicalAddress, + table_virt: VirtualAddress, + clear_frees: bool, + phantom: PhantomData, +} + +impl BuddyAllocator { + const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::(); + const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); + const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; + + pub unsafe fn new(mut bump_allocator: BumpAllocator, clear_frees: bool) -> Option { + // Allocate buddy table + let table_phys = bump_allocator.allocate(1)?; + let table_virt = A::phys_to_virt(table_phys); + for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + let virt = table_virt.add(i * mem::size_of::()); + A::write(virt, BuddyEntry::empty()); + } + + let mut allocator = Self { + table_phys, + table_virt, + clear_frees, + phantom: PhantomData, + }; + + // Add areas to buddy table, combining areas when possible + for area in bump_allocator.areas().iter() { + for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { + let virt = table_virt.add(i * mem::size_of::()); + let mut entry = A::read::(virt); + let inserted = if area.base.add(area.size) == entry.base { + // Combine entry at start + entry.base = area.base; + entry.size += area.size; + true + } else if area.base == entry.base.add(entry.size) { + // Combine entry at end + entry.size += area.size; + true + } else if entry.size == 0 { + // Create new entry + entry.base = area.base; + entry.size = area.size; + true + } else { + false + }; + if inserted { + A::write(virt, entry); + break; + } + } + } + + //TODO: sort areas? + + // Allocate buddy maps + for i in 0 .. Self::BUDDY_ENTRIES { + let virt = table_virt.add(i * mem::size_of::()); + let mut entry = A::read::(virt); + if entry.size > 0 { + println!("{}: {:X?}", i, entry); + + let pages = entry.size / A::PAGE_SIZE; + println!(" pages: {}", pages); + let map_pages = (pages + (Self::MAP_PAGE_BITS - 1)) / Self::MAP_PAGE_BITS; + println!(" map pages: {}", map_pages); + + for _ in 0 .. map_pages { + let map_phys = bump_allocator.allocate(1)?; + let map_virt = A::phys_to_virt(map_phys); + for i in 0..Self::MAP_PAGE_BYTES { + A::write(map_virt.add(i), 0); + } + A::write(map_virt.add(Self::MAP_PAGE_BYTES), BuddyMapFooter { + next: entry.map, + }); + entry.map = map_phys; + } + + A::write(virt, entry); + } + } + + // Mark unused areas as free + let mut area_offset = bump_allocator.offset(); + for area in bump_allocator.areas().iter() { + if area_offset < area.size { + let area_base = area.base.add(area_offset); + let area_size = area.size - area_offset; + allocator.free(area_base, area_size); + area_offset = 0; + } else { + area_offset -= area.size; + } + } + + Some(allocator) + } +} + +impl FrameAllocator for BuddyAllocator { + unsafe fn allocate(&mut self, size: usize) -> Option { + //TODO: support other sizes + if size != A::PAGE_SIZE { + return None; + } + + for i in 0 .. Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::()); + let entry = A::read::(virt); + + //TODO: improve performance + let mut map_phys = entry.map; + let mut offset = 0; + while map_phys.data() != 0 { + let map_virt = A::phys_to_virt(map_phys); + for i in 0 .. Self::MAP_PAGE_BYTES { + let map_byte_virt = map_virt.add(i); + let mut value: u8 = A::read(map_byte_virt); + if (value & u8::MAX) != 0 { + for bit in 0..8 { + if (value & (1 << bit)) != 0 { + value &= !(1 << bit); + A::write(map_byte_virt, value); + let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); + return Some(page_phys); + } + } + } + offset += A::PAGE_SIZE * 8; + } + + let footer = A::read::(map_virt); + map_phys = footer.next; + } + } + None + } + + unsafe fn free(&mut self, base: PhysicalAddress, size: usize) { + if self.clear_frees { + // Zero freed pages for security, also ensures all allocs are zerod + //TODO: improve performance + //TODO: assumes linear physical mapping + let mut zero_virt = A::phys_to_virt(base); + let zero_end = zero_virt.add(size); + while zero_virt < zero_end { + A::write(zero_virt, 0usize); + zero_virt = zero_virt.add(mem::size_of::()); + } + } + + for i in 0 .. Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::()); + let entry = A::read::(virt); + if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { + //TODO: Correct logic + let pages = size / A::PAGE_SIZE; + for page in 0 .. pages { + let page_base = base.add(page * A::PAGE_SIZE); + let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; + let mut map_page = index / Self::MAP_PAGE_BITS; + let map_bit = index % Self::MAP_PAGE_BITS; + + //TODO: improve performance + let mut map_phys = entry.map; + loop { + if map_phys.data() == 0 { unimplemented!() } + let map_virt = A::phys_to_virt(map_phys); + if map_page == 0 { + let map_byte_virt = map_virt.add(map_bit / 8); + let mut value: u8 = A::read(map_byte_virt); + value |= 1 << (map_bit % 8); + A::write(map_byte_virt, value); + break; + } else { + let footer = A::read::(map_virt); + map_phys = footer.next; + map_page -= 1; + } + } + } + } + } + } +} diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs new file mode 100644 index 0000000000..330a21af85 --- /dev/null +++ b/src/allocator/frame/bump.rs @@ -0,0 +1,55 @@ +use core::marker::PhantomData; + +use crate::{ + Arch, + FrameAllocator, + MemoryArea, + PhysicalAddress, +}; + +pub struct BumpAllocator { + areas: &'static [MemoryArea], + offset: usize, + phantom: PhantomData, +} + +impl BumpAllocator { + pub fn new(areas: &'static [MemoryArea], offset: usize) -> Self { + Self { + areas, + offset, + phantom: PhantomData, + } + } + + pub fn areas(&self) -> &'static [MemoryArea] { + self.areas + } + + pub fn offset(&self) -> usize { + self.offset + } +} + +impl FrameAllocator for BumpAllocator { + unsafe fn allocate(&mut self, count: usize) -> Option { + //TODO: support allocation of multiple pages + if count != 1 { + return None; + } + + let mut offset = self.offset; + for area in self.areas.iter() { + if offset < area.size { + self.offset += A::PAGE_SIZE; + return Some(area.base.add(offset)); + } + offset -= area.size; + } + None + } + + unsafe fn free(&mut self, address: PhysicalAddress, count: usize) { + unimplemented!(); + } +} diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs new file mode 100644 index 0000000000..7dc20163a3 --- /dev/null +++ b/src/allocator/frame/mod.rs @@ -0,0 +1,12 @@ +use crate::PhysicalAddress; + +pub use self::buddy::*; +pub use self::bump::*; + +mod buddy; +mod bump; + +pub trait FrameAllocator { + unsafe fn allocate(&mut self, count: usize) -> Option; + unsafe fn free(&mut self, address: PhysicalAddress, count: usize); +} diff --git a/src/allocator/mod.rs b/src/allocator/mod.rs new file mode 100644 index 0000000000..0027153038 --- /dev/null +++ b/src/allocator/mod.rs @@ -0,0 +1,3 @@ +pub use self::frame::*; + +mod frame; diff --git a/src/lib.rs b/src/lib.rs index d72240bac5..f100557a34 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,13 +2,12 @@ #![feature(asm)] pub use crate::{ + allocator::*, arch::*, - page::{ - PageEntry, - PageTable - }, + page::*, }; +mod allocator; mod arch; mod page; diff --git a/src/main.rs b/src/main.rs index 1c2f96b3c5..2564a17623 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,9 +4,13 @@ use rmm::{ GIGABYTE, TERABYTE, Arch, + BuddyAllocator, + BumpAllocator, EmulateArch, + FrameAllocator, MemoryArea, PageEntry, + PageMapper, PageTable, PhysicalAddress, VirtualAddress, @@ -49,34 +53,6 @@ unsafe fn dump_tables(table: PageTable) { } } -pub struct BumpAllocator { - areas: &'static [MemoryArea], - offset: usize, - phantom: PhantomData, -} - -impl BumpAllocator { - pub fn new(areas: &'static [MemoryArea], offset: usize) -> Self { - Self { - areas, - offset, - phantom: PhantomData, - } - } - - pub fn allocate(&mut self) -> Option { - let mut offset = self.offset; - for area in self.areas.iter() { - if offset < area.size { - self.offset += A::PAGE_SIZE; - return Some(area.base.add(offset)); - } - offset -= area.size; - } - None - } -} - pub struct SlabNode { next: PhysicalAddress, count: usize, @@ -190,267 +166,6 @@ impl SlabAllocator { } } -#[derive(Clone, Copy, Debug)] -#[repr(packed)] -pub struct BuddyEntry { - pub base: PhysicalAddress, - pub size: usize, - pub map: PhysicalAddress, -} - -impl BuddyEntry { - pub fn empty() -> Self { - Self { - base: PhysicalAddress::new(0), - size: 0, - map: PhysicalAddress::new(0), - } - } -} - -#[derive(Clone, Copy, Debug)] -#[repr(packed)] -pub struct BuddyMapFooter { - pub next: PhysicalAddress, - //TODO: index of last known free bit -} - -pub struct BuddyAllocator { - table_phys: PhysicalAddress, - table_virt: VirtualAddress, - clear_frees: bool, - phantom: PhantomData, -} - -impl BuddyAllocator { - const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::(); - const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); - const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; - - pub unsafe fn new(areas: &'static [MemoryArea], offset: usize, clear_frees: bool) -> Option { - // First, we need an allocator, so we can allocate the buddy tables - // Since the tables are static, we can use the bump allocator - let mut bump_allocator = BumpAllocator::::new(areas, offset); - - // Allocate buddy table - let table_phys = bump_allocator.allocate()?; - let table_virt = A::phys_to_virt(table_phys); - for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { - let virt = table_virt.add(i * mem::size_of::()); - A::write(virt, BuddyEntry::empty()); - } - - let mut allocator = Self { - table_phys, - table_virt, - clear_frees, - phantom: PhantomData, - }; - - // Add areas to buddy table, combining areas when possible - for area in areas.iter() { - for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { - let virt = table_virt.add(i * mem::size_of::()); - let mut entry = A::read::(virt); - let inserted = if area.base.add(area.size) == entry.base { - // Combine entry at start - entry.base = area.base; - entry.size += area.size; - true - } else if area.base == entry.base.add(entry.size) { - // Combine entry at end - entry.size += area.size; - true - } else if entry.size == 0 { - // Create new entry - entry.base = area.base; - entry.size = area.size; - true - } else { - false - }; - if inserted { - A::write(virt, entry); - break; - } - } - } - - //TODO: sort areas? - - // Allocate buddy maps - for i in 0 .. Self::BUDDY_ENTRIES { - let virt = table_virt.add(i * mem::size_of::()); - let mut entry = A::read::(virt); - if entry.size > 0 { - println!("{}: {:X?}", i, entry); - - let pages = entry.size / A::PAGE_SIZE; - println!(" pages: {}", pages); - let map_pages = (pages + (Self::MAP_PAGE_BITS - 1)) / Self::MAP_PAGE_BITS; - println!(" map pages: {}", map_pages); - - for _ in 0 .. map_pages { - let map_phys = bump_allocator.allocate()?; - let map_virt = A::phys_to_virt(map_phys); - for i in 0..Self::MAP_PAGE_BYTES { - A::write(map_virt.add(i), 0); - } - A::write(map_virt.add(Self::MAP_PAGE_BYTES), BuddyMapFooter { - next: entry.map, - }); - entry.map = map_phys; - } - - A::write(virt, entry); - } - } - - // Mark unused areas as free - let mut area_offset = bump_allocator.offset; - for area in areas.iter() { - if area_offset < area.size { - let area_base = area.base.add(area_offset); - let area_size = area.size - area_offset; - allocator.free(area_base, area_size); - area_offset = 0; - } else { - area_offset -= area.size; - } - } - - Some(allocator) - } - - pub unsafe fn allocate(&mut self, size: usize) -> Option { - //TODO: support other sizes - if size != A::PAGE_SIZE { - return None; - } - - for i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(i * mem::size_of::()); - let entry = A::read::(virt); - - //TODO: improve performance - let mut map_phys = entry.map; - let mut offset = 0; - while map_phys.data() != 0 { - let map_virt = A::phys_to_virt(map_phys); - for i in 0 .. Self::MAP_PAGE_BYTES { - let map_byte_virt = map_virt.add(i); - let mut value: u8 = A::read(map_byte_virt); - if (value & u8::MAX) != 0 { - for bit in 0..8 { - if (value & (1 << bit)) != 0 { - value &= !(1 << bit); - A::write(map_byte_virt, value); - let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); - return Some(page_phys); - } - } - } - offset += A::PAGE_SIZE * 8; - } - - let footer = A::read::(map_virt); - map_phys = footer.next; - } - } - None - } - - pub unsafe fn free(&mut self, base: PhysicalAddress, size: usize) { - if self.clear_frees { - // Zero freed pages for security, also ensures all allocs are zerod - //TODO: improve performance - //TODO: assumes linear physical mapping - let mut zero_virt = A::phys_to_virt(base); - let zero_end = zero_virt.add(size); - while zero_virt < zero_end { - A::write(zero_virt, 0usize); - zero_virt = zero_virt.add(mem::size_of::()); - } - } - - for i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(i * mem::size_of::()); - let entry = A::read::(virt); - if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { - //TODO: Correct logic - let pages = size / A::PAGE_SIZE; - for page in 0 .. pages { - let page_base = base.add(page * A::PAGE_SIZE); - let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; - let mut map_page = index / Self::MAP_PAGE_BITS; - let map_bit = index % Self::MAP_PAGE_BITS; - - //TODO: improve performance - let mut map_phys = entry.map; - loop { - if map_phys.data() == 0 { unimplemented!() } - let map_virt = A::phys_to_virt(map_phys); - if map_page == 0 { - let map_byte_virt = map_virt.add(map_bit / 8); - let mut value: u8 = A::read(map_byte_virt); - value |= 1 << (map_bit % 8); - A::write(map_byte_virt, value); - break; - } else { - let footer = A::read::(map_virt); - map_phys = footer.next; - map_page -= 1; - } - } - } - } - } - } -} - -pub struct Mapper { - table_addr: PhysicalAddress, - allocator: BumpAllocator, -} - -impl Mapper { - pub unsafe fn new(mut allocator: BumpAllocator) -> Option { - let table_addr = allocator.allocate()?; - Some(Self { - table_addr, - allocator, - }) - } - - pub unsafe fn map(&mut self, virt: VirtualAddress, entry: PageEntry) -> Option<()> { - let mut table = PageTable::new( - VirtualAddress::new(0), - self.table_addr, - A::PAGE_LEVELS - 1 - ); - loop { - let i = table.index_of(virt)?; - if table.level() == 0 { - //TODO: check for overwriting entry - table.set_entry(i, entry); - return Some(()); - } else { - let next_opt = table.next(i); - let next = match next_opt { - Some(some) => some, - None => { - let phys = self.allocator.allocate()?; - //TODO: correct flags? - table.set_entry(i, PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); - table.next(i)? - } - }; - table = next; - } - } - } -} - unsafe fn new_tables(areas: &'static [MemoryArea]) { // First, calculate how much memory we have let mut size = 0; @@ -461,29 +176,33 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { println!("Memory: {}", format_size(size)); // Create a basic allocator for the first pages - let allocator = BumpAllocator::::new(areas, 0); + let mut bump_allocator = BumpAllocator::::new(areas, 0); - // Map all physical areas at PHYS_OFFSET - let mut mapper = Mapper::new(allocator).expect("failed to create Mapper"); - for area in areas.iter() { - for i in 0..area.size / A::PAGE_SIZE { - let phys = area.base.add(i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - mapper.map( - virt, - PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT) - ).expect("failed to map frame"); + { + // Map all physical areas at PHYS_OFFSET + let mut mapper = PageMapper::::new( + &mut bump_allocator + ).expect("failed to create Mapper"); + for area in areas.iter() { + for i in 0..area.size / A::PAGE_SIZE { + let phys = area.base.add(i * A::PAGE_SIZE); + let virt = A::phys_to_virt(phys); + mapper.map( + virt, + PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT) + ).expect("failed to map frame"); + } } + + // Use the new table + mapper.activate(); } - // Use the new table - A::set_table(mapper.table_addr); - // Create the physical memory map - let offset = mapper.allocator.offset; + let offset = bump_allocator.offset(); println!("Permanently used: {}", format_size(offset)); - let mut allocator = BuddyAllocator::::new(areas, offset, true).unwrap(); + let mut allocator = BuddyAllocator::::new(bump_allocator, true).unwrap(); for i in 0..16 { let phys_opt = allocator.allocate(4 * KILOBYTE); println!("4 KB page {}: {:X?}", i, phys_opt); diff --git a/src/page/mapper.rs b/src/page/mapper.rs new file mode 100644 index 0000000000..cb1b57f44f --- /dev/null +++ b/src/page/mapper.rs @@ -0,0 +1,59 @@ +use core::marker::PhantomData; + +use crate::{ + Arch, + FrameAllocator, + PageEntry, + PageTable, + PhysicalAddress, + VirtualAddress, +}; + +pub struct PageMapper<'f, A, F> { + table_addr: PhysicalAddress, + allocator: &'f mut F, + phantom: PhantomData, +} + +impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { + pub unsafe fn new(allocator: &'f mut F) -> Option { + let table_addr = allocator.allocate(1)?; + Some(Self { + table_addr, + allocator, + phantom: PhantomData, + }) + } + + pub unsafe fn activate(&mut self) { + A::set_table(self.table_addr); + } + + pub unsafe fn map(&mut self, virt: VirtualAddress, entry: PageEntry) -> Option<()> { + let mut table = PageTable::new( + VirtualAddress::new(0), + self.table_addr, + A::PAGE_LEVELS - 1 + ); + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + //TODO: check for overwriting entry + table.set_entry(i, entry); + return Some(()); + } else { + let next_opt = table.next(i); + let next = match next_opt { + Some(some) => some, + None => { + let phys = self.allocator.allocate(1)?; + //TODO: correct flags? + table.set_entry(i, PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); + table.next(i)? + } + }; + table = next; + } + } + } +} diff --git a/src/page/mod.rs b/src/page/mod.rs index bef67a6fd6..268d3be9d3 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -1,7 +1,9 @@ pub use self::{ entry::PageEntry, + mapper::PageMapper, table::PageTable, }; mod entry; +mod mapper; mod table; From a64e790471adffc7364b5379b88e96544c52aed1 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 10:54:59 -0600 Subject: [PATCH 036/120] Fix warnings --- src/allocator/frame/buddy.rs | 3 --- src/allocator/frame/bump.rs | 2 +- src/main.rs | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 945b02be96..2d9b5ce489 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -7,7 +7,6 @@ use crate::{ Arch, BumpAllocator, FrameAllocator, - MemoryArea, PhysicalAddress, VirtualAddress, }; @@ -38,7 +37,6 @@ struct BuddyMapFooter { } pub struct BuddyAllocator { - table_phys: PhysicalAddress, table_virt: VirtualAddress, clear_frees: bool, phantom: PhantomData, @@ -59,7 +57,6 @@ impl BuddyAllocator { } let mut allocator = Self { - table_phys, table_virt, clear_frees, phantom: PhantomData, diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 330a21af85..aece627dea 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -49,7 +49,7 @@ impl FrameAllocator for BumpAllocator { None } - unsafe fn free(&mut self, address: PhysicalAddress, count: usize) { + unsafe fn free(&mut self, _address: PhysicalAddress, _count: usize) { unimplemented!(); } } diff --git a/src/main.rs b/src/main.rs index 2564a17623..b3ab505f40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,12 +13,10 @@ use rmm::{ PageMapper, PageTable, PhysicalAddress, - VirtualAddress, }; use core::{ marker::PhantomData, - mem, }; pub fn format_size(size: usize) -> String { From 1b58d2a956b97b2d680c1fb338a4a548d52b0f4f Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 11:13:49 -0600 Subject: [PATCH 037/120] Add FrameCount struct, improve zeroing page performance --- src/allocator/frame/buddy.rs | 35 +++++++++++++++-------------------- src/allocator/frame/bump.rs | 7 ++++--- src/allocator/frame/mod.rs | 29 ++++++++++++++++++++++++++--- src/arch/emulate.rs | 33 +++++++++++++++++++++++++++++++++ src/arch/mod.rs | 5 +++++ src/main.rs | 7 ++++--- src/page/mapper.rs | 4 ++-- 7 files changed, 89 insertions(+), 31 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 2d9b5ce489..947c61d512 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -7,6 +7,7 @@ use crate::{ Arch, BumpAllocator, FrameAllocator, + FrameCount, PhysicalAddress, VirtualAddress, }; @@ -49,7 +50,7 @@ impl BuddyAllocator { pub unsafe fn new(mut bump_allocator: BumpAllocator, clear_frees: bool) -> Option { // Allocate buddy table - let table_phys = bump_allocator.allocate(1)?; + let table_phys = bump_allocator.allocate_one()?; let table_virt = A::phys_to_virt(table_phys); for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { let virt = table_virt.add(i * mem::size_of::()); @@ -106,7 +107,7 @@ impl BuddyAllocator { println!(" map pages: {}", map_pages); for _ in 0 .. map_pages { - let map_phys = bump_allocator.allocate(1)?; + let map_phys = bump_allocator.allocate_one()?; let map_virt = A::phys_to_virt(map_phys); for i in 0..Self::MAP_PAGE_BYTES { A::write(map_virt.add(i), 0); @@ -127,7 +128,7 @@ impl BuddyAllocator { if area_offset < area.size { let area_base = area.base.add(area_offset); let area_size = area.size - area_offset; - allocator.free(area_base, area_size); + allocator.free(area_base, FrameCount::new(area_size / A::PAGE_SIZE)); area_offset = 0; } else { area_offset -= area.size; @@ -139,9 +140,9 @@ impl BuddyAllocator { } impl FrameAllocator for BuddyAllocator { - unsafe fn allocate(&mut self, size: usize) -> Option { + unsafe fn allocate(&mut self, count: FrameCount) -> Option { //TODO: support other sizes - if size != A::PAGE_SIZE { + if count.data() != 1 { return None; } @@ -177,27 +178,21 @@ impl FrameAllocator for BuddyAllocator { None } - unsafe fn free(&mut self, base: PhysicalAddress, size: usize) { - if self.clear_frees { - // Zero freed pages for security, also ensures all allocs are zerod - //TODO: improve performance - //TODO: assumes linear physical mapping - let mut zero_virt = A::phys_to_virt(base); - let zero_end = zero_virt.add(size); - while zero_virt < zero_end { - A::write(zero_virt, 0usize); - zero_virt = zero_virt.add(mem::size_of::()); - } - } - + unsafe fn free(&mut self, base: PhysicalAddress, count: FrameCount) { + let size = count.data() * A::PAGE_SIZE; for i in 0 .. Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::()); let entry = A::read::(virt); if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { //TODO: Correct logic - let pages = size / A::PAGE_SIZE; - for page in 0 .. pages { + for page in 0 .. count.data() { let page_base = base.add(page * A::PAGE_SIZE); + + if self.clear_frees { + let page_virt = A::phys_to_virt(page_base); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); + } + let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; let mut map_page = index / Self::MAP_PAGE_BITS; let map_bit = index % Self::MAP_PAGE_BITS; diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index aece627dea..af9822104f 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -3,6 +3,7 @@ use core::marker::PhantomData; use crate::{ Arch, FrameAllocator, + FrameCount, MemoryArea, PhysicalAddress, }; @@ -32,9 +33,9 @@ impl BumpAllocator { } impl FrameAllocator for BumpAllocator { - unsafe fn allocate(&mut self, count: usize) -> Option { + unsafe fn allocate(&mut self, count: FrameCount) -> Option { //TODO: support allocation of multiple pages - if count != 1 { + if count.data() != 1 { return None; } @@ -49,7 +50,7 @@ impl FrameAllocator for BumpAllocator { None } - unsafe fn free(&mut self, _address: PhysicalAddress, _count: usize) { + unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { unimplemented!(); } } diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index 7dc20163a3..33d07943bf 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -6,7 +6,30 @@ pub use self::bump::*; mod buddy; mod bump; -pub trait FrameAllocator { - unsafe fn allocate(&mut self, count: usize) -> Option; - unsafe fn free(&mut self, address: PhysicalAddress, count: usize); +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct FrameCount(usize); + +impl FrameCount { + pub fn new(count: usize) -> Self { + Self(count) + } + + pub fn data(&self) -> usize { + self.0 + } +} + +pub trait FrameAllocator { + unsafe fn allocate(&mut self, count: FrameCount) -> Option; + + unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount); + + unsafe fn allocate_one(&mut self) -> Option { + self.allocate(FrameCount::new(1)) + } + + unsafe fn free_one(&mut self, address: PhysicalAddress) { + self.free(address, FrameCount::new(1)); + } } diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index e4ed0ce220..f8d3f019a3 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -74,6 +74,11 @@ impl Arch for EmulateArch { MACHINE.as_mut().unwrap().write(address, value) } + #[inline(always)] + unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { + MACHINE.as_mut().unwrap().write_bytes(address, value, count) + } + #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { MACHINE.as_mut().unwrap().invalidate(address); @@ -149,6 +154,16 @@ impl Machine { } } + fn write_phys_bytes(&mut self, phys: PhysicalAddress, value: u8, count: usize) { + if phys.add(count).data() <= self.memory.len() { + unsafe { + ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()) as *mut u8, value, count); + } + } else { + panic!("write_phys_bytes: 0x{:X} count 0x{:X} outside of memory", phys.data(), count); + } + } + fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, usize)> { let virt_data = virt.data(); let page = virt_data & A::PAGE_ADDRESS_MASK; @@ -194,6 +209,24 @@ impl Machine { } } + fn write_bytes(&mut self, virt: VirtualAddress, value: u8, count: usize) { + //TODO: allow writing past page boundaries + let virt_data = virt.data(); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK) { + panic!("write_bytes: 0x{:X} count 0x{:X} passes page boundary", virt_data, count); + } + + if let Some((phys, flags)) = self.translate(virt) { + if flags & A::ENTRY_FLAG_WRITABLE != 0 { + self.write_phys_bytes(phys, value, count); + } else { + panic!("write_bytes: 0x{:X} count 0x{:X} not writable", virt_data, count); + } + } else { + panic!("write_bytes: 0x{:X} count 0x{:X} not present", virt_data, count); + } + } + fn invalidate(&mut self, _address: VirtualAddress) { unimplemented!(); } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index e0559713d8..53efdd3628 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -55,6 +55,11 @@ pub trait Arch { ptr::write(address.data() as *mut T, value) } + #[inline(always)] + unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { + ptr::write_bytes(address.data() as *mut u8, value, count) + } + unsafe fn invalidate(address: VirtualAddress); #[inline(always)] diff --git a/src/main.rs b/src/main.rs index b3ab505f40..9d7f3f2f18 100644 --- a/src/main.rs +++ b/src/main.rs @@ -202,11 +202,12 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let mut allocator = BuddyAllocator::::new(bump_allocator, true).unwrap(); for i in 0..16 { - let phys_opt = allocator.allocate(4 * KILOBYTE); - println!("4 KB page {}: {:X?}", i, phys_opt); + let phys_opt = allocator.allocate_one(); + println!("page {}: {:X?}", i, phys_opt); if i % 2 == 0 { if let Some(phys) = phys_opt { - allocator.free(phys, 4 * KILOBYTE); + println!("free {}: {:X?}", i, phys_opt); + allocator.free_one(phys); } } } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index cb1b57f44f..c2cc225843 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -17,7 +17,7 @@ pub struct PageMapper<'f, A, F> { impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { pub unsafe fn new(allocator: &'f mut F) -> Option { - let table_addr = allocator.allocate(1)?; + let table_addr = allocator.allocate_one()?; Some(Self { table_addr, allocator, @@ -46,7 +46,7 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { let next = match next_opt { Some(some) => some, None => { - let phys = self.allocator.allocate(1)?; + let phys = self.allocator.allocate_one()?; //TODO: correct flags? table.set_entry(i, PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); table.next(i)? From 8484d4447ce4abde04e9a458773277645a01e538 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 11:16:59 -0600 Subject: [PATCH 038/120] Fix compilation using no_std --- src/allocator/frame/buddy.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 947c61d512..a1bb8d99f2 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -99,13 +99,8 @@ impl BuddyAllocator { let virt = table_virt.add(i * mem::size_of::()); let mut entry = A::read::(virt); if entry.size > 0 { - println!("{}: {:X?}", i, entry); - let pages = entry.size / A::PAGE_SIZE; - println!(" pages: {}", pages); let map_pages = (pages + (Self::MAP_PAGE_BITS - 1)) / Self::MAP_PAGE_BITS; - println!(" map pages: {}", map_pages); - for _ in 0 .. map_pages { let map_phys = bump_allocator.allocate_one()?; let map_virt = A::phys_to_virt(map_phys); From b4c8ab797d6a2b6d5b1cbd3f9ad75a8d3a6c0f55 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 12:41:46 -0600 Subject: [PATCH 039/120] Zero all allocations --- src/allocator/frame/buddy.rs | 12 +++--------- src/allocator/frame/bump.rs | 5 ++++- src/lib.rs | 2 +- src/main.rs | 2 +- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index a1bb8d99f2..654a8fb0e4 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -39,7 +39,6 @@ struct BuddyMapFooter { pub struct BuddyAllocator { table_virt: VirtualAddress, - clear_frees: bool, phantom: PhantomData, } @@ -48,7 +47,7 @@ impl BuddyAllocator { const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; - pub unsafe fn new(mut bump_allocator: BumpAllocator, clear_frees: bool) -> Option { + pub unsafe fn new(mut bump_allocator: BumpAllocator) -> Option { // Allocate buddy table let table_phys = bump_allocator.allocate_one()?; let table_virt = A::phys_to_virt(table_phys); @@ -59,7 +58,6 @@ impl BuddyAllocator { let mut allocator = Self { table_virt, - clear_frees, phantom: PhantomData, }; @@ -159,6 +157,8 @@ impl FrameAllocator for BuddyAllocator { value &= !(1 << bit); A::write(map_byte_virt, value); let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); + let page_virt = A::phys_to_virt(page_phys); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); return Some(page_phys); } } @@ -182,12 +182,6 @@ impl FrameAllocator for BuddyAllocator { //TODO: Correct logic for page in 0 .. count.data() { let page_base = base.add(page * A::PAGE_SIZE); - - if self.clear_frees { - let page_virt = A::phys_to_virt(page_base); - A::write_bytes(page_virt, 0, A::PAGE_SIZE); - } - let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; let mut map_page = index / Self::MAP_PAGE_BITS; let map_bit = index % Self::MAP_PAGE_BITS; diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index af9822104f..7f4a2a2243 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -42,8 +42,11 @@ impl FrameAllocator for BumpAllocator { let mut offset = self.offset; for area in self.areas.iter() { if offset < area.size { + let page_phys = area.base.add(offset); + let page_virt = A::phys_to_virt(page_phys); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); self.offset += A::PAGE_SIZE; - return Some(area.base.add(offset)); + return Some(page_phys); } offset -= area.size; } diff --git a/src/lib.rs b/src/lib.rs index f100557a34..e0b4f6de08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,7 @@ impl VirtualAddress { } } -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub struct MemoryArea { pub base: PhysicalAddress, pub size: usize, diff --git a/src/main.rs b/src/main.rs index 9d7f3f2f18..31ed6150af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -200,7 +200,7 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let offset = bump_allocator.offset(); println!("Permanently used: {}", format_size(offset)); - let mut allocator = BuddyAllocator::::new(bump_allocator, true).unwrap(); + let mut allocator = BuddyAllocator::::new(bump_allocator).unwrap(); for i in 0..16 { let phys_opt = allocator.allocate_one(); println!("page {}: {:X?}", i, phys_opt); From db7869d99573e8249a04c3cc8c6f54348e9f77c4 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 13:52:22 -0600 Subject: [PATCH 040/120] Add table function for mapper, to get inner page table --- src/page/mapper.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index c2cc225843..fa60be1cc3 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -29,12 +29,16 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { A::set_table(self.table_addr); } - pub unsafe fn map(&mut self, virt: VirtualAddress, entry: PageEntry) -> Option<()> { - let mut table = PageTable::new( + pub unsafe fn table(&self) -> PageTable { + PageTable::new( VirtualAddress::new(0), self.table_addr, A::PAGE_LEVELS - 1 - ); + ) + } + + pub unsafe fn map(&mut self, virt: VirtualAddress, entry: PageEntry) -> Option<()> { + let mut table = self.table(); loop { let i = table.index_of(virt)?; if table.level() == 0 { From 6375c175f702e0a448baaa1a030347f2bc1387e2 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 13:52:45 -0600 Subject: [PATCH 041/120] Do not zero tables when not necessary in buddy allocator --- src/allocator/frame/buddy.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 654a8fb0e4..7d8fb0e8a6 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -102,9 +102,6 @@ impl BuddyAllocator { for _ in 0 .. map_pages { let map_phys = bump_allocator.allocate_one()?; let map_virt = A::phys_to_virt(map_phys); - for i in 0..Self::MAP_PAGE_BYTES { - A::write(map_virt.add(i), 0); - } A::write(map_virt.add(Self::MAP_PAGE_BYTES), BuddyMapFooter { next: entry.map, }); From e8ea483832a7ad3dab7aefad5fb0b3842e642db3 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 14:56:57 -0600 Subject: [PATCH 042/120] Add page flushing, add support for mapping anonymous pages --- src/main.rs | 33 +++++++++++++++------------ src/page/flush.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++++ src/page/mapper.rs | 34 ++++++++++++++++++++------- src/page/mod.rs | 8 ++++--- 4 files changed, 106 insertions(+), 26 deletions(-) create mode 100644 src/page/flush.rs diff --git a/src/main.rs b/src/main.rs index 31ed6150af..a531da1eb3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use rmm::{ PageMapper, PageTable, PhysicalAddress, + VirtualAddress, }; use core::{ @@ -178,17 +179,19 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { { // Map all physical areas at PHYS_OFFSET - let mut mapper = PageMapper::::new( + let mut mapper = PageMapper::::create( &mut bump_allocator ).expect("failed to create Mapper"); for area in areas.iter() { for i in 0..area.size / A::PAGE_SIZE { let phys = area.base.add(i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); - mapper.map( + let flush = mapper.map_phys( virt, - PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT) - ).expect("failed to map frame"); + phys, + A::ENTRY_FLAG_WRITABLE + ).expect("failed to map page to frame"); + flush.ignore(); // Not the active table } } @@ -211,17 +214,17 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { } } } - // for i in 0..16 { - // let phys_opt = allocator.allocate(2 * MEGABYTE); - // println!("2 MB page {}: {:X?}", i, phys_opt); - // if i % 2 == 0 { - // if let Some(phys) = phys_opt { - // allocator.free(phys, 2 * MEGABYTE); - // } - // } - // } - // - // println!("Remaining: {}", format_size(allocator.remaining())); + + let mut mapper = PageMapper::::current( + &mut allocator + ); + for i in 0..16 { + let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); + let flush = mapper.map( + virt, + A::ENTRY_FLAG_USER | A::ENTRY_FLAG_WRITABLE + ).expect("failed to map page"); + } } unsafe fn inner() { diff --git a/src/page/flush.rs b/src/page/flush.rs new file mode 100644 index 0000000000..9aae51f599 --- /dev/null +++ b/src/page/flush.rs @@ -0,0 +1,57 @@ +use core::{ + marker::PhantomData, + mem, +}; + +use crate::{ + Arch, + VirtualAddress, +}; + +#[must_use = "The page table must be flushed, or the changes unsafely ignored"] +pub struct PageFlush { + virt: VirtualAddress, + phantom: PhantomData, +} + +impl PageFlush { + pub fn new(virt: VirtualAddress) -> Self { + Self { + virt, + phantom: PhantomData, + } + } + + pub fn flush(self) { + unsafe { A::invalidate(self.virt); } + } + + pub unsafe fn ignore(self) { + mem::forget(self); + } +} + +#[must_use = "The page table must be flushed, or the changes unsafely ignored"] +pub struct PageFlushAll { + phantom: PhantomData, +} + +impl PageFlushAll { + pub fn new() -> Self { + Self { + phantom: PhantomData, + } + } + + pub fn consume(&self, flush: PageFlush) { + unsafe { flush.ignore(); } + } + + pub fn flush(self) { + unsafe { A::invalidate_all(); } + } + + pub unsafe fn ignore(self) { + mem::forget(self); + } +} diff --git a/src/page/mapper.rs b/src/page/mapper.rs index fa60be1cc3..9fc80884b4 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -4,6 +4,7 @@ use crate::{ Arch, FrameAllocator, PageEntry, + PageFlush, PageTable, PhysicalAddress, VirtualAddress, @@ -16,13 +17,22 @@ pub struct PageMapper<'f, A, F> { } impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { - pub unsafe fn new(allocator: &'f mut F) -> Option { - let table_addr = allocator.allocate_one()?; - Some(Self { + pub unsafe fn new(table_addr: PhysicalAddress, allocator: &'f mut F) -> Self { + Self { table_addr, allocator, phantom: PhantomData, - }) + } + } + + pub unsafe fn create(allocator: &'f mut F) -> Option { + let table_addr = allocator.allocate_one()?; + Some(Self::new(table_addr, allocator)) + } + + pub unsafe fn current(allocator: &'f mut F) -> Self { + let table_addr = A::table(); + Self::new(table_addr, allocator) } pub unsafe fn activate(&mut self) { @@ -37,22 +47,30 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { ) } - pub unsafe fn map(&mut self, virt: VirtualAddress, entry: PageEntry) -> Option<()> { + pub unsafe fn map(&mut self, virt: VirtualAddress, flags: usize) -> Option> { + let phys = self.allocator.allocate_one()?; + self.map_phys(virt, phys, flags) + } + + pub unsafe fn map_phys(&mut self, virt: VirtualAddress, phys: PhysicalAddress, flags: usize) -> Option> { + //TODO: verify virt and phys are aligned + //TODO: verify flags have correct bits + let entry = PageEntry::new(phys.data() | flags | A::ENTRY_FLAG_PRESENT); let mut table = self.table(); loop { let i = table.index_of(virt)?; if table.level() == 0 { //TODO: check for overwriting entry table.set_entry(i, entry); - return Some(()); + return Some(PageFlush::new(virt)); } else { let next_opt = table.next(i); let next = match next_opt { Some(some) => some, None => { - let phys = self.allocator.allocate_one()?; + let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - table.set_entry(i, PageEntry::new(phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); + table.set_entry(i, PageEntry::new(next_phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); table.next(i)? } }; diff --git a/src/page/mod.rs b/src/page/mod.rs index 268d3be9d3..48a07e717d 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -1,9 +1,11 @@ pub use self::{ - entry::PageEntry, - mapper::PageMapper, - table::PageTable, + entry::*, + flush::*, + mapper::*, + table::*, }; mod entry; +mod flush; mod mapper; mod table; From f97a80fecb98a10037ca937e72462a2ed1c62059 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 15:11:26 -0600 Subject: [PATCH 043/120] Flush user table changes --- src/main.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main.rs b/src/main.rs index a531da1eb3..a027100a18 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use rmm::{ FrameAllocator, MemoryArea, PageEntry, + PageFlushAll, PageMapper, PageTable, PhysicalAddress, @@ -218,13 +219,16 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let mut mapper = PageMapper::::current( &mut allocator ); + let flush_all = PageFlushAll::new(); for i in 0..16 { let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); let flush = mapper.map( virt, A::ENTRY_FLAG_USER | A::ENTRY_FLAG_WRITABLE ).expect("failed to map page"); + flush_all.consume(flush); } + flush_all.flush(); } unsafe fn inner() { From 5990a04e1398f9376f60393fb0df07003a606d32 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 15:13:01 -0600 Subject: [PATCH 044/120] Rename Mapper::active to Mapper:make_current --- src/main.rs | 3 +-- src/page/mapper.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index a027100a18..eb899ca8df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,6 @@ use rmm::{ EmulateArch, FrameAllocator, MemoryArea, - PageEntry, PageFlushAll, PageMapper, PageTable, @@ -197,7 +196,7 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { } // Use the new table - mapper.activate(); + mapper.make_current(); } // Create the physical memory map diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 9fc80884b4..0c9b03e6eb 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -35,7 +35,7 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { Self::new(table_addr, allocator) } - pub unsafe fn activate(&mut self) { + pub unsafe fn make_current(&mut self) { A::set_table(self.table_addr); } From e6d93d5743a92a10a6c51e0316abc5ce549cc65a Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 15:51:13 -0600 Subject: [PATCH 045/120] Allow const creation of buddy allocator --- src/allocator/frame/buddy.rs | 13 ++++++++++++- src/lib.rs | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 7d8fb0e8a6..f8b4f5ffce 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -47,6 +47,13 @@ impl BuddyAllocator { const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; + pub const unsafe fn empty() -> Self { + Self { + table_virt: VirtualAddress::new(0), + phantom: PhantomData, + } + } + pub unsafe fn new(mut bump_allocator: BumpAllocator) -> Option { // Allocate buddy table let table_phys = bump_allocator.allocate_one()?; @@ -132,7 +139,7 @@ impl BuddyAllocator { impl FrameAllocator for BuddyAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option { //TODO: support other sizes - if count.data() != 1 { + if self.table_virt.data() == 0 || count.data() != 1 { return None; } @@ -171,6 +178,10 @@ impl FrameAllocator for BuddyAllocator { } unsafe fn free(&mut self, base: PhysicalAddress, count: FrameCount) { + if self.table_virt.data() == 0 { + return; + } + let size = count.data() * A::PAGE_SIZE; for i in 0 .. Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::()); diff --git a/src/lib.rs b/src/lib.rs index e0b4f6de08..a4ae108f9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ #![cfg_attr(not(feature = "std"), no_std)] #![feature(asm)] +#![feature(const_fn)] pub use crate::{ allocator::*, From fb88d1669f3a28d2e8ee84266c8aa3ce0bce8eb1 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 19:59:13 -0600 Subject: [PATCH 046/120] Add unmap function --- src/main.rs | 10 ++++++++++ src/page/mapper.rs | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/main.rs b/src/main.rs index eb899ca8df..801389fbe4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -228,6 +228,16 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { flush_all.consume(flush); } flush_all.flush(); + + let flush_all = PageFlushAll::new(); + for i in 0..16 { + let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); + let flush = mapper.unmap( + virt, + ).expect("failed to unmap page"); + flush_all.consume(flush); + } + flush_all.flush(); } unsafe fn inner() { diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 0c9b03e6eb..524e170719 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -78,4 +78,27 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { } } } + + pub unsafe fn unmap(&mut self, virt: VirtualAddress) -> Option> { + let (old, flush) = self.unmap_phys(virt)?; + self.allocator.free_one(old.address()); + Some(flush) + } + + pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress) -> Option<(PageEntry, PageFlush)> { + //TODO: verify virt is aligned + let mut table = self.table(); + //TODO: unmap parents + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + let entry_opt = table.entry(i); + table.set_entry(i, PageEntry::new(0)); + let entry = entry_opt?; + return Some((entry, PageFlush::new(virt))); + } else { + table = table.next(i)?; + } + } + } } From 811dd09de44ae86e3a547a5abf805dd1458652c0 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 8 Sep 2020 20:20:56 -0600 Subject: [PATCH 047/120] Remove unused function for empty buddy allocator --- src/allocator/frame/buddy.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index f8b4f5ffce..23e9ef54c7 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -47,13 +47,6 @@ impl BuddyAllocator { const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; - pub const unsafe fn empty() -> Self { - Self { - table_virt: VirtualAddress::new(0), - phantom: PhantomData, - } - } - pub unsafe fn new(mut bump_allocator: BumpAllocator) -> Option { // Allocate buddy table let table_phys = bump_allocator.allocate_one()?; From fed3110ae89ad10c4b980edd82cfb65a0715464b Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 9 Sep 2020 10:55:29 -0600 Subject: [PATCH 048/120] Better unimplemented messages --- src/allocator/frame/buddy.rs | 2 +- src/allocator/frame/bump.rs | 2 +- src/arch/emulate.rs | 2 +- src/arch/x86_64.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 23e9ef54c7..28ccec0695 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -190,7 +190,7 @@ impl FrameAllocator for BuddyAllocator { //TODO: improve performance let mut map_phys = entry.map; loop { - if map_phys.data() == 0 { unimplemented!() } + if map_phys.data() == 0 { unimplemented!("map_phys.data() == 0") } let map_virt = A::phys_to_virt(map_phys); if map_page == 0 { let map_byte_virt = map_virt.add(map_bit / 8); diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 7f4a2a2243..ebeff49f52 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -54,6 +54,6 @@ impl FrameAllocator for BumpAllocator { } unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { - unimplemented!(); + unimplemented!("BumpAllocator::free not implemented"); } } diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index f8d3f019a3..851684c732 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -228,7 +228,7 @@ impl Machine { } fn invalidate(&mut self, _address: VirtualAddress) { - unimplemented!(); + unimplemented!("EmulateArch::invalidate not implemented"); } //TODO: cleanup diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index f5d3bf084b..c2e2acfbd8 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -23,7 +23,7 @@ impl Arch for X8664Arch { const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards unsafe fn init() -> &'static [MemoryArea] { - unimplemented!() + unimplemented!("X8664Arch::init unimplemented"); } #[inline(always)] From 711414223be0ca520940f9aa1d7e5de3a2a31e95 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 9 Sep 2020 11:11:39 -0600 Subject: [PATCH 049/120] Fix buddy map footer read address --- src/allocator/frame/buddy.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 28ccec0695..a88110721b 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -163,7 +163,7 @@ impl FrameAllocator for BuddyAllocator { offset += A::PAGE_SIZE * 8; } - let footer = A::read::(map_virt); + let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); map_phys = footer.next; } } @@ -199,7 +199,7 @@ impl FrameAllocator for BuddyAllocator { A::write(map_byte_virt, value); break; } else { - let footer = A::read::(map_virt); + let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); map_phys = footer.next; map_page -= 1; } From 0c44dde349ded1273a11aa3a543be36a7a56e7d4 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 9 Sep 2020 16:02:44 -0600 Subject: [PATCH 050/120] Implement multi-page allocations in buddy allocator (poorly) --- src/allocator/frame/buddy.rs | 83 ++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index a88110721b..1dfa1fba72 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -131,34 +131,49 @@ impl BuddyAllocator { impl FrameAllocator for BuddyAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option { - //TODO: support other sizes - if self.table_virt.data() == 0 || count.data() != 1 { + if self.table_virt.data() == 0 { return None; } - for i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(i * mem::size_of::()); + for entry_i in 0 .. Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(entry_i * mem::size_of::()); let entry = A::read::(virt); + let mut start_map_phys = entry.map; + let mut start_offset = 0; + let mut start_i = 0; + let mut start_bit = 0; + let mut start_page_phys = PhysicalAddress::new(0); + let mut found = 0; + //TODO: improve performance - let mut map_phys = entry.map; - let mut offset = 0; - while map_phys.data() != 0 { + let mut map_phys = start_map_phys; + let mut offset = start_offset; + 'find: while map_phys.data() != 0 { let map_virt = A::phys_to_virt(map_phys); for i in 0 .. Self::MAP_PAGE_BYTES { let map_byte_virt = map_virt.add(i); - let mut value: u8 = A::read(map_byte_virt); + let value: u8 = A::read(map_byte_virt); if (value & u8::MAX) != 0 { for bit in 0..8 { if (value & (1 << bit)) != 0 { - value &= !(1 << bit); - A::write(map_byte_virt, value); - let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); - let page_virt = A::phys_to_virt(page_phys); - A::write_bytes(page_virt, 0, A::PAGE_SIZE); - return Some(page_phys); + if found == 0 { + start_map_phys = map_phys; + start_offset = offset; + start_i = i; + start_bit = bit; + start_page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); + } + found += 1; + if found == count.data() { + break 'find; + } + } else { + found = 0; } } + } else { + found = 0; } offset += A::PAGE_SIZE * 8; } @@ -166,6 +181,46 @@ impl FrameAllocator for BuddyAllocator { let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); map_phys = footer.next; } + + if found == count.data() { + map_phys = start_map_phys; + offset = start_offset; + found = 0; + while map_phys.data() != 0 { + let map_virt = A::phys_to_virt(map_phys); + for i in start_i .. Self::MAP_PAGE_BYTES { + let map_byte_virt = map_virt.add(i); + let mut value: u8 = A::read(map_byte_virt); + if (value & u8::MAX) != 0 { + for bit in start_bit..8 { + if (value & (1 << bit)) != 0 { + value &= !(1 << bit); + A::write(map_byte_virt, value); + + let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); + let page_virt = A::phys_to_virt(page_phys); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); + + found += 1; + if found == count.data() { + return Some(start_page_phys); + } + } else { + panic!("check your logic, bit was set"); + } + } + start_bit = 0; + } else { + panic!("check your logic, byte was set"); + } + offset += A::PAGE_SIZE * 8; + } + start_i = 0; + + let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); + map_phys = footer.next; + } + } } None } From f17a1b52bd144dcc7ab10aebd4ecc977dbc47055 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 9 Sep 2020 16:02:59 -0600 Subject: [PATCH 051/120] Test multi-page allocation --- src/main.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 801389fbe4..1c7b72d425 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ use rmm::{ BumpAllocator, EmulateArch, FrameAllocator, + FrameCount, MemoryArea, PageFlushAll, PageMapper, @@ -204,13 +205,27 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { println!("Permanently used: {}", format_size(offset)); let mut allocator = BuddyAllocator::::new(bump_allocator).unwrap(); + for i in 0..16 { - let phys_opt = allocator.allocate_one(); - println!("page {}: {:X?}", i, phys_opt); - if i % 2 == 0 { - if let Some(phys) = phys_opt { - println!("free {}: {:X?}", i, phys_opt); - allocator.free_one(phys); + { + let phys_opt = allocator.allocate_one(); + println!("page {}: {:X?}", i, phys_opt); + if i % 3 == 0 { + if let Some(phys) = phys_opt { + println!("free {}: {:X?}", i, phys_opt); + allocator.free_one(phys); + } + } + } + + { + let phys_opt = allocator.allocate(FrameCount::new(16)); + println!("page*16 {}: {:X?}", i, phys_opt); + if i % 2 == 0 { + if let Some(phys) = phys_opt { + println!("free*16 {}: {:X?}", i, phys_opt); + allocator.free(phys, FrameCount::new(16)); + } } } } From a12be1b17266f2df893d08d7dd917b0da6875f5c Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 9 Sep 2020 20:19:28 -0600 Subject: [PATCH 052/120] Keep track of last free page --- src/allocator/frame/buddy.rs | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 1dfa1fba72..2b9a6685ea 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -34,7 +34,7 @@ impl BuddyEntry { #[repr(packed)] struct BuddyMapFooter { next: PhysicalAddress, - //TODO: index of last known free bit + skip: usize, } pub struct BuddyAllocator { @@ -104,6 +104,7 @@ impl BuddyAllocator { let map_virt = A::phys_to_virt(map_phys); A::write(map_virt.add(Self::MAP_PAGE_BYTES), BuddyMapFooter { next: entry.map, + skip: Self::MAP_PAGE_BYTES, }); entry.map = map_phys; } @@ -151,7 +152,11 @@ impl FrameAllocator for BuddyAllocator { let mut offset = start_offset; 'find: while map_phys.data() != 0 { let map_virt = A::phys_to_virt(map_phys); - for i in 0 .. Self::MAP_PAGE_BYTES { + let footer_virt = map_virt.add(Self::MAP_PAGE_BYTES); + let mut footer = A::read::(footer_virt); + offset += footer.skip * A::PAGE_SIZE * 8; + + for i in footer.skip .. Self::MAP_PAGE_BYTES { let map_byte_virt = map_virt.add(i); let value: u8 = A::read(map_byte_virt); if (value & u8::MAX) != 0 { @@ -166,19 +171,31 @@ impl FrameAllocator for BuddyAllocator { } found += 1; if found == count.data() { + // If skip is set to the start of the newly allocated frames + if footer.skip == start_i && footer.skip != i { + // Set it to the end of the newly allocated frames + footer.skip = i; + A::write(footer_virt, footer); + } break 'find; } } else { found = 0; } } + } else { + // If skip is set to the current 8 frames, and they are already used + if footer.skip == i { + // Set skip to the next current frame + footer.skip = i + 1; + A::write(footer_virt, footer); + } found = 0; } offset += A::PAGE_SIZE * 8; } - let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); map_phys = footer.next; } @@ -217,7 +234,8 @@ impl FrameAllocator for BuddyAllocator { } start_i = 0; - let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); + let footer_virt = map_virt.add(Self::MAP_PAGE_BYTES); + let footer = A::read::(footer_virt); map_phys = footer.next; } } @@ -247,14 +265,20 @@ impl FrameAllocator for BuddyAllocator { loop { if map_phys.data() == 0 { unimplemented!("map_phys.data() == 0") } let map_virt = A::phys_to_virt(map_phys); + let footer_virt = map_virt.add(Self::MAP_PAGE_BYTES); + let mut footer = A::read::(footer_virt); if map_page == 0 { - let map_byte_virt = map_virt.add(map_bit / 8); + let map_byte = map_bit / 8; + if map_byte < footer.skip { + footer.skip = map_byte; + A::write(footer_virt, footer); + } + let map_byte_virt = map_virt.add(map_byte); let mut value: u8 = A::read(map_byte_virt); value |= 1 << (map_bit % 8); A::write(map_byte_virt, value); break; } else { - let footer = A::read::(map_virt.add(Self::MAP_PAGE_BYTES)); map_phys = footer.next; map_page -= 1; } From a775c9e98724e8c2b2608954b6ea9d9c5c7becba Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 14 Sep 2020 09:08:28 -0600 Subject: [PATCH 053/120] Require Arch to implement Clone and Copy --- src/arch/emulate.rs | 1 + src/arch/mod.rs | 3 ++- src/arch/x86_64.rs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 851684c732..38b51c1e65 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -15,6 +15,7 @@ use crate::{ arch::x86_64::X8664Arch, }; +#[derive(Clone, Copy)] pub struct EmulateArch; impl Arch for EmulateArch { diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 53efdd3628..27681785ce 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -14,7 +14,7 @@ pub use self::x86_64::X8664Arch; mod emulate; mod x86_64; -pub trait Arch { +pub trait Arch: Clone + Copy { const PAGE_SHIFT: usize; const PAGE_ENTRY_SHIFT: usize; const PAGE_LEVELS: usize; @@ -71,6 +71,7 @@ pub trait Arch { unsafe fn set_table(address: PhysicalAddress); + #[inline(always)] unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { VirtualAddress::new(phys.data() + Self::PHYS_OFFSET) } diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index c2e2acfbd8..9225ee8d0a 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -5,6 +5,7 @@ use crate::{ VirtualAddress, }; +#[derive(Clone, Copy)] pub struct X8664Arch; impl Arch for X8664Arch { From e94d7e77722fbfcd8e2b6abec2ff92975ed03253 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 14 Sep 2020 09:41:42 -0600 Subject: [PATCH 054/120] Rewrite buddy allocator --- src/allocator/frame/buddy.rs | 325 +++++++++++++++++------------------ 1 file changed, 153 insertions(+), 172 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 2b9a6685ea..6153b19d4b 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -12,29 +12,61 @@ use crate::{ VirtualAddress, }; +#[repr(transparent)] +struct BuddyUsage(u8); + #[derive(Clone, Copy, Debug)] #[repr(packed)] -struct BuddyEntry { +struct BuddyEntry { base: PhysicalAddress, size: usize, - map: PhysicalAddress, + // Number of first free page + skip: usize, + // Count of used pages + used: usize, + phantom: PhantomData, } -impl BuddyEntry { - pub fn empty() -> Self { +impl BuddyEntry { + fn empty() -> Self { Self { base: PhysicalAddress::new(0), size: 0, - map: PhysicalAddress::new(0), + skip: 0, + used: 0, + phantom: PhantomData, } } -} -#[derive(Clone, Copy, Debug)] -#[repr(packed)] -struct BuddyMapFooter { - next: PhysicalAddress, - skip: usize, + #[inline(always)] + fn pages(&self) -> usize { + self.size >> A::PAGE_SHIFT + } + + fn usage_pages(&self) -> usize { + let bytes = self.pages() * mem::size_of::(); + // Round bytes used for usage to next page + (bytes + A::PAGE_OFFSET_MASK) >> A::PAGE_SHIFT + } + + unsafe fn usage_addr(&self, page: usize) -> Option { + if page < self.pages() { + let phys = self.base.add(page * mem::size_of::()); + Some(A::phys_to_virt(phys)) + } else { + None + } + } + + unsafe fn usage(&self, page: usize) -> Option { + let addr = self.usage_addr(page)?; + Some(A::read(addr)) + } + + unsafe fn set_usage(&self, page: usize, usage: BuddyUsage) -> Option<()> { + let addr = self.usage_addr(page)?; + Some(A::write(addr, usage)) + } } pub struct BuddyAllocator { @@ -43,17 +75,15 @@ pub struct BuddyAllocator { } impl BuddyAllocator { - const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::(); - const MAP_PAGE_BYTES: usize = (A::PAGE_SIZE - mem::size_of::()); - const MAP_PAGE_BITS: usize = Self::MAP_PAGE_BYTES * 8; + const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::>(); pub unsafe fn new(mut bump_allocator: BumpAllocator) -> Option { // Allocate buddy table let table_phys = bump_allocator.allocate_one()?; let table_virt = A::phys_to_virt(table_phys); - for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { - let virt = table_virt.add(i * mem::size_of::()); - A::write(virt, BuddyEntry::empty()); + for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { + let virt = table_virt.add(i * mem::size_of::>()); + A::write(virt, BuddyEntry::::empty()); } let mut allocator = Self { @@ -61,11 +91,22 @@ impl BuddyAllocator { phantom: PhantomData, }; - // Add areas to buddy table, combining areas when possible - for area in bump_allocator.areas().iter() { - for i in 0 .. (A::PAGE_SIZE / mem::size_of::()) { - let virt = table_virt.add(i * mem::size_of::()); - let mut entry = A::read::(virt); + // Add areas to buddy table, combining areas when possible, and skipping frames used + // by the bump allocator + let mut offset = bump_allocator.offset(); + for old_area in bump_allocator.areas().iter() { + let mut area = old_area.clone(); + if offset >= area.size { + offset -= area.size; + continue; + } else if offset > 0 { + area.base = area.base.add(offset); + area.size -= offset; + offset = 0; + } + for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { + let virt = table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); let inserted = if area.base.add(area.size) == entry.base { // Combine entry at start entry.base = area.base; @@ -94,36 +135,32 @@ impl BuddyAllocator { // Allocate buddy maps for i in 0 .. Self::BUDDY_ENTRIES { - let virt = table_virt.add(i * mem::size_of::()); - let mut entry = A::read::(virt); - if entry.size > 0 { - let pages = entry.size / A::PAGE_SIZE; - let map_pages = (pages + (Self::MAP_PAGE_BITS - 1)) / Self::MAP_PAGE_BITS; - for _ in 0 .. map_pages { - let map_phys = bump_allocator.allocate_one()?; - let map_virt = A::phys_to_virt(map_phys); - A::write(map_virt.add(Self::MAP_PAGE_BYTES), BuddyMapFooter { - next: entry.map, - skip: Self::MAP_PAGE_BYTES, - }); - entry.map = map_phys; + let virt = table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); + + // Only set up entries that have enough space for their own usage map + let usage_pages = entry.usage_pages(); + if entry.pages() > usage_pages { + // Mark all usage bytes as unused + let usage_start = entry.usage_addr(0)?; + for page in 0..usage_pages { + A::write_bytes(usage_start.add(page << A::PAGE_SHIFT), 0, A::PAGE_SIZE); } - A::write(virt, entry); + // Mark bytes used for usage as used + for page in 0..usage_pages { + entry.set_usage(page, BuddyUsage(1))?; + } } - } - // Mark unused areas as free - let mut area_offset = bump_allocator.offset(); - for area in bump_allocator.areas().iter() { - if area_offset < area.size { - let area_base = area.base.add(area_offset); - let area_size = area.size - area_offset; - allocator.free(area_base, FrameCount::new(area_size / A::PAGE_SIZE)); - area_offset = 0; - } else { - area_offset -= area.size; - } + // Skip the pages used for usage + entry.skip = usage_pages; + + // Set used pages to pages used for usage + entry.used = usage_pages; + + // Write updated entry + A::write(virt, entry); } Some(allocator) @@ -137,109 +174,53 @@ impl FrameAllocator for BuddyAllocator { } for entry_i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(entry_i * mem::size_of::()); - let entry = A::read::(virt); + let virt = self.table_virt.add(entry_i * mem::size_of::>()); + let mut entry = A::read::>(virt); - let mut start_map_phys = entry.map; - let mut start_offset = 0; - let mut start_i = 0; - let mut start_bit = 0; - let mut start_page_phys = PhysicalAddress::new(0); - let mut found = 0; + let mut free_page = entry.skip; + let mut free_count = 0; + for page in entry.skip .. entry.pages() { + let mut usage = entry.usage(page)?; + if usage.0 == 0 { + free_count += 1; - //TODO: improve performance - let mut map_phys = start_map_phys; - let mut offset = start_offset; - 'find: while map_phys.data() != 0 { - let map_virt = A::phys_to_virt(map_phys); - let footer_virt = map_virt.add(Self::MAP_PAGE_BYTES); - let mut footer = A::read::(footer_virt); - offset += footer.skip * A::PAGE_SIZE * 8; - - for i in footer.skip .. Self::MAP_PAGE_BYTES { - let map_byte_virt = map_virt.add(i); - let value: u8 = A::read(map_byte_virt); - if (value & u8::MAX) != 0 { - for bit in 0..8 { - if (value & (1 << bit)) != 0 { - if found == 0 { - start_map_phys = map_phys; - start_offset = offset; - start_i = i; - start_bit = bit; - start_page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); - } - found += 1; - if found == count.data() { - // If skip is set to the start of the newly allocated frames - if footer.skip == start_i && footer.skip != i { - // Set it to the end of the newly allocated frames - footer.skip = i; - A::write(footer_virt, footer); - } - break 'find; - } - } else { - found = 0; - } - } - - } else { - // If skip is set to the current 8 frames, and they are already used - if footer.skip == i { - // Set skip to the next current frame - footer.skip = i + 1; - A::write(footer_virt, footer); - } - found = 0; + if free_count == count.data() { + break; } - offset += A::PAGE_SIZE * 8; + } else { + free_page = page + 1; + free_count = 0; } - - map_phys = footer.next; } - if found == count.data() { - map_phys = start_map_phys; - offset = start_offset; - found = 0; - while map_phys.data() != 0 { - let map_virt = A::phys_to_virt(map_phys); - for i in start_i .. Self::MAP_PAGE_BYTES { - let map_byte_virt = map_virt.add(i); - let mut value: u8 = A::read(map_byte_virt); - if (value & u8::MAX) != 0 { - for bit in start_bit..8 { - if (value & (1 << bit)) != 0 { - value &= !(1 << bit); - A::write(map_byte_virt, value); + if free_count == count.data() { + for page in free_page .. free_page + free_count { + // Update usage + let mut usage = entry.usage(page)?; + usage.0 += 1; + entry.set_usage(page, usage); - let page_phys = entry.base.add(offset + bit * A::PAGE_SIZE); - let page_virt = A::phys_to_virt(page_phys); - A::write_bytes(page_virt, 0, A::PAGE_SIZE); - - found += 1; - if found == count.data() { - return Some(start_page_phys); - } - } else { - panic!("check your logic, bit was set"); - } - } - start_bit = 0; - } else { - panic!("check your logic, byte was set"); - } - offset += A::PAGE_SIZE * 8; - } - start_i = 0; - - let footer_virt = map_virt.add(Self::MAP_PAGE_BYTES); - let footer = A::read::(footer_virt); - map_phys = footer.next; + // Zero page + let page_phys = entry.base.add(page << A::PAGE_SHIFT); + let page_virt = A::phys_to_virt(page_phys); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); } + + // Update skip if necessary + if entry.skip == free_page { + entry.skip = free_page + free_count; + } + + // Update used page count + entry.used += free_count; + + // Write updated entry + A::write(virt, entry); + + return Some(entry.base.add(free_page << A::PAGE_SHIFT)); } } + None } @@ -250,40 +231,40 @@ impl FrameAllocator for BuddyAllocator { let size = count.data() * A::PAGE_SIZE; for i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(i * mem::size_of::()); - let entry = A::read::(virt); - if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { - //TODO: Correct logic - for page in 0 .. count.data() { - let page_base = base.add(page * A::PAGE_SIZE); - let index = (page_base.data() - entry.base.data()) / A::PAGE_SIZE; - let mut map_page = index / Self::MAP_PAGE_BITS; - let map_bit = index % Self::MAP_PAGE_BITS; + let virt = self.table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); - //TODO: improve performance - let mut map_phys = entry.map; - loop { - if map_phys.data() == 0 { unimplemented!("map_phys.data() == 0") } - let map_virt = A::phys_to_virt(map_phys); - let footer_virt = map_virt.add(Self::MAP_PAGE_BYTES); - let mut footer = A::read::(footer_virt); - if map_page == 0 { - let map_byte = map_bit / 8; - if map_byte < footer.skip { - footer.skip = map_byte; - A::write(footer_virt, footer); - } - let map_byte_virt = map_virt.add(map_byte); - let mut value: u8 = A::read(map_byte_virt); - value |= 1 << (map_bit % 8); - A::write(map_byte_virt, value); - break; - } else { - map_phys = footer.next; - map_page -= 1; - } + if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { + let start_page = (base.data() - entry.base.data()) >> A::PAGE_SHIFT; + let end_page = start_page + count.data(); + + for page in start_page..start_page + count.data() { + let mut usage = entry.usage(page).expect("failed to get usage during free"); + + if usage.0 > 0 { + usage.0 -= 1; + } else { + panic!("tried to free already free frame"); } + + // If page was freed + if usage.0 == 0 { + // Update skip if necessary + if page < entry.skip { + entry.skip = page; + } + + // Update used page count + entry.used -= 1; + } + + entry.set_usage(page, usage).expect("failed to set usage during free"); } + + // Write updated entry + A::write(virt, entry); + + return; } } } From 8e0df608e26afb3527973574e888226155ebf43e Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 14 Sep 2020 09:41:56 -0600 Subject: [PATCH 055/120] Add allocator usage information --- src/allocator/frame/buddy.rs | 16 ++++++++++++++++ src/allocator/frame/bump.rs | 13 +++++++++++++ src/allocator/frame/mod.rs | 21 +++++++++++++++++++++ src/main.rs | 6 ++++++ 4 files changed, 56 insertions(+) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 6153b19d4b..1832012a96 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -8,6 +8,7 @@ use crate::{ BumpAllocator, FrameAllocator, FrameCount, + FrameUsage, PhysicalAddress, VirtualAddress, }; @@ -268,4 +269,19 @@ impl FrameAllocator for BuddyAllocator { } } } + + unsafe fn usage(&self) -> FrameUsage { + let mut total = 0; + let mut used = 0; + for i in 0 .. Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::>()); + let entry = A::read::>(virt); + total += entry.size >> A::PAGE_SHIFT; + used += entry.used; + } + FrameUsage { + used: FrameCount::new(used), + total: FrameCount::new(total), + } + } } diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index ebeff49f52..5231187c87 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -4,6 +4,7 @@ use crate::{ Arch, FrameAllocator, FrameCount, + FrameUsage, MemoryArea, PhysicalAddress, }; @@ -56,4 +57,16 @@ impl FrameAllocator for BumpAllocator { unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { unimplemented!("BumpAllocator::free not implemented"); } + + unsafe fn usage(&self) -> FrameUsage { + let mut total = 0; + for area in self.areas.iter() { + total += area.size >> A::PAGE_SHIFT; + } + let used = self.offset >> A::PAGE_SHIFT; + FrameUsage { + used: FrameCount::new(used), + total: FrameCount::new(total), + } + } } diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index 33d07943bf..bdb772675e 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -20,6 +20,25 @@ impl FrameCount { } } +pub struct FrameUsage { + used: FrameCount, + total: FrameCount, +} + +impl FrameUsage { + pub fn used(&self) -> FrameCount { + self.used + } + + pub fn free(&self) -> FrameCount { + FrameCount(self.total.0 - self.used.0) + } + + pub fn total(&self) -> FrameCount { + self.total + } +} + pub trait FrameAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option; @@ -32,4 +51,6 @@ pub trait FrameAllocator { unsafe fn free_one(&mut self, address: PhysicalAddress) { self.free(address, FrameCount::new(1)); } + + unsafe fn usage(&self) -> FrameUsage; } diff --git a/src/main.rs b/src/main.rs index 1c7b72d425..f6b94891da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -253,6 +253,12 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { flush_all.consume(flush); } flush_all.flush(); + + let usage = allocator.usage(); + println!("Allocator usage:"); + println!(" Used: {}", format_size(usage.used().data() * A::PAGE_SIZE)); + println!(" Free: {}", format_size(usage.free().data() * A::PAGE_SIZE)); + println!(" Total: {}", format_size(usage.total().data() * A::PAGE_SIZE)); } unsafe fn inner() { From 936352a04901ed733d5f3e4de77b3d037e63215b Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 14 Sep 2020 09:47:23 -0600 Subject: [PATCH 056/120] FrameCount::new function --- src/allocator/frame/buddy.rs | 5 +---- src/allocator/frame/bump.rs | 5 +---- src/allocator/frame/mod.rs | 4 ++++ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 1832012a96..0be62acc2a 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -279,9 +279,6 @@ impl FrameAllocator for BuddyAllocator { total += entry.size >> A::PAGE_SHIFT; used += entry.used; } - FrameUsage { - used: FrameCount::new(used), - total: FrameCount::new(total), - } + FrameUsage::new(FrameCount::new(used), FrameCount::new(total)) } } diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 5231187c87..b044984c8b 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -64,9 +64,6 @@ impl FrameAllocator for BumpAllocator { total += area.size >> A::PAGE_SHIFT; } let used = self.offset >> A::PAGE_SHIFT; - FrameUsage { - used: FrameCount::new(used), - total: FrameCount::new(total), - } + FrameUsage::new(FrameCount::new(used), FrameCount::new(total)) } } diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index bdb772675e..5b6898a236 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -26,6 +26,10 @@ pub struct FrameUsage { } impl FrameUsage { + pub fn new(used: FrameCount, total: FrameCount) -> Self { + Self { used, total } + } + pub fn used(&self) -> FrameCount { self.used } From cdbeecfffedf802a6fd61d93b767ff273c055d80 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 14 Sep 2020 09:47:29 -0600 Subject: [PATCH 057/120] Remove some warnings --- src/allocator/frame/buddy.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 0be62acc2a..3e371b3091 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -16,7 +16,7 @@ use crate::{ #[repr(transparent)] struct BuddyUsage(u8); -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy)] #[repr(packed)] struct BuddyEntry { base: PhysicalAddress, @@ -87,7 +87,7 @@ impl BuddyAllocator { A::write(virt, BuddyEntry::::empty()); } - let mut allocator = Self { + let allocator = Self { table_virt, phantom: PhantomData, }; @@ -181,7 +181,7 @@ impl FrameAllocator for BuddyAllocator { let mut free_page = entry.skip; let mut free_count = 0; for page in entry.skip .. entry.pages() { - let mut usage = entry.usage(page)?; + let usage = entry.usage(page)?; if usage.0 == 0 { free_count += 1; @@ -237,8 +237,6 @@ impl FrameAllocator for BuddyAllocator { if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { let start_page = (base.data() - entry.base.data()) >> A::PAGE_SHIFT; - let end_page = start_page + count.data(); - for page in start_page..start_page + count.data() { let mut usage = entry.usage(page).expect("failed to get usage during free"); From 9a716604fcbb5638d0a1e6c527bad8776fc07a4e Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 20 Dec 2020 16:36:52 +0100 Subject: [PATCH 058/120] Implement Copy and Clone manually for BuddyEntry. This fixes a warning that may in the future become an error, about the possibility for unaligned references, since the derive macros apparently rely on creating references to fields. Unaligned references are direct UB. --- src/allocator/frame/buddy.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 3e371b3091..16240ec883 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -16,7 +16,6 @@ use crate::{ #[repr(transparent)] struct BuddyUsage(u8); -#[derive(Clone, Copy)] #[repr(packed)] struct BuddyEntry { base: PhysicalAddress, @@ -28,6 +27,19 @@ struct BuddyEntry { phantom: PhantomData, } +impl Clone for BuddyEntry { + fn clone(&self) -> Self { + Self { + base: self.base, + size: self.size, + skip: self.skip, + used: self.used, + phantom: PhantomData, + } + } +} +impl Copy for BuddyEntry {} + impl BuddyEntry { fn empty() -> Self { Self { From fad48af98563da297bb2032ce9170a7fc049e969 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 13 Jan 2021 10:47:35 -0700 Subject: [PATCH 059/120] WIP: aarch64 support --- src/arch/aarch64.rs | 70 +++++++++++++++++++++++++++++++++++++++++++++ src/arch/mod.rs | 2 ++ 2 files changed, 72 insertions(+) create mode 100644 src/arch/aarch64.rs diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs new file mode 100644 index 0000000000..7a22a5a9d8 --- /dev/null +++ b/src/arch/aarch64.rs @@ -0,0 +1,70 @@ +use crate::{ + Arch, + MemoryArea, + PhysicalAddress, + VirtualAddress, +}; + +#[derive(Clone, Copy)] +pub struct AArch64Arch; + +impl Arch for AArch64Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + + //TODO + const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_WRITABLE: usize = 1 << 1; + const ENTRY_FLAG_USER: usize = 1 << 2; + const ENTRY_FLAG_HUGE: usize = 1 << 7; + const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; + + const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("AArch64Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + unimplemented!() + } + + #[inline(always)] + unsafe fn table() -> PhysicalAddress { + unimplemented!() + } + + #[inline(always)] + unsafe fn set_table(address: PhysicalAddress) { + unimplemented!() + } +} + +#[cfg(test)] +mod tests { + use crate::Arch; + use super::AArch64Arch; + + #[test] + fn constants() { + assert_eq!(AArch64Arch::PAGE_SIZE, 4096); + assert_eq!(AArch64Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(AArch64Arch::PAGE_ADDRESS_SHIFT, 48); + assert_eq!(AArch64Arch::PAGE_ADDRESS_SIZE, 0x0001_0000_0000_0000); + assert_eq!(AArch64Arch::PAGE_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(AArch64Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(AArch64Arch::PAGE_ENTRIES, 512); + assert_eq!(AArch64Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(AArch64Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); + + assert_eq!(AArch64Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); + assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(AArch64Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + + assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 27681785ce..5825bdc987 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -6,10 +6,12 @@ use crate::{ VirtualAddress, }; +pub use self::aarch64::AArch64Arch; #[cfg(feature = "std")] pub use self::emulate::EmulateArch; pub use self::x86_64::X8664Arch; +mod aarch64; #[cfg(feature = "std")] mod emulate; mod x86_64; From dafd9cb3c4a1d4917c4e3c41dc17e4e4db1be084 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Thu, 14 Jan 2021 09:16:37 -0700 Subject: [PATCH 060/120] Add ENTRY_FLAG_DEFAULT_PAGE and ENTRY_FLAG_DEFAULT_TABLE --- src/arch/emulate.rs | 4 ++-- src/arch/mod.rs | 4 ++-- src/arch/x86_64.rs | 6 ++++-- src/page/mapper.rs | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 38b51c1e65..65db01049d 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -24,11 +24,11 @@ impl Arch for EmulateArch { const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS; const ENTRY_ADDRESS_SHIFT: usize = X8664Arch::ENTRY_ADDRESS_SHIFT; + const ENTRY_FLAG_DEFAULT_PAGE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_PAGE; + const ENTRY_FLAG_DEFAULT_TABLE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_TABLE; const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_WRITABLE: usize = X8664Arch::ENTRY_FLAG_WRITABLE; const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; - const ENTRY_FLAG_HUGE: usize = X8664Arch::ENTRY_FLAG_HUGE; - const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 5825bdc987..72b8eef1c5 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -22,11 +22,11 @@ pub trait Arch: Clone + Copy { const PAGE_LEVELS: usize; const ENTRY_ADDRESS_SHIFT: usize; + const ENTRY_FLAG_DEFAULT_PAGE: usize; + const ENTRY_FLAG_DEFAULT_TABLE: usize; const ENTRY_FLAG_PRESENT: usize; const ENTRY_FLAG_WRITABLE: usize; const ENTRY_FLAG_USER: usize; - const ENTRY_FLAG_HUGE: usize; - const ENTRY_FLAG_GLOBAL: usize; const ENTRY_FLAG_NO_EXEC: usize; const PHYS_OFFSET: usize; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 9225ee8d0a..4768a2b9bd 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -14,11 +14,13 @@ impl Arch for X8664Arch { const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_FLAG_DEFAULT_PAGE: usize = ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_WRITABLE: usize = 1 << 1; const ENTRY_FLAG_USER: usize = 1 << 2; - const ENTRY_FLAG_HUGE: usize = 1 << 7; - const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; + // Not used: const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 524e170719..ee9f7b9277 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -55,7 +55,7 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { pub unsafe fn map_phys(&mut self, virt: VirtualAddress, phys: PhysicalAddress, flags: usize) -> Option> { //TODO: verify virt and phys are aligned //TODO: verify flags have correct bits - let entry = PageEntry::new(phys.data() | flags | A::ENTRY_FLAG_PRESENT); + let entry = PageEntry::new(phys.data() | flags | A::ENTRY_FLAG_DEFAULT_PAGE); let mut table = self.table(); loop { let i = table.index_of(virt)?; @@ -70,7 +70,7 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - table.set_entry(i, PageEntry::new(next_phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_PRESENT)); + table.set_entry(i, PageEntry::new(next_phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_DEFAULT_TABLE)); table.next(i)? } }; From c5774c55293ea6293510723ab6e4318dd2467376 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Thu, 14 Jan 2021 09:16:46 -0700 Subject: [PATCH 061/120] Add TableKind for future use --- src/lib.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a4ae108f9e..81d047f23a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,7 +17,17 @@ pub const MEGABYTE: usize = KILOBYTE * KILOBYTE; pub const GIGABYTE: usize = KILOBYTE * MEGABYTE; pub const TERABYTE: usize = KILOBYTE * GIGABYTE; -// Physical memory address +/// Specific table to be used, needed on some architectures +//TODO: Use this throughout the code +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum TableKind { + /// Userspace page table + User, + /// Kernel page table + Kernel, +} + +/// Physical memory address #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct PhysicalAddress(usize); @@ -39,7 +49,7 @@ impl PhysicalAddress { } } -// Virtual memory address +/// Virtual memory address #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct VirtualAddress(usize); From 1214f3dcdc06077e659ec9292768c296eb6b5c3b Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Thu, 14 Jan 2021 10:01:36 -0700 Subject: [PATCH 062/120] Add PageFlags to abstract differences between architectures --- src/arch/aarch64.rs | 57 +++++++++++++++++++++++++++++++++++------- src/arch/emulate.rs | 9 ++++--- src/arch/mod.rs | 3 ++- src/arch/x86_64.rs | 7 +++--- src/main.rs | 5 ++-- src/page/flags.rs | 61 +++++++++++++++++++++++++++++++++++++++++++++ src/page/mapper.rs | 9 ++++--- src/page/mod.rs | 2 ++ 8 files changed, 130 insertions(+), 23 deletions(-) create mode 100644 src/page/flags.rs diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 7a22a5a9d8..caa1bb666a 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -2,6 +2,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -11,16 +12,26 @@ pub struct AArch64Arch; impl Arch for AArch64Arch { const PAGE_SHIFT: usize = 12; // 4096 bytes const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each - const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT + const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 //TODO const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_FLAG_DEFAULT_PAGE: usize + = Self::ENTRY_FLAG_PRESENT + | 1 << 10 // Access flag + ; + const ENTRY_FLAG_DEFAULT_TABLE: usize + = Self::ENTRY_FLAG_PRESENT + | 1 << 1 // Table flag + | 1 << 10 // Access flag + ; const ENTRY_FLAG_PRESENT: usize = 1 << 0; - const ENTRY_FLAG_WRITABLE: usize = 1 << 1; - const ENTRY_FLAG_USER: usize = 1 << 2; - const ENTRY_FLAG_HUGE: usize = 1 << 7; - const ENTRY_FLAG_GLOBAL: usize = 1 << 8; - const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; + const ENTRY_FLAG_READONLY: usize = 1 << 7; + const ENTRY_FLAG_READWRITE: usize = 0; + const ENTRY_FLAG_USER: usize = 1 << 6; + // This sets both userspace and privileged execute never + //TODO: Separate the two? + const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards @@ -30,17 +41,45 @@ impl Arch for AArch64Arch { #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - unimplemented!() + //TODO: can one address be invalidated? + Self::invalidate_all(); + } + + #[inline(always)] + unsafe fn invalidate_all() { + asm!("tlbi vmalle1is"); } #[inline(always)] unsafe fn table() -> PhysicalAddress { - unimplemented!() + let address: usize; + //TODO: set this dynamically + let table_kind = TableKind::Kernel; + match table_kind { + TableKind::User => { + asm!("mrs {0}, ttbr0_el1", out(reg) address); + }, + TableKind::Kernel => { + asm!("mrs {0}, ttbr1_el1", out(reg) address); + } + } + PhysicalAddress::new(address) } #[inline(always)] unsafe fn set_table(address: PhysicalAddress) { - unimplemented!() + //TODO: set this dynamically + let table_kind = TableKind::Kernel; + match table_kind { + TableKind::User => { + asm!("msr ttbr0_el1, {0}", in(reg) address.data()); + }, + TableKind::Kernel => { + asm!("msr ttbr1_el1, {0}", in(reg) address.data()); + } + } + //TODO: Does this need to be called? + Self::invalidate_all(); } } diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 65db01049d..6f0342808d 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -27,7 +27,8 @@ impl Arch for EmulateArch { const ENTRY_FLAG_DEFAULT_PAGE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_PAGE; const ENTRY_FLAG_DEFAULT_TABLE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_TABLE; const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; - const ENTRY_FLAG_WRITABLE: usize = X8664Arch::ENTRY_FLAG_WRITABLE; + const ENTRY_FLAG_READONLY: usize = X8664Arch::ENTRY_FLAG_READONLY; + const ENTRY_FLAG_READWRITE: usize = X8664Arch::ENTRY_FLAG_READWRITE; const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; @@ -40,7 +41,7 @@ impl Arch for EmulateArch { // PML4 index 256 (PHYS_OFFSET) link to PDP let pml4 = 0; let pdp = pml4 + Self::PAGE_SIZE; - let flags = Self::ENTRY_FLAG_WRITABLE | Self::ENTRY_FLAG_PRESENT; + let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT; machine.write_phys::(PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), pdp | flags); // PDP link to PD @@ -200,7 +201,7 @@ impl Machine { } if let Some((phys, flags)) = self.translate(virt) { - if flags & A::ENTRY_FLAG_WRITABLE != 0 { + if flags & A::ENTRY_FLAG_READWRITE != 0 { self.write_phys(phys, value); } else { panic!("write: 0x{:X} size 0x{:X} not writable", virt_data, size); @@ -218,7 +219,7 @@ impl Machine { } if let Some((phys, flags)) = self.translate(virt) { - if flags & A::ENTRY_FLAG_WRITABLE != 0 { + if flags & A::ENTRY_FLAG_READWRITE != 0 { self.write_phys_bytes(phys, value, count); } else { panic!("write_bytes: 0x{:X} count 0x{:X} not writable", virt_data, count); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 72b8eef1c5..ceecab5c41 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -25,7 +25,8 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_DEFAULT_PAGE: usize; const ENTRY_FLAG_DEFAULT_TABLE: usize; const ENTRY_FLAG_PRESENT: usize; - const ENTRY_FLAG_WRITABLE: usize; + const ENTRY_FLAG_READONLY: usize; + const ENTRY_FLAG_READWRITE: usize; const ENTRY_FLAG_USER: usize; const ENTRY_FLAG_NO_EXEC: usize; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 4768a2b9bd..19ff13883f 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -14,10 +14,11 @@ impl Arch for X8664Arch { const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT const ENTRY_ADDRESS_SHIFT: usize = 52; - const ENTRY_FLAG_DEFAULT_PAGE: usize = ENTRY_FLAG_PRESENT; - const ENTRY_FLAG_DEFAULT_TABLE: usize = ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; - const ENTRY_FLAG_WRITABLE: usize = 1 << 1; + const ENTRY_FLAG_READONLY: usize = 0; + const ENTRY_FLAG_READWRITE: usize = 1 << 1; const ENTRY_FLAG_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; // Not used: const ENTRY_FLAG_GLOBAL: usize = 1 << 8; diff --git a/src/main.rs b/src/main.rs index f6b94891da..6264e7bf35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use rmm::{ FrameAllocator, FrameCount, MemoryArea, + PageFlags, PageFlushAll, PageMapper, PageTable, @@ -190,7 +191,7 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let flush = mapper.map_phys( virt, phys, - A::ENTRY_FLAG_WRITABLE + PageFlags::::new().write(true) ).expect("failed to map page to frame"); flush.ignore(); // Not the active table } @@ -238,7 +239,7 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); let flush = mapper.map( virt, - A::ENTRY_FLAG_USER | A::ENTRY_FLAG_WRITABLE + PageFlags::::new().user(true).write(true) ).expect("failed to map page"); flush_all.consume(flush); } diff --git a/src/page/flags.rs b/src/page/flags.rs new file mode 100644 index 0000000000..432c7fc44c --- /dev/null +++ b/src/page/flags.rs @@ -0,0 +1,61 @@ +use core::marker::PhantomData; + +use crate::Arch; + +#[derive(Clone, Copy, Debug)] +pub struct PageFlags { + data: usize, + phantom: PhantomData, +} + +impl PageFlags { + #[inline(always)] + pub fn new() -> Self { + unsafe { + Self::from_data( + // Flags set to present, kernel space, read-only, no-execute by default + A::ENTRY_FLAG_DEFAULT_PAGE | + A::ENTRY_FLAG_READONLY | + A::ENTRY_FLAG_NO_EXEC + ) + } + } + + #[inline(always)] + pub unsafe fn from_data(data: usize) -> Self { + Self { data, phantom: PhantomData } + } + + #[inline(always)] + pub fn data(&self) -> usize { + self.data + } + + #[inline(always)] + pub fn custom_flag(mut self, flag: usize, value: bool) -> Self { + if value { + self.data |= flag; + } else { + self.data &= !flag; + } + self + } + + #[inline(always)] + pub fn user(self, value: bool) -> Self { + self.custom_flag(A::ENTRY_FLAG_USER, value) + } + + #[inline(always)] + pub fn write(self, value: bool) -> Self { + // Architecture may use readonly or readwrite, support either + self.custom_flag(A::ENTRY_FLAG_READONLY, !value) + .custom_flag(A::ENTRY_FLAG_READWRITE, value) + } + + #[inline(always)] + pub fn execute(self, value: bool) -> Self { + //TODO: write xor execute? + self.custom_flag(A::ENTRY_FLAG_NO_EXEC, !value) + } +} diff --git a/src/page/mapper.rs b/src/page/mapper.rs index ee9f7b9277..6617547ca7 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -4,6 +4,7 @@ use crate::{ Arch, FrameAllocator, PageEntry, + PageFlags, PageFlush, PageTable, PhysicalAddress, @@ -47,15 +48,15 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { ) } - pub unsafe fn map(&mut self, virt: VirtualAddress, flags: usize) -> Option> { + pub unsafe fn map(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { let phys = self.allocator.allocate_one()?; self.map_phys(virt, phys, flags) } - pub unsafe fn map_phys(&mut self, virt: VirtualAddress, phys: PhysicalAddress, flags: usize) -> Option> { + pub unsafe fn map_phys(&mut self, virt: VirtualAddress, phys: PhysicalAddress, flags: PageFlags) -> Option> { //TODO: verify virt and phys are aligned //TODO: verify flags have correct bits - let entry = PageEntry::new(phys.data() | flags | A::ENTRY_FLAG_DEFAULT_PAGE); + let entry = PageEntry::new(phys.data() | flags.data()); let mut table = self.table(); loop { let i = table.index_of(virt)?; @@ -70,7 +71,7 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - table.set_entry(i, PageEntry::new(next_phys.data() | A::ENTRY_FLAG_WRITABLE | A::ENTRY_FLAG_DEFAULT_TABLE)); + table.set_entry(i, PageEntry::new(next_phys.data() | A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE)); table.next(i)? } }; diff --git a/src/page/mod.rs b/src/page/mod.rs index 48a07e717d..bcf360b322 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -1,11 +1,13 @@ pub use self::{ entry::*, + flags::*, flush::*, mapper::*, table::*, }; mod entry; +mod flags; mod flush; mod mapper; mod table; From cb6b44d69e16925ba588731d8c14b0bbcc2619f8 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Thu, 14 Jan 2021 12:35:27 -0700 Subject: [PATCH 063/120] Use devmap offset for physmap on aarch64 --- src/arch/aarch64.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index caa1bb666a..bef46dfe89 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -33,7 +33,8 @@ impl Arch for AArch64Arch { //TODO: Separate the two? const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; - const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards + //TODO: adjust to match x86_64? + const PHYS_OFFSET: usize = 0xfffffe0000000000; unsafe fn init() -> &'static [MemoryArea] { unimplemented!("AArch64Arch::init unimplemented"); From 132d91d3aaa624d1bc8709555a64ff289f7d5e4f Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Thu, 14 Jan 2021 15:54:51 -0700 Subject: [PATCH 064/120] Fixed page flags for aarch64 --- src/arch/aarch64.rs | 1 + src/page/flags.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index bef46dfe89..758db81e4e 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -18,6 +18,7 @@ impl Arch for AArch64Arch { const ENTRY_ADDRESS_SHIFT: usize = 52; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT + | 1 << 1 // Page flag | 1 << 10 // Access flag ; const ENTRY_FLAG_DEFAULT_TABLE: usize diff --git a/src/page/flags.rs b/src/page/flags.rs index 432c7fc44c..91a181dd60 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -2,7 +2,7 @@ use core::marker::PhantomData; use crate::Arch; -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy)] pub struct PageFlags { data: usize, phantom: PhantomData, From c46d148e263e78a69ea04686c918105610cefb72 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 2 May 2021 21:25:27 -0600 Subject: [PATCH 065/120] Fix tests for aarch64 --- src/arch/aarch64.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 758db81e4e..5a89b4b6d6 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -106,6 +106,6 @@ mod tests { assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); assert_eq!(AArch64Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); - assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); } } From 2f74326384bf89f887dafe8d8cd5780a54092a90 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 3 May 2021 11:20:49 -0600 Subject: [PATCH 066/120] Add riscv64 sv39 and sv48 --- src/arch/aarch64.rs | 1 + src/arch/emulate.rs | 1 + src/arch/mod.rs | 6 +++ src/arch/riscv64/mod.rs | 5 +++ src/arch/riscv64/sv39.rs | 86 ++++++++++++++++++++++++++++++++++++++++ src/arch/riscv64/sv48.rs | 86 ++++++++++++++++++++++++++++++++++++++++ src/arch/x86_64.rs | 1 + src/page/flags.rs | 2 + 8 files changed, 188 insertions(+) create mode 100644 src/arch/riscv64/mod.rs create mode 100644 src/arch/riscv64/sv39.rs create mode 100644 src/arch/riscv64/sv48.rs diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 5a89b4b6d6..f9896e3879 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -33,6 +33,7 @@ impl Arch for AArch64Arch { // This sets both userspace and privileged execute never //TODO: Separate the two? const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; + const ENTRY_FLAG_EXEC: usize = 0; //TODO: adjust to match x86_64? const PHYS_OFFSET: usize = 0xfffffe0000000000; diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 6f0342808d..35098e2828 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -31,6 +31,7 @@ impl Arch for EmulateArch { const ENTRY_FLAG_READWRITE: usize = X8664Arch::ENTRY_FLAG_READWRITE; const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; + const ENTRY_FLAG_EXEC: usize = X8664Arch::ENTRY_FLAG_EXEC; const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index ceecab5c41..3c9d0e0c31 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -7,11 +7,16 @@ use crate::{ }; pub use self::aarch64::AArch64Arch; +pub use self::riscv64::{ + RiscV64Sv39Arch, + RiscV64Sv48Arch +}; #[cfg(feature = "std")] pub use self::emulate::EmulateArch; pub use self::x86_64::X8664Arch; mod aarch64; +mod riscv64; #[cfg(feature = "std")] mod emulate; mod x86_64; @@ -29,6 +34,7 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_READWRITE: usize; const ENTRY_FLAG_USER: usize; const ENTRY_FLAG_NO_EXEC: usize; + const ENTRY_FLAG_EXEC: usize; const PHYS_OFFSET: usize; diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs new file mode 100644 index 0000000000..894ef349e4 --- /dev/null +++ b/src/arch/riscv64/mod.rs @@ -0,0 +1,5 @@ +pub use sv39::RiscV64Sv39Arch; +pub use sv48::RiscV64Sv48Arch; + +mod sv39; +mod sv48; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs new file mode 100644 index 0000000000..e8cb7cd8f4 --- /dev/null +++ b/src/arch/riscv64/sv39.rs @@ -0,0 +1,86 @@ +use crate::{ + Arch, + MemoryArea, + PhysicalAddress, + VirtualAddress, +}; + +#[derive(Clone, Copy)] +pub struct RiscV64Sv39Arch; + +impl Arch for RiscV64Sv39Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 3; // L0, L1, L2 + + //TODO + const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_FLAG_DEFAULT_PAGE: usize + = Self::ENTRY_FLAG_PRESENT + | 1 << 1 // Read flag + ; + const ENTRY_FLAG_DEFAULT_TABLE: usize + = Self::ENTRY_FLAG_PRESENT + ; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 0; + const ENTRY_FLAG_READWRITE: usize = 1 << 2; + const ENTRY_FLAG_USER: usize = 1 << 4; + const ENTRY_FLAG_NO_EXEC: usize = 0; + const ENTRY_FLAG_EXEC: usize = 1 << 3; + + //TODO: adjust to match x86_64? + const PHYS_OFFSET: usize = 0xfffffe0000000000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("RiscV64Sv39Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + //TODO: can one address be invalidated? + Self::invalidate_all(); + } + + #[inline(always)] + unsafe fn table() -> PhysicalAddress { + let satp: usize; + asm!("csrr {0}, satp", out(reg) satp); + PhysicalAddress::new( + (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT // Convert from PPN + ) + } + + #[inline(always)] + unsafe fn set_table(address: PhysicalAddress) { + let satp = + (8 << 60) | // Sv39 MODE + (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) + asm!("csrw satp, {0}", in(reg) satp); + } +} + +#[cfg(test)] +mod tests { + use crate::Arch; + use super::RiscV64Sv39Arch; + + #[test] + fn constants() { + assert_eq!(RiscV64Sv39Arch::PAGE_SIZE, 4096); + assert_eq!(RiscV64Sv39Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(RiscV64Sv39Arch::PAGE_ADDRESS_SHIFT, 39); + assert_eq!(RiscV64Sv39Arch::PAGE_ADDRESS_SIZE, 0x0000_0080_0000_0000); + assert_eq!(RiscV64Sv39Arch::PAGE_ADDRESS_MASK, 0x0000_007F_FFFF_F000); + assert_eq!(RiscV64Sv39Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(RiscV64Sv39Arch::PAGE_ENTRIES, 512); + assert_eq!(RiscV64Sv39Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(RiscV64Sv39Arch::PAGE_NEGATIVE_MASK, 0xFFFF_FF80_0000_0000); + + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + + assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); + } +} diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs new file mode 100644 index 0000000000..f49076cc66 --- /dev/null +++ b/src/arch/riscv64/sv48.rs @@ -0,0 +1,86 @@ +use crate::{ + Arch, + MemoryArea, + PhysicalAddress, + VirtualAddress, +}; + +#[derive(Clone, Copy)] +pub struct RiscV64Sv48Arch; + +impl Arch for RiscV64Sv48Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each + const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 + + //TODO + const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_FLAG_DEFAULT_PAGE: usize + = Self::ENTRY_FLAG_PRESENT + | 1 << 1 // Read flag + ; + const ENTRY_FLAG_DEFAULT_TABLE: usize + = Self::ENTRY_FLAG_PRESENT + ; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 0; + const ENTRY_FLAG_READWRITE: usize = 1 << 2; + const ENTRY_FLAG_USER: usize = 1 << 4; + const ENTRY_FLAG_NO_EXEC: usize = 0; + const ENTRY_FLAG_EXEC: usize = 1 << 3; + + //TODO: adjust to match x86_64? + const PHYS_OFFSET: usize = 0xfffffe0000000000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("RiscV64Sv48Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + //TODO: can one address be invalidated? + Self::invalidate_all(); + } + + #[inline(always)] + unsafe fn table() -> PhysicalAddress { + let satp: usize; + asm!("csrr {0}, satp", out(reg) satp); + PhysicalAddress::new( + (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT // Convert from PPN + ) + } + + #[inline(always)] + unsafe fn set_table(address: PhysicalAddress) { + let satp = + (9 << 60) | // Sv48 MODE + (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) + asm!("csrw satp, {0}", in(reg) satp); + } +} + +#[cfg(test)] +mod tests { + use crate::Arch; + use super::RiscV64Sv48Arch; + + #[test] + fn constants() { + assert_eq!(RiscV64Sv48Arch::PAGE_SIZE, 4096); + assert_eq!(RiscV64Sv48Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(RiscV64Sv48Arch::PAGE_ADDRESS_SHIFT, 48); + assert_eq!(RiscV64Sv48Arch::PAGE_ADDRESS_SIZE, 0x0001_0000_0000_0000); + assert_eq!(RiscV64Sv48Arch::PAGE_ADDRESS_MASK, 0x0000_FFFF_FFFF_F000); + assert_eq!(RiscV64Sv48Arch::PAGE_ENTRY_SIZE, 8); + assert_eq!(RiscV64Sv48Arch::PAGE_ENTRIES, 512); + assert_eq!(RiscV64Sv48Arch::PAGE_ENTRY_MASK, 0x1FF); + assert_eq!(RiscV64Sv48Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); + + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + + assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); + } +} diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 19ff13883f..3b0a2572da 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -23,6 +23,7 @@ impl Arch for X8664Arch { // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; // Not used: const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; + const ENTRY_FLAG_EXEC: usize = 0; const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards diff --git a/src/page/flags.rs b/src/page/flags.rs index 91a181dd60..6a2f084ed3 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -56,6 +56,8 @@ impl PageFlags { #[inline(always)] pub fn execute(self, value: bool) -> Self { //TODO: write xor execute? + // Architecture may use no exec or exec, support either self.custom_flag(A::ENTRY_FLAG_NO_EXEC, !value) + .custom_flag(A::ENTRY_FLAG_EXEC, value) } } From 5e47692b8d6481c165f8f4723d7f83a04834609b Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 3 May 2021 16:39:04 -0600 Subject: [PATCH 067/120] Support more operations on PageFlags --- src/page/flags.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/page/flags.rs b/src/page/flags.rs index 6a2f084ed3..6058637e65 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -1,4 +1,7 @@ -use core::marker::PhantomData; +use core::{ + fmt, + marker::PhantomData +}; use crate::Arch; @@ -21,6 +24,18 @@ impl PageFlags { } } + #[inline(always)] + pub fn new_table() -> Self { + unsafe { + Self::from_data( + // Flags set to present, kernel space, read-only, no-execute by default + A::ENTRY_FLAG_DEFAULT_TABLE | + A::ENTRY_FLAG_READONLY | + A::ENTRY_FLAG_NO_EXEC + ) + } + } + #[inline(always)] pub unsafe fn from_data(data: usize) -> Self { Self { data, phantom: PhantomData } @@ -31,6 +46,7 @@ impl PageFlags { self.data } + #[must_use] #[inline(always)] pub fn custom_flag(mut self, flag: usize, value: bool) -> Self { if value { @@ -41,11 +57,28 @@ impl PageFlags { self } + #[inline(always)] + pub fn has_flag(&self, flag: usize) -> bool { + self.data & flag == flag + } + + #[inline(always)] + pub fn has_present(&self) -> bool { + self.has_flag(A::ENTRY_FLAG_PRESENT) + } + + #[must_use] #[inline(always)] pub fn user(self, value: bool) -> Self { self.custom_flag(A::ENTRY_FLAG_USER, value) } + #[inline(always)] + pub fn has_user(&self) -> bool { + self.has_flag(A::ENTRY_FLAG_USER) + } + + #[must_use] #[inline(always)] pub fn write(self, value: bool) -> Self { // Architecture may use readonly or readwrite, support either @@ -53,6 +86,13 @@ impl PageFlags { .custom_flag(A::ENTRY_FLAG_READWRITE, value) } + #[inline(always)] + pub fn has_write(&self) -> bool { + // Architecture may use readonly or readwrite, support either + self.data & (A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE) == A::ENTRY_FLAG_READWRITE + } + + #[must_use] #[inline(always)] pub fn execute(self, value: bool) -> Self { //TODO: write xor execute? @@ -60,4 +100,18 @@ impl PageFlags { self.custom_flag(A::ENTRY_FLAG_NO_EXEC, !value) .custom_flag(A::ENTRY_FLAG_EXEC, value) } + + #[inline(always)] + pub fn has_execute(&self) -> bool { + // Architecture may use no exec or exec, support either + self.data & (A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_EXEC) == A::ENTRY_FLAG_EXEC + } } + +impl fmt::Debug for PageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PageFlags") + .field("data", &self.data) + .finish() + } +} \ No newline at end of file From c81c4de223583b23d2f91f0978e378ef1dc3676f Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 3 May 2021 21:29:36 -0600 Subject: [PATCH 068/120] Add method to get TableKind from VirtualAddress --- src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 81d047f23a..c15793c598 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -69,6 +69,15 @@ impl VirtualAddress { pub fn add(self, offset: usize) -> Self { Self(self.0 + offset) } + + #[inline(always)] + pub fn kind(&self) -> TableKind { + if (self.0 as isize) < 0 { + TableKind::Kernel + } else { + TableKind::User + } + } } #[derive(Clone, Copy, Debug)] From c66956ca2ac1b4df3904afbafe53321a4b6c50af Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 17 Jun 2021 18:42:03 +0200 Subject: [PATCH 069/120] Remove unused #![feature(const_fn)]. --- src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index c15793c598..487e089e88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ #![cfg_attr(not(feature = "std"), no_std)] #![feature(asm)] -#![feature(const_fn)] pub use crate::{ allocator::*, From 32caee3095e54a8d9181841fe48d9ea8d43f4e6c Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 8 Jul 2021 13:27:08 +0200 Subject: [PATCH 070/120] Add arch functions for checking canonical addrs. --- src/arch/aarch64.rs | 13 ++++++++++++ src/arch/mod.rs | 4 ++++ src/arch/riscv64/sv39.rs | 8 +++++++ src/arch/riscv64/sv48.rs | 7 +++++++ src/arch/x86_64.rs | 45 +++++++++++++++++++++++++++++++++++++++- src/lib.rs | 2 +- 6 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index f9896e3879..3f5305a5d3 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -84,6 +84,19 @@ impl Arch for AArch64Arch { //TODO: Does this need to be called? Self::invalidate_all(); } + + fn virt_is_valid(address: VirtualAddress) -> bool { + // TODO + true + } + fn virt_kind(address: VirtualAddress) -> TableKind { + // TODO: Is this correct? + if address.data() & (1 << 63) == (1 << 63) { + TableKind::Kernel + } else { + TableKind::User + } + } } #[cfg(test)] diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 3c9d0e0c31..a24eae36e0 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -3,6 +3,7 @@ use core::ptr; use crate::{ MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -84,4 +85,7 @@ pub trait Arch: Clone + Copy { unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { VirtualAddress::new(phys.data() + Self::PHYS_OFFSET) } + + fn virt_is_valid(address: VirtualAddress) -> bool; + fn virt_kind(address: VirtualAddress) -> TableKind; } diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index e8cb7cd8f4..ced17ecb0d 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -2,6 +2,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -58,6 +59,13 @@ impl Arch for RiscV64Sv39Arch { (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); } + + fn virt_is_valid(address: VirtualAddress) -> bool { + todo!() + } + fn virt_kind(address: VirtualAddress) -> TableKind { + todo!() + } } #[cfg(test)] diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index f49076cc66..f26aeb7216 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -2,6 +2,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -58,6 +59,12 @@ impl Arch for RiscV64Sv48Arch { (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); } + fn virt_is_valid(address: VirtualAddress) -> bool { + todo!() + } + fn virt_kind(address: VirtualAddress) -> TableKind { + todo!() + } } #[cfg(test)] diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 3b0a2572da..ead8013100 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -2,6 +2,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -47,12 +48,37 @@ impl Arch for X8664Arch { unsafe fn set_table(address: PhysicalAddress) { asm!("mov cr3, {0}", in(reg) address.data()); } + + fn virt_is_valid(address: VirtualAddress) -> bool { + // On x86_64, an address is valid if and only if it is canonical. It may still point to + // unmapped memory, but will always be valid once translated via the page table has + // suceeded. + address.is_canonical() + } + fn virt_kind(address: VirtualAddress) -> TableKind { + if address.data() & (1 << 48) == (1 << 48) { + TableKind::Kernel + } else { + TableKind::User + } + } +} + +impl VirtualAddress { + #[cfg(any(doc, target_arch = "x86_64"))] + #[doc(cfg(target_arch = "x86_64"))] + pub fn is_canonical(self) -> bool { + let masked = self.data() & 0xFFFF_8000_0000_0000; + // TODO: 5-level paging + masked == 0xFFFF_8000_0000_0000 + || masked == 0 + } } #[cfg(test)] mod tests { use crate::Arch; - use super::X8664Arch; + use super::{VirtualAddress, X8664Arch}; #[test] fn constants() { @@ -72,4 +98,21 @@ mod tests { assert_eq!(X8664Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } + #[test] + fn is_canonical() { + fn yes(address: usize) { + assert!(VirtualAddress::new(address).is_canonical()); + } + fn no(address: usize) { + assert!(!VirtualAddress::new(address).is_canonical()); + } + + yes(0xFFFF_8000_1337_1337); + yes(0xFFFF_FFFF_FFFF_FFFF); + yes(0x0000_0000_0000_0042); + yes(0x0000_7FFF_FFFF_FFFF); + no(0x1337_0000_0000_0000); + no(0x1337_8000_0000_0000); + no(0x0000_8000_0000_0000); + } } diff --git a/src/lib.rs b/src/lib.rs index 487e089e88..58d27a564b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ #![cfg_attr(not(feature = "std"), no_std)] -#![feature(asm)] +#![feature(asm, doc_cfg)] pub use crate::{ allocator::*, From b75c329a273d194e313bc36a1ceab5362fd5f8e2 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 8 Jul 2021 13:35:51 +0200 Subject: [PATCH 071/120] Add virt_kind and virt_is_valid for emulation. --- src/arch/emulate.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 35098e2828..fade414bc0 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -101,6 +101,18 @@ impl Arch for EmulateArch { unsafe fn set_table(address: PhysicalAddress) { MACHINE.as_mut().unwrap().set_table(address); } + fn virt_kind(address: VirtualAddress) -> crate::TableKind { + // TODO + if address.data() & (1 << 63) == (1 << 63) { + crate::TableKind::Kernel + } else { + crate::TableKind::User + } + } + fn virt_is_valid(_address: VirtualAddress) -> bool { + // TODO + true + } } const MEMORY_SIZE: usize = 64 * MEGABYTE; From 6bc59e70131135984a41216c45e4dc3a6395a30a Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 8 Jul 2021 16:06:51 +0200 Subject: [PATCH 072/120] Remove virt_kind, impl virt_is_valid() for riscv. All there is left now is AArch64. --- src/arch/aarch64.rs | 10 +--------- src/arch/emulate.rs | 10 +--------- src/arch/mod.rs | 1 - src/arch/riscv64/sv39.rs | 33 +++++++++++++++++++++++++++++---- src/arch/riscv64/sv48.rs | 30 ++++++++++++++++++++++++++---- src/arch/x86_64.rs | 8 -------- 6 files changed, 57 insertions(+), 35 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 3f5305a5d3..be42f31aab 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -86,17 +86,9 @@ impl Arch for AArch64Arch { } fn virt_is_valid(address: VirtualAddress) -> bool { - // TODO + // FIXME true } - fn virt_kind(address: VirtualAddress) -> TableKind { - // TODO: Is this correct? - if address.data() & (1 << 63) == (1 << 63) { - TableKind::Kernel - } else { - TableKind::User - } - } } #[cfg(test)] diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index fade414bc0..a73bb084c8 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -101,16 +101,8 @@ impl Arch for EmulateArch { unsafe fn set_table(address: PhysicalAddress) { MACHINE.as_mut().unwrap().set_table(address); } - fn virt_kind(address: VirtualAddress) -> crate::TableKind { - // TODO - if address.data() & (1 << 63) == (1 << 63) { - crate::TableKind::Kernel - } else { - crate::TableKind::User - } - } fn virt_is_valid(_address: VirtualAddress) -> bool { - // TODO + // TODO: Don't see why an emulated arch would have any problems with canonicalness... true } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index a24eae36e0..7194e16cec 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -87,5 +87,4 @@ pub trait Arch: Clone + Copy { } fn virt_is_valid(address: VirtualAddress) -> bool; - fn virt_kind(address: VirtualAddress) -> TableKind; } diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index ced17ecb0d..7b14130b5b 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -61,10 +61,10 @@ impl Arch for RiscV64Sv39Arch { } fn virt_is_valid(address: VirtualAddress) -> bool { - todo!() - } - fn virt_kind(address: VirtualAddress) -> TableKind { - todo!() + const MASK: usize = 0xFFFF_FFC0_0000_0000; + let masked = address.data() & MASK; + + masked == MASK || masked == 0 } } @@ -91,4 +91,29 @@ mod tests { assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); } + #[test] + fn is_canonical() { + use super::VirtualAddress; + + #[track_caller] + fn yes(addr: usize) { + assert!(RiscV64Sv39Arch::virt_is_valid(VirtualAddress::new(addr))); + } + #[track_caller] + fn no(addr: usize) { + assert!(!RiscV64Sv39Arch::virt_is_valid(VirtualAddress::new(addr))); + } + + yes(0xFFFF_FFFF_FFFF_FFFF); + yes(0xFFFF_FFF0_1337_1337); + no(0x0000_0F00_0000_0000); + no(0x1337_0000_0000_0000); + no(1 << 38); + yes(1 << 37); + + // Check for off-by-one errors. + yes(0xFFFF_FFC0_0000_0000 | (1 << 37)); + yes(0xFFFF_FFE0_0000_0000 | (1 << 37)); + no(0xFFFF_FF80_0000_0000 | (1 << 37)); + } } diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index f26aeb7216..5e1eff9d59 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -60,10 +60,12 @@ impl Arch for RiscV64Sv48Arch { asm!("csrw satp, {0}", in(reg) satp); } fn virt_is_valid(address: VirtualAddress) -> bool { - todo!() - } - fn virt_kind(address: VirtualAddress) -> TableKind { - todo!() + // RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64. + let mask = 0xFFFF_8000_0000_0000; + let masked = address.data() & mask; + + masked == mask + || masked == 0 } } @@ -90,4 +92,24 @@ mod tests { assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); } + #[test] + fn is_canonical() { + use super::VirtualAddress; + + // Close to identical when compared to x86_64 test. + fn yes(address: usize) { + assert!(RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); + } + fn no(address: usize) { + assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); + } + + yes(0xFFFF_8000_1337_1337); + yes(0xFFFF_FFFF_FFFF_FFFF); + yes(0x0000_0000_0000_0042); + yes(0x0000_7FFF_FFFF_FFFF); + no(0x1337_0000_0000_0000); + no(0x1337_8000_0000_0000); + no(0x0000_8000_0000_0000); + } } diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index ead8013100..da100f7bee 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -2,7 +2,6 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, - TableKind, VirtualAddress, }; @@ -55,13 +54,6 @@ impl Arch for X8664Arch { // suceeded. address.is_canonical() } - fn virt_kind(address: VirtualAddress) -> TableKind { - if address.data() & (1 << 48) == (1 << 48) { - TableKind::Kernel - } else { - TableKind::User - } - } } impl VirtualAddress { From e2648005bace5893ee6adb02d1e8ae33f36ae7c0 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 6 Mar 2022 11:34:22 +0100 Subject: [PATCH 073/120] Update to latest toolchain. --- src/arch/aarch64.rs | 2 ++ src/arch/riscv64/sv39.rs | 2 ++ src/arch/riscv64/sv48.rs | 2 ++ src/arch/x86_64.rs | 2 ++ src/lib.rs | 2 +- 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index be42f31aab..3d8711adb7 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -1,3 +1,5 @@ +use core::arch::asm; + use crate::{ Arch, MemoryArea, diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 7b14130b5b..2e52caeb3e 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -1,3 +1,5 @@ +use core::arch::asm; + use crate::{ Arch, MemoryArea, diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 5e1eff9d59..a16d8391d8 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -1,3 +1,5 @@ +use core::arch::asm; + use crate::{ Arch, MemoryArea, diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index da100f7bee..555de30c18 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -1,3 +1,5 @@ +use core::arch::asm; + use crate::{ Arch, MemoryArea, diff --git a/src/lib.rs b/src/lib.rs index 58d27a564b..099c240427 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ #![cfg_attr(not(feature = "std"), no_std)] -#![feature(asm, doc_cfg)] +#![feature(doc_cfg)] pub use crate::{ allocator::*, From ec284ca8e34f62f54e209e69a15813f62d2a97eb Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 26 Jul 2022 19:01:42 -0600 Subject: [PATCH 074/120] Update rust-toolchain --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 40973dae63..9eed51006f 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-07-27 +nightly-2022-03-18 From 0944b17983223966e339a25f9328bdb77a59d5c7 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Tue, 26 Jul 2022 20:13:55 -0600 Subject: [PATCH 075/120] Add initial x86 32-bit support --- src/arch/mod.rs | 39 ++++++++++++++-------- src/arch/x86.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++++ src/arch/x86_64.rs | 3 +- src/lib.rs | 1 + src/main.rs | 2 ++ 5 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 src/arch/x86.rs diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 7194e16cec..62c1bb5ee7 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -7,19 +7,32 @@ use crate::{ VirtualAddress, }; -pub use self::aarch64::AArch64Arch; -pub use self::riscv64::{ - RiscV64Sv39Arch, - RiscV64Sv48Arch +//TODO: Support having all page tables compile on all architectures +#[cfg(target_pointer_width = "32")] +pub use self::{ + x86::X86Arch, }; -#[cfg(feature = "std")] +#[cfg(target_pointer_width = "64")] +pub use self::{ + aarch64::AArch64Arch, + riscv64::{ + RiscV64Sv39Arch, + RiscV64Sv48Arch + }, + x86_64::X8664Arch, +}; +#[cfg(all(feature = "std", target_pointer_width = "64"))] pub use self::emulate::EmulateArch; -pub use self::x86_64::X8664Arch; +#[cfg(target_pointer_width = "64")] mod aarch64; +#[cfg(target_pointer_width = "64")] mod riscv64; -#[cfg(feature = "std")] +#[cfg(all(feature = "std", target_pointer_width = "64"))] mod emulate; +#[cfg(target_pointer_width = "32")] +mod x86; +#[cfg(target_pointer_width = "64")] mod x86_64; pub trait Arch: Clone + Copy { @@ -42,16 +55,16 @@ pub trait Arch: Clone + Copy { const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT; const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1; const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT; - const PAGE_ADDRESS_SIZE: usize = 1 << Self::PAGE_ADDRESS_SHIFT; - const PAGE_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_SIZE - Self::PAGE_SIZE; + const PAGE_ADDRESS_SIZE: u64 = 1 << (Self::PAGE_ADDRESS_SHIFT as u64); + const PAGE_ADDRESS_MASK: usize = (Self::PAGE_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; - const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1); + const PAGE_NEGATIVE_MASK: usize = !((Self::PAGE_ADDRESS_SIZE as u64) - 1) as usize; - const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_SHIFT; - const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - Self::PAGE_SIZE; - const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; + const ENTRY_ADDRESS_SIZE: u64 = 1 << Self::ENTRY_ADDRESS_SHIFT; + const ENTRY_ADDRESS_MASK: usize = (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; + const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK as usize; unsafe fn init() -> &'static [MemoryArea]; diff --git a/src/arch/x86.rs b/src/arch/x86.rs new file mode 100644 index 0000000000..0572f4c5fc --- /dev/null +++ b/src/arch/x86.rs @@ -0,0 +1,83 @@ +//TODO: USE PAE +use core::arch::asm; + +use crate::{ + Arch, + MemoryArea, + PhysicalAddress, + VirtualAddress, +}; + +#[derive(Clone, Copy)] +pub struct X86Arch; + +impl Arch for X86Arch { + const PAGE_SHIFT: usize = 12; // 4096 bytes + const PAGE_ENTRY_SHIFT: usize = 10; // 1024 entries, 4 bytes each + const PAGE_LEVELS: usize = 2; // PD, PT + + const ENTRY_ADDRESS_SHIFT: usize = 32; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_PRESENT: usize = 1 << 0; + const ENTRY_FLAG_READONLY: usize = 0; + const ENTRY_FLAG_READWRITE: usize = 1 << 1; + const ENTRY_FLAG_USER: usize = 1 << 2; + // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; + // Not used: const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_EXEC: usize = 0; // NOT AVAILABLE UNLESS PAE IS USED! + const ENTRY_FLAG_EXEC: usize = 0; + + const PHYS_OFFSET: usize = 0x8000_0000; + + unsafe fn init() -> &'static [MemoryArea] { + unimplemented!("X86Arch::init unimplemented"); + } + + #[inline(always)] + unsafe fn invalidate(address: VirtualAddress) { + asm!("invlpg [{0}]", in(reg) address.data()); + } + + #[inline(always)] + unsafe fn table() -> PhysicalAddress { + let address: usize; + asm!("mov {0}, cr3", out(reg) address); + PhysicalAddress::new(address) + } + + #[inline(always)] + unsafe fn set_table(address: PhysicalAddress) { + asm!("mov cr3, {0}", in(reg) address.data()); + } + + fn virt_is_valid(address: VirtualAddress) -> bool { + // On 32-bit x86, every virtual address is valid + true + } +} + +#[cfg(test)] +mod tests { + use crate::Arch; + use super::{VirtualAddress, X86Arch}; + + #[test] + fn constants() { + assert_eq!(X86Arch::PAGE_SIZE, 4096); + assert_eq!(X86Arch::PAGE_OFFSET_MASK, 0xFFF); + assert_eq!(X86Arch::PAGE_ADDRESS_SHIFT, 32); + assert_eq!(X86Arch::PAGE_ADDRESS_SIZE, 0x0000_0001_0000_0000); + assert_eq!(X86Arch::PAGE_ADDRESS_MASK, 0xFFFF_F000); + assert_eq!(X86Arch::PAGE_ENTRY_SIZE, 4); + assert_eq!(X86Arch::PAGE_ENTRIES, 1024); + assert_eq!(X86Arch::PAGE_ENTRY_MASK, 0x3FF); + assert_eq!(X86Arch::PAGE_NEGATIVE_MASK, 0x0000_0000_0000); + + assert_eq!(X86Arch::ENTRY_ADDRESS_SIZE, 0x0000_0001_0000_0000); + assert_eq!(X86Arch::ENTRY_ADDRESS_MASK, 0xFFFF_F000); + assert_eq!(X86Arch::ENTRY_FLAGS_MASK, 0x0000_0FFF); + + assert_eq!(X86Arch::PHYS_OFFSET, 0x8000_0000); + } +} diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 555de30c18..74c37548f5 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -27,7 +27,7 @@ impl Arch for X8664Arch { const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; const ENTRY_FLAG_EXEC: usize = 0; - const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1); // PML4 slot 256 and onwards + const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1) as usize; // PML4 slot 256 and onwards unsafe fn init() -> &'static [MemoryArea] { unimplemented!("X8664Arch::init unimplemented"); @@ -59,7 +59,6 @@ impl Arch for X8664Arch { } impl VirtualAddress { - #[cfg(any(doc, target_arch = "x86_64"))] #[doc(cfg(target_arch = "x86_64"))] pub fn is_canonical(self) -> bool { let masked = self.data() & 0xFFFF_8000_0000_0000; diff --git a/src/lib.rs b/src/lib.rs index 099c240427..51c6f938b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod page; pub const KILOBYTE: usize = 1024; pub const MEGABYTE: usize = KILOBYTE * KILOBYTE; pub const GIGABYTE: usize = KILOBYTE * MEGABYTE; +#[cfg(target_pointer_width = "64")] pub const TERABYTE: usize = KILOBYTE * GIGABYTE; /// Specific table to be used, needed on some architectures diff --git a/src/main.rs b/src/main.rs index 6264e7bf35..621a1df9e5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,5 @@ +#![cfg(target_pointer_width = "64")] + use rmm::{ KILOBYTE, MEGABYTE, From 2f16dddf25ba3acabd253118f05a38e9edb6091d Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Mon, 4 Jul 2022 10:41:53 +0200 Subject: [PATCH 076/120] Add a page flusher trait. --- src/page/flush.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/page/flush.rs b/src/page/flush.rs index 9aae51f599..5dedf6afae 100644 --- a/src/page/flush.rs +++ b/src/page/flush.rs @@ -8,6 +8,10 @@ use crate::{ VirtualAddress, }; +pub trait Flusher { + fn consume(&mut self, flush: PageFlush); +} + #[must_use = "The page table must be flushed, or the changes unsafely ignored"] pub struct PageFlush { virt: VirtualAddress, @@ -43,10 +47,6 @@ impl PageFlushAll { } } - pub fn consume(&self, flush: PageFlush) { - unsafe { flush.ignore(); } - } - pub fn flush(self) { unsafe { A::invalidate_all(); } } @@ -55,3 +55,16 @@ impl PageFlushAll { mem::forget(self); } } +impl Flusher for PageFlushAll { + fn consume(&mut self, flush: PageFlush) { + unsafe { flush.ignore(); } + } +} +impl + ?Sized> Flusher for &mut T { + fn consume(&mut self, flush: PageFlush) { + >::consume(self, flush) + } +} +impl Flusher for () { + fn consume(&mut self, _: PageFlush) {} +} From c847b1e2a88dee08ad8813d334d6cf0e4df6ab94 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 17 Jul 2022 13:25:34 +0200 Subject: [PATCH 077/120] Implement FrameAllocator for mutable refs of impls. --- src/allocator/frame/mod.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index 5b6898a236..99c9d3215c 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -20,6 +20,7 @@ impl FrameCount { } } +#[derive(Debug)] pub struct FrameUsage { used: FrameCount, total: FrameCount, @@ -58,3 +59,21 @@ pub trait FrameAllocator { unsafe fn usage(&self) -> FrameUsage; } + +impl FrameAllocator for &mut T where T: FrameAllocator { + unsafe fn allocate(&mut self, count: FrameCount) -> Option { + T::allocate(self, count) + } + unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) { + T::free(self, address, count) + } + unsafe fn allocate_one(&mut self) -> Option { + T::allocate_one(self) + } + unsafe fn free_one(&mut self, address: PhysicalAddress) { + T::free_one(self, address) + } + unsafe fn usage(&self) -> FrameUsage { + T::usage(self) + } +} From 7a209c83c9af82108d8853b88dab4bf30c650572 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 17 Jul 2022 13:26:10 +0200 Subject: [PATCH 078/120] Implement Debug for X8664Arch. --- src/arch/x86_64.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 74c37548f5..1008df8e8f 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -7,7 +7,7 @@ use crate::{ VirtualAddress, }; -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct X8664Arch; impl Arch for X8664Arch { From fad6afa7d8486e1b54f186a0bdd718113f353685 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 17 Jul 2022 13:26:56 +0200 Subject: [PATCH 079/120] Flush on Drop, unless explicitly ignored. --- src/page/flush.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/page/flush.rs b/src/page/flush.rs index 5dedf6afae..1114c33bc8 100644 --- a/src/page/flush.rs +++ b/src/page/flush.rs @@ -35,9 +35,10 @@ impl PageFlush { } } -#[must_use = "The page table must be flushed, or the changes unsafely ignored"] -pub struct PageFlushAll { - phantom: PhantomData, +// TODO: Might remove Drop and add #[must_use] again, but ergonomically I prefer being able to pass +// a flusher, and have it dropped by the end of the function it is passed to, in order to flush. +pub struct PageFlushAll { + phantom: PhantomData A>, } impl PageFlushAll { @@ -47,14 +48,17 @@ impl PageFlushAll { } } - pub fn flush(self) { - unsafe { A::invalidate_all(); } - } + pub fn flush(self) {} pub unsafe fn ignore(self) { mem::forget(self); } } +impl Drop for PageFlushAll { + fn drop(&mut self) { + unsafe { A::invalidate_all(); } + } +} impl Flusher for PageFlushAll { fn consume(&mut self, flush: PageFlush) { unsafe { flush.ignore(); } From efc67a20127c60886fa6c0643164804e91a32e70 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 17 Jul 2022 13:30:44 +0200 Subject: [PATCH 080/120] Use wrapped flags for PageEntry. --- src/page/entry.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/page/entry.rs b/src/page/entry.rs index 6732da4970..0f593a26a4 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -2,6 +2,7 @@ use core::marker::PhantomData; use crate::{ Arch, + PageFlags, PhysicalAddress, }; @@ -28,8 +29,13 @@ impl PageEntry { } #[inline(always)] - pub fn flags(&self) -> usize { - self.data & A::ENTRY_FLAGS_MASK + pub fn flags(&self) -> PageFlags { + unsafe { PageFlags::from_data(self.data & A::ENTRY_FLAGS_MASK) } + } + #[inline(always)] + pub fn set_flags(&mut self, flags: PageFlags) { + self.data &= !A::ENTRY_FLAGS_MASK; + self.data |= flags.data(); } #[inline(always)] From 229ae5da40803e57a037571aa387cf9d235d7ed6 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 17 Jul 2022 14:05:33 +0200 Subject: [PATCH 081/120] Add remaining interfaces to user RMM for user mem. --- src/page/entry.rs | 10 ++++-- src/page/flags.rs | 8 +++-- src/page/mapper.rs | 87 +++++++++++++++++++++++++++++++++++----------- src/page/table.rs | 18 +++++----- 4 files changed, 89 insertions(+), 34 deletions(-) diff --git a/src/page/entry.rs b/src/page/entry.rs index 0f593a26a4..800482cc2f 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -24,8 +24,14 @@ impl PageEntry { } #[inline(always)] - pub fn address(&self) -> PhysicalAddress { - PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK) + pub fn address(&self) -> Result { + let addr = PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK); + + if self.present() { + Ok(addr) + } else { + Err(addr) + } } #[inline(always)] diff --git a/src/page/flags.rs b/src/page/flags.rs index 6058637e65..22b8780caf 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -111,7 +111,11 @@ impl PageFlags { impl fmt::Debug for PageFlags { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PageFlags") - .field("data", &self.data) + .field("present", &self.has_present()) + .field("write", &self.has_write()) + .field("executable", &self.has_execute()) + .field("user", &self.has_user()) + .field("bits", &format_args!("{:#0x}", self.data)) .finish() } -} \ No newline at end of file +} diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 6617547ca7..d4f478bf80 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -8,44 +8,61 @@ use crate::{ PageFlush, PageTable, PhysicalAddress, + TableKind, VirtualAddress, }; -pub struct PageMapper<'f, A, F> { +pub struct PageMapper { table_addr: PhysicalAddress, - allocator: &'f mut F, - phantom: PhantomData, + allocator: F, + _phantom: PhantomData A>, } -impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { - pub unsafe fn new(table_addr: PhysicalAddress, allocator: &'f mut F) -> Self { +impl PageMapper { + pub unsafe fn new(table_addr: PhysicalAddress, allocator: F) -> Self { Self { table_addr, allocator, - phantom: PhantomData, + _phantom: PhantomData, } } - pub unsafe fn create(allocator: &'f mut F) -> Option { + pub unsafe fn create(mut allocator: F) -> Option { let table_addr = allocator.allocate_one()?; Some(Self::new(table_addr, allocator)) } - pub unsafe fn current(allocator: &'f mut F) -> Self { + pub unsafe fn current(allocator: F) -> Self { let table_addr = A::table(); Self::new(table_addr, allocator) } + pub fn is_current(&self) -> bool { + unsafe { self.table().phys() == A::table() } + } - pub unsafe fn make_current(&mut self) { + pub unsafe fn make_current(&self) { A::set_table(self.table_addr); } - pub unsafe fn table(&self) -> PageTable { - PageTable::new( - VirtualAddress::new(0), - self.table_addr, - A::PAGE_LEVELS - 1 - ) + pub fn table(&self) -> PageTable { + // SAFETY: The only way to initialize a PageMapper is via new(), and we assume it upholds + // all necessary invariants for this to be safe. + unsafe { + PageTable::new( + VirtualAddress::new(0), + self.table_addr, + A::PAGE_LEVELS - 1 + ) + } + } + + pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + self.visit(virt, |p1, i| { + let mut entry = p1.entry(i)?; + entry.set_flags(flags); + p1.set_entry(i, entry); + Some(PageFlush::new(virt)) + }).flatten() } pub unsafe fn map(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { @@ -71,7 +88,8 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - table.set_entry(i, PageEntry::new(next_phys.data() | A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE)); + let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { A::ENTRY_FLAG_USER } else { 0 }; + table.set_entry(i, PageEntry::new(next_phys.data() | flags)); table.next(i)? } }; @@ -79,14 +97,35 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { } } } + pub unsafe fn map_linearly(&mut self, phys: PhysicalAddress, flags: PageFlags) -> Option<(VirtualAddress, PageFlush)> { + let virt = A::phys_to_virt(phys); + self.map_phys(virt, phys, flags).map(|flush| (virt, flush)) + } + fn visit(&self, virt: VirtualAddress, f: impl FnOnce(&mut PageTable, usize) -> T) -> Option { + let mut table = self.table(); + unsafe { + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + return Some(f(&mut table, i)); + } else { + table = table.next(i)?; + } + } + } + } + pub fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags)> { + let entry = self.visit(virt, |p1, i| unsafe { p1.entry(i) })??; + Some((entry.address().ok()?, entry.flags())) + } pub unsafe fn unmap(&mut self, virt: VirtualAddress) -> Option> { - let (old, flush) = self.unmap_phys(virt)?; - self.allocator.free_one(old.address()); + let (old, _, flush) = self.unmap_phys(virt)?; + self.allocator.free_one(old); Some(flush) } - pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress) -> Option<(PageEntry, PageFlush)> { + pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { //TODO: verify virt is aligned let mut table = self.table(); //TODO: unmap parents @@ -96,10 +135,18 @@ impl<'f, A: Arch, F: FrameAllocator> PageMapper<'f, A, F> { let entry_opt = table.entry(i); table.set_entry(i, PageEntry::new(0)); let entry = entry_opt?; - return Some((entry, PageFlush::new(virt))); + return Some((entry.address().ok()?, entry.flags(), PageFlush::new(virt))); } else { table = table.next(i)?; } } } } +impl core::fmt::Debug for PageMapper { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PageMapper") + .field("frame", &self.table_addr) + .field("allocator", &self.allocator) + .finish() + } +} diff --git a/src/page/table.rs b/src/page/table.rs index 23a8fa1293..0abc78486d 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -93,16 +93,14 @@ impl PageTable { } pub unsafe fn next(&self, i: usize) -> Option { - if self.level > 0 { - let entry = self.entry(i)?; - if entry.present() { - return Some(PageTable::new( - self.entry_base(i)?, - entry.address(), - self.level - 1 - )); - } + if self.level == 0 { + return None; } - None + + Some(PageTable::new( + self.entry_base(i)?, + self.entry(i)?.address().ok()?, + self.level - 1, + )) } } From 9462df03e786312b6ce197cf56113d411412cbb2 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 23 Jul 2022 00:18:56 +0200 Subject: [PATCH 082/120] Add support for unmapping parents. Currently, this uses a relatively naive method of simply scanning the 512 entries for the PRESENT flag. But, unless the optimizer cannot, it can be reduced to calculating the bitwise OR of every entry and then checking that. If this turns out to be too slow, which it might be when unmapping lots of pages, then we can (1) either fall back to using a counter like the old paging code did, or even better (2) use the now-1:1 grants tree to check if it became empty. Putting the grants code in RMM might be suboptimal, so instead we can add "unmap_range" and have the kernel paging code take the offset of the next grant, if any, and then possibly unmap entire P1s/P2s/P3s -- whatever is in the page tables within that range. Note that I am fairly certain that method (1) was the cause of the visually notorious orbital memory corruption bug. --- src/page/mapper.rs | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index d4f478bf80..17d4b11f80 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -119,27 +119,45 @@ impl PageMapper { Some((entry.address().ok()?, entry.flags())) } - pub unsafe fn unmap(&mut self, virt: VirtualAddress) -> Option> { - let (old, _, flush) = self.unmap_phys(virt)?; + pub unsafe fn unmap(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option> { + let (old, _, flush) = self.unmap_phys(virt, unmap_parents)?; self.allocator.free_one(old); Some(flush) } - pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { + pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { //TODO: verify virt is aligned let mut table = self.table(); - //TODO: unmap parents - loop { - let i = table.index_of(virt)?; - if table.level() == 0 { - let entry_opt = table.entry(i); + let level = table.level(); + unmap_phys_inner(virt, &mut table, level, false, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + } +} +unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable, initial_level: usize, unmap_parents: bool, allocator: &mut impl FrameAllocator) -> Option<(PhysicalAddress, PageFlags)> { + let i = table.index_of(virt)?; + + if table.level() == 0 { + let entry_opt = table.entry(i); + table.set_entry(i, PageEntry::new(0)); + let entry = entry_opt?; + + Some((entry.address().ok()?, entry.flags())) + } else { + let mut subtable = table.next(i)?; + + let res = unmap_phys_inner(virt, &mut subtable, initial_level, unmap_parents, allocator)?; + + if unmap_parents { + // TODO: Use a counter? This would reduce the remaining number of available bits, but could be + // faster (benchmark is needed). + let is_still_populated = (0..A::PAGE_ENTRIES).map(|j| subtable.entry(j).expect("must be within bounds")).any(|e| e.present()); + + if !is_still_populated { + allocator.free_one(table.phys()); table.set_entry(i, PageEntry::new(0)); - let entry = entry_opt?; - return Some((entry.address().ok()?, entry.flags(), PageFlush::new(virt))); - } else { - table = table.next(i)?; } } + + Some(res) } } impl core::fmt::Debug for PageMapper { From 23d4995e502e42b8a73970a46716c863ee3f4dc1 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 29 Jul 2022 18:37:37 -0600 Subject: [PATCH 083/120] Fix warnings --- src/allocator/frame/buddy.rs | 6 +++--- src/arch/aarch64.rs | 6 +++--- src/arch/mod.rs | 1 - src/arch/riscv64/sv39.rs | 3 +-- src/arch/riscv64/sv48.rs | 3 +-- src/page/mapper.rs | 2 +- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 16240ec883..7aad111abf 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -120,7 +120,7 @@ impl BuddyAllocator { for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { let virt = table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); - let inserted = if area.base.add(area.size) == entry.base { + let inserted = if area.base.add(area.size) == { entry.base } { // Combine entry at start entry.base = area.base; entry.size += area.size; @@ -247,8 +247,8 @@ impl FrameAllocator for BuddyAllocator { let virt = self.table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); - if base >= entry.base && base.add(size) <= entry.base.add(entry.size) { - let start_page = (base.data() - entry.base.data()) >> A::PAGE_SHIFT; + if base >= { entry.base } && base.add(size) <= entry.base.add(entry.size) { + let start_page = (base.data() - { entry.base }.data()) >> A::PAGE_SHIFT; for page in start_page..start_page + count.data() { let mut usage = entry.usage(page).expect("failed to get usage during free"); diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 3d8711adb7..2bc24eaae0 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -45,7 +45,7 @@ impl Arch for AArch64Arch { } #[inline(always)] - unsafe fn invalidate(address: VirtualAddress) { + unsafe fn invalidate(_address: VirtualAddress) { //TODO: can one address be invalidated? Self::invalidate_all(); } @@ -87,8 +87,8 @@ impl Arch for AArch64Arch { Self::invalidate_all(); } - fn virt_is_valid(address: VirtualAddress) -> bool { - // FIXME + fn virt_is_valid(_address: VirtualAddress) -> bool { + //TODO: what makes an address valid on aarch64? true } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 62c1bb5ee7..8c9b0affab 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -3,7 +3,6 @@ use core::ptr; use crate::{ MemoryArea, PhysicalAddress, - TableKind, VirtualAddress, }; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 2e52caeb3e..ada80010a2 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -4,7 +4,6 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, - TableKind, VirtualAddress, }; @@ -40,7 +39,7 @@ impl Arch for RiscV64Sv39Arch { } #[inline(always)] - unsafe fn invalidate(address: VirtualAddress) { + unsafe fn invalidate(_address: VirtualAddress) { //TODO: can one address be invalidated? Self::invalidate_all(); } diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index a16d8391d8..8ce33331ef 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -4,7 +4,6 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, - TableKind, VirtualAddress, }; @@ -40,7 +39,7 @@ impl Arch for RiscV64Sv48Arch { } #[inline(always)] - unsafe fn invalidate(address: VirtualAddress) { + unsafe fn invalidate(_address: VirtualAddress) { //TODO: can one address be invalidated? Self::invalidate_all(); } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 17d4b11f80..ce99c6a88e 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -129,7 +129,7 @@ impl PageMapper { //TODO: verify virt is aligned let mut table = self.table(); let level = table.level(); - unmap_phys_inner(virt, &mut table, level, false, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) } } unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable, initial_level: usize, unmap_parents: bool, allocator: &mut impl FrameAllocator) -> Option<(PhysicalAddress, PageFlags)> { From e9950ee6da9e56574b1d534f36600c25505ffb7d Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 29 Jul 2022 18:53:12 -0600 Subject: [PATCH 084/120] Fix more warnings --- src/arch/x86.rs | 2 +- src/page/mapper.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 0572f4c5fc..d804ed213a 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -51,7 +51,7 @@ impl Arch for X86Arch { asm!("mov cr3, {0}", in(reg) address.data()); } - fn virt_is_valid(address: VirtualAddress) -> bool { + fn virt_is_valid(_address: VirtualAddress) -> bool { // On 32-bit x86, every virtual address is valid true } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index ce99c6a88e..1e0e98fa13 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -125,11 +125,12 @@ impl PageMapper { Some(flush) } - pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { + pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, _unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { //TODO: verify virt is aligned let mut table = self.table(); let level = table.level(); - unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + //TODO: use unmap_parents + unmap_phys_inner(virt, &mut table, level, false, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) } } unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable, initial_level: usize, unmap_parents: bool, allocator: &mut impl FrameAllocator) -> Option<(PhysicalAddress, PageFlags)> { From e05868e1dcf52b55e03a65435325cee3916ca83a Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 19 Aug 2022 19:52:00 -0600 Subject: [PATCH 085/120] Improve aarch64 paging instructions --- src/arch/aarch64.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 2bc24eaae0..2f19014316 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -45,14 +45,20 @@ impl Arch for AArch64Arch { } #[inline(always)] - unsafe fn invalidate(_address: VirtualAddress) { - //TODO: can one address be invalidated? - Self::invalidate_all(); + unsafe fn invalidate(address: VirtualAddress) { + asm!(" + dsb ishst + tlbi vaae1, {} + ", in(reg) (address.data() >> Self::PAGE_SHIFT)); } #[inline(always)] unsafe fn invalidate_all() { - asm!("tlbi vmalle1is"); + asm!(" + tlbi vmalle1 + dsb ish + isb + "); } #[inline(always)] @@ -83,7 +89,6 @@ impl Arch for AArch64Arch { asm!("msr ttbr1_el1, {0}", in(reg) address.data()); } } - //TODO: Does this need to be called? Self::invalidate_all(); } From c5eb435d79a558a4a398a80f0bda57af6192e1fe Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 19 Aug 2022 20:53:15 -0600 Subject: [PATCH 086/120] Match PHYS_OFFSET accross archs --- src/arch/aarch64.rs | 5 ++--- src/arch/riscv64/sv39.rs | 5 ++--- src/arch/riscv64/sv48.rs | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 2f19014316..608869918f 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -37,8 +37,7 @@ impl Arch for AArch64Arch { const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; const ENTRY_FLAG_EXEC: usize = 0; - //TODO: adjust to match x86_64? - const PHYS_OFFSET: usize = 0xfffffe0000000000; + const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; unsafe fn init() -> &'static [MemoryArea] { unimplemented!("AArch64Arch::init unimplemented"); @@ -119,6 +118,6 @@ mod tests { assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); assert_eq!(AArch64Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); - assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); + assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } } diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index ada80010a2..c8e5ce1b66 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -31,8 +31,7 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; - //TODO: adjust to match x86_64? - const PHYS_OFFSET: usize = 0xfffffe0000000000; + const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; unsafe fn init() -> &'static [MemoryArea] { unimplemented!("RiscV64Sv39Arch::init unimplemented"); @@ -90,7 +89,7 @@ mod tests { assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); - assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); + assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } #[test] fn is_canonical() { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 8ce33331ef..89cbeb4b69 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -31,8 +31,7 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; - //TODO: adjust to match x86_64? - const PHYS_OFFSET: usize = 0xfffffe0000000000; + const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; unsafe fn init() -> &'static [MemoryArea] { unimplemented!("RiscV64Sv48Arch::init unimplemented"); @@ -91,7 +90,7 @@ mod tests { assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); - assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_FE00_0000_0000); + assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } #[test] fn is_canonical() { From df733ed5714aaa7f7b3e15707bd7ef8a44802779 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sat, 20 Aug 2022 13:00:48 -0600 Subject: [PATCH 087/120] Use TableKind for most table operations --- .gitignore | 1 + Cargo.lock | 5 ----- src/arch/aarch64.rs | 8 ++------ src/arch/mod.rs | 8 +++++--- src/arch/riscv64/sv39.rs | 5 +++-- src/arch/riscv64/sv48.rs | 5 +++-- src/arch/x86.rs | 5 +++-- src/arch/x86_64.rs | 5 +++-- src/page/mapper.rs | 19 +++++++++++-------- src/page/table.rs | 5 +++-- 10 files changed, 34 insertions(+), 32 deletions(-) delete mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index ea8c4bf7f3..06aba01b65 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +Cargo.lock /target diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index e919422bb9..0000000000 --- a/Cargo.lock +++ /dev/null @@ -1,5 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "rmm" -version = "0.1.0" diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 608869918f..b9f3a62aaf 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -61,10 +61,8 @@ impl Arch for AArch64Arch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(table_kind: TableKind) -> PhysicalAddress { let address: usize; - //TODO: set this dynamically - let table_kind = TableKind::Kernel; match table_kind { TableKind::User => { asm!("mrs {0}, ttbr0_el1", out(reg) address); @@ -77,9 +75,7 @@ impl Arch for AArch64Arch { } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { - //TODO: set this dynamically - let table_kind = TableKind::Kernel; + unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress) { match table_kind { TableKind::User => { asm!("msr ttbr0_el1, {0}", in(reg) address.data()); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 8c9b0affab..2f16b7631a 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -3,6 +3,7 @@ use core::ptr; use crate::{ MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -86,12 +87,13 @@ pub trait Arch: Clone + Copy { #[inline(always)] unsafe fn invalidate_all() { - Self::set_table(Self::table()); + //TODO: this stub only works on x86_64, maybe make the arch implement this? + Self::set_table(TableKind::User, Self::table(TableKind::User)); } - unsafe fn table() -> PhysicalAddress; + unsafe fn table(table_kind: TableKind) -> PhysicalAddress; - unsafe fn set_table(address: PhysicalAddress); + unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress); #[inline(always)] unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index c8e5ce1b66..e3f1926b06 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -4,6 +4,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -44,7 +45,7 @@ impl Arch for RiscV64Sv39Arch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( @@ -53,7 +54,7 @@ impl Arch for RiscV64Sv39Arch { } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { let satp = (8 << 60) | // Sv39 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 89cbeb4b69..cea0a537ba 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -4,6 +4,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -44,7 +45,7 @@ impl Arch for RiscV64Sv48Arch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( @@ -53,7 +54,7 @@ impl Arch for RiscV64Sv48Arch { } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { let satp = (9 << 60) | // Sv48 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) diff --git a/src/arch/x86.rs b/src/arch/x86.rs index d804ed213a..dcf0600941 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -5,6 +5,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -40,14 +41,14 @@ impl Arch for X86Arch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { let address: usize; asm!("mov {0}, cr3", out(reg) address); PhysicalAddress::new(address) } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { asm!("mov cr3, {0}", in(reg) address.data()); } diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 1008df8e8f..0256e3b81e 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -4,6 +4,7 @@ use crate::{ Arch, MemoryArea, PhysicalAddress, + TableKind, VirtualAddress, }; @@ -39,14 +40,14 @@ impl Arch for X8664Arch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { let address: usize; asm!("mov {0}, cr3", out(reg) address); PhysicalAddress::new(address) } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { asm!("mov cr3, {0}", in(reg) address.data()); } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 1e0e98fa13..2c1c67ba38 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -13,35 +13,38 @@ use crate::{ }; pub struct PageMapper { + table_kind: TableKind, table_addr: PhysicalAddress, allocator: F, _phantom: PhantomData A>, } impl PageMapper { - pub unsafe fn new(table_addr: PhysicalAddress, allocator: F) -> Self { + pub unsafe fn new(table_kind: TableKind, table_addr: PhysicalAddress, allocator: F) -> Self { Self { + table_kind, table_addr, allocator, _phantom: PhantomData, } } - pub unsafe fn create(mut allocator: F) -> Option { + pub unsafe fn create(table_kind: TableKind, mut allocator: F) -> Option { let table_addr = allocator.allocate_one()?; - Some(Self::new(table_addr, allocator)) + Some(Self::new(table_kind, table_addr, allocator)) } - pub unsafe fn current(allocator: F) -> Self { - let table_addr = A::table(); - Self::new(table_addr, allocator) + pub unsafe fn current(table_kind: TableKind, allocator: F) -> Self { + let table_addr = A::table(table_kind); + Self::new(table_kind, table_addr, allocator) } + pub fn is_current(&self) -> bool { - unsafe { self.table().phys() == A::table() } + unsafe { self.table().phys() == A::table(self.table_kind) } } pub unsafe fn make_current(&self) { - A::set_table(self.table_addr); + A::set_table(self.table_kind, self.table_addr); } pub fn table(&self) -> PageTable { diff --git a/src/page/table.rs b/src/page/table.rs index 0abc78486d..3c003cb7ec 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -3,6 +3,7 @@ use core::marker::PhantomData; use crate::{ Arch, PhysicalAddress, + TableKind, VirtualAddress, }; use super::PageEntry; @@ -19,10 +20,10 @@ impl PageTable { Self { base, phys, level, phantom: PhantomData } } - pub unsafe fn top() -> Self { + pub unsafe fn top(table_kind: TableKind) -> Self { Self::new( VirtualAddress::new(0), - A::table(), + A::table(table_kind), A::PAGE_LEVELS - 1 ) } From 0d4ff5d4f36f26289556c6e7a13e4b40ff736b83 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 21 Aug 2022 13:20:44 -0600 Subject: [PATCH 088/120] Add functions for accessing mapper allocator --- src/page/mapper.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 2c1c67ba38..0d8618a7b4 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -59,6 +59,14 @@ impl PageMapper { } } + pub fn allocator(&self) -> &F { + &self.allocator + } + + pub fn allocator_mut(&mut self) -> &mut F { + &mut self.allocator + } + pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { self.visit(virt, |p1, i| { let mut entry = p1.entry(i)?; @@ -132,7 +140,7 @@ impl PageMapper { //TODO: verify virt is aligned let mut table = self.table(); let level = table.level(); - //TODO: use unmap_parents + //TODO: use unmap_parents? unmap_phys_inner(virt, &mut table, level, false, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) } } @@ -150,6 +158,8 @@ unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable< let res = unmap_phys_inner(virt, &mut subtable, initial_level, unmap_parents, allocator)?; + //TODO: This is a bad idea for architectures where the kernel mappings are done in the process tables, + // as these mappings may become out of sync if unmap_parents { // TODO: Use a counter? This would reduce the remaining number of available bits, but could be // faster (benchmark is needed). From 61ba2e6c8e2bba0b6460b681b7f566fa0fe231ca Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 22 Aug 2022 18:49:32 -0600 Subject: [PATCH 089/120] Fix unmap_parents --- src/page/mapper.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 0d8618a7b4..33b700f35e 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -136,12 +136,11 @@ impl PageMapper { Some(flush) } - pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, _unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { + pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { //TODO: verify virt is aligned let mut table = self.table(); let level = table.level(); - //TODO: use unmap_parents? - unmap_phys_inner(virt, &mut table, level, false, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) } } unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable, initial_level: usize, unmap_parents: bool, allocator: &mut impl FrameAllocator) -> Option<(PhysicalAddress, PageFlags)> { @@ -166,7 +165,7 @@ unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable< let is_still_populated = (0..A::PAGE_ENTRIES).map(|j| subtable.entry(j).expect("must be within bounds")).any(|e| e.present()); if !is_still_populated { - allocator.free_one(table.phys()); + allocator.free_one(subtable.phys()); table.set_entry(i, PageEntry::new(0)); } } From 27bb8e44ddfb962741e7023b35d0e9eafff041be Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 26 Aug 2022 11:07:21 -0600 Subject: [PATCH 090/120] Do not allow phys_to_virt overflow --- src/arch/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 2f16b7631a..e483899c94 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -97,7 +97,10 @@ pub trait Arch: Clone + Copy { #[inline(always)] unsafe fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress { - VirtualAddress::new(phys.data() + Self::PHYS_OFFSET) + match phys.data().checked_add(Self::PHYS_OFFSET) { + Some(some) => VirtualAddress::new(some), + None => panic!("phys_to_virt({:#x}) overflow", phys.data()), + } } fn virt_is_valid(address: VirtualAddress) -> bool; From 81b03cc69397d8f729c5fd199574ba9c29d3aa26 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 28 Aug 2022 09:19:07 -0600 Subject: [PATCH 091/120] Improve aarch64 tlb flushing --- src/arch/aarch64.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index b9f3a62aaf..db3d894b08 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -47,14 +47,17 @@ impl Arch for AArch64Arch { unsafe fn invalidate(address: VirtualAddress) { asm!(" dsb ishst - tlbi vaae1, {} + tlbi vaae1is, {} + dsb ish + isb ", in(reg) (address.data() >> Self::PAGE_SHIFT)); } #[inline(always)] unsafe fn invalidate_all() { asm!(" - tlbi vmalle1 + dsb ishst + tlbi vmalle1is dsb ish isb "); From 7aeb9f0ac812266aa3ec9a018eebb49854658ee4 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sat, 11 Feb 2023 14:50:26 -0700 Subject: [PATCH 092/120] Remove rust-toolchain --- rust-toolchain | 1 - 1 file changed, 1 deletion(-) delete mode 100644 rust-toolchain diff --git a/rust-toolchain b/rust-toolchain deleted file mode 100644 index 9eed51006f..0000000000 --- a/rust-toolchain +++ /dev/null @@ -1 +0,0 @@ -nightly-2022-03-18 From d89e6008e8febda1ef1c66af88e24aba1d47fbb5 Mon Sep 17 00:00:00 2001 From: uuuvn Date: Mon, 12 Jun 2023 10:11:01 +0000 Subject: [PATCH 093/120] Add GLOBAL flag and prettify some stuff --- src/arch/aarch64.rs | 2 ++ src/arch/mod.rs | 2 ++ src/arch/riscv64/sv39.rs | 2 ++ src/arch/riscv64/sv48.rs | 2 ++ src/arch/x86.rs | 3 ++- src/arch/x86_64.rs | 3 ++- src/lib.rs | 6 +++--- src/main.rs | 4 ---- src/page/flags.rs | 32 ++++++++++++++++++++++++-------- 9 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index db3d894b08..9974f72413 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -36,6 +36,8 @@ impl Arch for AArch64Arch { //TODO: Separate the two? const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_GLOBAL: usize = 0; + const ENTRY_FLAG_NO_GLOBAL: usize = 1 << 11; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index e483899c94..f22eaae406 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -49,6 +49,8 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_USER: usize; const ENTRY_FLAG_NO_EXEC: usize; const ENTRY_FLAG_EXEC: usize; + const ENTRY_FLAG_GLOBAL: usize; + const ENTRY_FLAG_NO_GLOBAL: usize; const PHYS_OFFSET: usize; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index e3f1926b06..ced66efa3b 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -31,6 +31,8 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_USER: usize = 1 << 4; const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; + const ENTRY_FLAG_GLOBAL: usize = 1 << 5; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index cea0a537ba..1246842bfd 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -31,6 +31,8 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_FLAG_USER: usize = 1 << 4; const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; + const ENTRY_FLAG_GLOBAL: usize = 1 << 5; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index dcf0600941..424a1332b0 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -25,7 +25,8 @@ impl Arch for X86Arch { const ENTRY_FLAG_READWRITE: usize = 1 << 1; const ENTRY_FLAG_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; - // Not used: const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; // NOT AVAILABLE UNLESS PAE IS USED! const ENTRY_FLAG_EXEC: usize = 0; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 0256e3b81e..29e8d14002 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -24,7 +24,8 @@ impl Arch for X8664Arch { const ENTRY_FLAG_READWRITE: usize = 1 << 1; const ENTRY_FLAG_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; - // Not used: const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_GLOBAL: usize = 1 << 8; + const ENTRY_FLAG_NO_GLOBAL: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; const ENTRY_FLAG_EXEC: usize = 0; diff --git a/src/lib.rs b/src/lib.rs index 51c6f938b5..0c58fd1a8d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,10 +12,10 @@ mod arch; mod page; pub const KILOBYTE: usize = 1024; -pub const MEGABYTE: usize = KILOBYTE * KILOBYTE; -pub const GIGABYTE: usize = KILOBYTE * MEGABYTE; +pub const MEGABYTE: usize = KILOBYTE * 1024; +pub const GIGABYTE: usize = MEGABYTE * 1024; #[cfg(target_pointer_width = "64")] -pub const TERABYTE: usize = KILOBYTE * GIGABYTE; +pub const TERABYTE: usize = GIGABYTE * 1024; /// Specific table to be used, needed on some architectures //TODO: Use this throughout the code diff --git a/src/main.rs b/src/main.rs index 621a1df9e5..a501837db4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,10 +20,6 @@ use rmm::{ VirtualAddress, }; -use core::{ - marker::PhantomData, -}; - pub fn format_size(size: usize) -> String { if size >= 2 * TERABYTE { format!("{} TB", size / TERABYTE) diff --git a/src/page/flags.rs b/src/page/flags.rs index 22b8780caf..3339d2cd2b 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -1,14 +1,11 @@ -use core::{ - fmt, - marker::PhantomData -}; +use core::{fmt, marker::PhantomData}; use crate::Arch; #[derive(Clone, Copy)] pub struct PageFlags { data: usize, - phantom: PhantomData, + arch: PhantomData, } impl PageFlags { @@ -19,7 +16,8 @@ impl PageFlags { // Flags set to present, kernel space, read-only, no-execute by default A::ENTRY_FLAG_DEFAULT_PAGE | A::ENTRY_FLAG_READONLY | - A::ENTRY_FLAG_NO_EXEC + A::ENTRY_FLAG_NO_EXEC | + A::ENTRY_FLAG_NO_GLOBAL, ) } } @@ -31,14 +29,18 @@ impl PageFlags { // Flags set to present, kernel space, read-only, no-execute by default A::ENTRY_FLAG_DEFAULT_TABLE | A::ENTRY_FLAG_READONLY | - A::ENTRY_FLAG_NO_EXEC + A::ENTRY_FLAG_NO_EXEC | + A::ENTRY_FLAG_NO_GLOBAL, ) } } #[inline(always)] pub unsafe fn from_data(data: usize) -> Self { - Self { data, phantom: PhantomData } + Self { + data, + arch: PhantomData, + } } #[inline(always)] @@ -106,6 +108,20 @@ impl PageFlags { // Architecture may use no exec or exec, support either self.data & (A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_EXEC) == A::ENTRY_FLAG_EXEC } + + #[must_use] + #[inline(always)] + pub fn global(self, value: bool) -> Self { + // Architecture may use global or non global, support either + self.custom_flag(A::ENTRY_FLAG_NO_GLOBAL, !value) + .custom_flag(A::ENTRY_FLAG_GLOBAL, value) + } + + #[inline(always)] + pub fn is_global(&self) -> bool { + // Architecture may use global or non global, support either + self.data & (A::ENTRY_FLAG_NO_GLOBAL | A::ENTRY_FLAG_NO_GLOBAL) == A::ENTRY_FLAG_GLOBAL + } } impl fmt::Debug for PageFlags { From 339984e49b1dcdbe6551f03a4084b2e8ab499b6c Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 16 Jul 2023 18:32:34 +0200 Subject: [PATCH 094/120] Fix is_global. --- src/page/flags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page/flags.rs b/src/page/flags.rs index 3339d2cd2b..b65d31344e 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -120,7 +120,7 @@ impl PageFlags { #[inline(always)] pub fn is_global(&self) -> bool { // Architecture may use global or non global, support either - self.data & (A::ENTRY_FLAG_NO_GLOBAL | A::ENTRY_FLAG_NO_GLOBAL) == A::ENTRY_FLAG_GLOBAL + self.data & (A::ENTRY_FLAG_GLOBAL | A::ENTRY_FLAG_NO_GLOBAL) == A::ENTRY_FLAG_GLOBAL } } From 0aa7fea250f9f8ea7d0f30df8954ea89ee04db49 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Wed, 21 Jun 2023 12:58:39 +0200 Subject: [PATCH 095/120] Fix remap bug: require present before remapping. --- src/page/mapper.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 33b700f35e..b32c2e2b66 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -70,6 +70,7 @@ impl PageMapper { pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { self.visit(virt, |p1, i| { let mut entry = p1.entry(i)?; + entry.address().ok()?; entry.set_flags(flags); p1.set_entry(i, entry); Some(PageFlush::new(virt)) From 8e22e69c940e61e3268c2e6adeffca9bc65d1cd5 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Mon, 26 Jun 2023 12:22:33 +0200 Subject: [PATCH 096/120] Add remap_with{,_full}. --- src/page/mapper.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index b32c2e2b66..f75c01a9af 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -67,15 +67,24 @@ impl PageMapper { &mut self.allocator } - pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + pub unsafe fn remap_with_full(&mut self, virt: VirtualAddress, f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags)) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { self.visit(virt, |p1, i| { - let mut entry = p1.entry(i)?; - entry.address().ok()?; - entry.set_flags(flags); - p1.set_entry(i, entry); - Some(PageFlush::new(virt)) + let old_entry = p1.entry(i)?; + let old_phys = old_entry.address().ok()?; + let old_flags = old_entry.flags(); + let (new_phys, new_flags) = f(old_phys, old_flags); + // TODO: Higher-level PageEntry::new interface? + let new_entry = PageEntry::new(new_phys.data() | new_flags.data()); + p1.set_entry(i, new_entry); + Some((old_flags, old_phys, PageFlush::new(virt))) }).flatten() } + pub unsafe fn remap_with(&mut self, virt: VirtualAddress, map_flags: impl FnOnce(PageFlags) -> PageFlags) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + self.remap_with_full(virt, |same_phys, old_flags| (same_phys, map_flags(old_flags))) + } + pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) + } pub unsafe fn map(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { let phys = self.allocator.allocate_one()?; From b5937b7b914102fc9554b9b5ef0071246082b457 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Fri, 28 Jul 2023 13:38:24 +0200 Subject: [PATCH 097/120] Set aarch64 "not global" flag by default. --- src/arch/aarch64.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 9974f72413..d4ebf2185a 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -22,6 +22,7 @@ impl Arch for AArch64Arch { = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Page flag | 1 << 10 // Access flag + | Self::ENTRY_FLAG_NO_GLOBAL ; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT From 86aba2dbe5aed1f9f0b137c082c16d57b9797eff Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Fri, 28 Jul 2023 13:39:32 +0200 Subject: [PATCH 098/120] Better VirtualAddress Debug impl. --- src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0c58fd1a8d..57086d38df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,7 @@ impl PhysicalAddress { } /// Virtual memory address -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct VirtualAddress(usize); @@ -79,6 +79,11 @@ impl VirtualAddress { } } } +impl core::fmt::Debug for VirtualAddress { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[virt {:#0x}]", self.data()) + } +} #[derive(Clone, Copy, Debug)] pub struct MemoryArea { From aad492f8056589278986efd11532f646a0fc78a1 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:21:22 +0100 Subject: [PATCH 099/120] Derive Hash for VirtualAddress --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 57086d38df..259c4eefbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,7 @@ impl PhysicalAddress { } /// Virtual memory address -#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct VirtualAddress(usize); From ed455915cae1753c1347c641735cc97f37969f7b Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Mon, 16 Oct 2023 20:43:36 +0200 Subject: [PATCH 100/120] Allow arbitrary bump allocation size. --- src/allocator/frame/bump.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index b044984c8b..d3a2b4eac1 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -35,18 +35,17 @@ impl BumpAllocator { impl FrameAllocator for BumpAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option { - //TODO: support allocation of multiple pages - if count.data() != 1 { - return None; - } - let mut offset = self.offset; for area in self.areas.iter() { if offset < area.size { + if area.size - offset < count.data() { + return None; + } + let page_phys = area.base.add(offset); let page_virt = A::phys_to_virt(page_phys); - A::write_bytes(page_virt, 0, A::PAGE_SIZE); - self.offset += A::PAGE_SIZE; + A::write_bytes(page_virt, 0, count.data() * A::PAGE_SIZE); + self.offset += count.data() * A::PAGE_SIZE; return Some(page_phys); } offset -= area.size; From 5e75df56c56f1ed1c1d5427ffec3f33973f4b444 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 26 Oct 2023 13:28:08 +0200 Subject: [PATCH 101/120] More BumpAllocator APIs. --- src/allocator/frame/bump.rs | 42 ++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index d3a2b4eac1..85334b4af6 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -12,21 +12,44 @@ use crate::{ pub struct BumpAllocator { areas: &'static [MemoryArea], offset: usize, - phantom: PhantomData, + abs_offset: PhysicalAddress, + _marker: PhantomData A>, } impl BumpAllocator { pub fn new(areas: &'static [MemoryArea], offset: usize) -> Self { Self { areas, - offset, - phantom: PhantomData, + offset: 0, + abs_offset: areas[0].base, + _marker: PhantomData, } } - pub fn areas(&self) -> &'static [MemoryArea] { self.areas } + /// Returns one semifree and the fully free areas. The offset is the number of bytes after + /// which the first area is free. + pub fn free_areas(&self) -> (&'static [MemoryArea], usize) { + let mut areas = self.areas; + let mut offset = self.offset; + + loop { + let Some(area) = areas.first() else { + return (&[], 0); + }; + + if offset > area.size { + areas = &areas[1..]; + offset -= area.size; + } else { + return (areas, offset); + } + } + } + pub fn abs_offset(&self) -> PhysicalAddress { + self.abs_offset + } pub fn offset(&self) -> usize { self.offset @@ -38,7 +61,14 @@ impl FrameAllocator for BumpAllocator { let mut offset = self.offset; for area in self.areas.iter() { if offset < area.size { - if area.size - offset < count.data() { + if area.size - offset < count.data() * A::PAGE_SIZE { + /* + // The area may be too small for this alloc request. In that case, skip to the + // next area. + self.offset += area.size - offset; + offset = 0; + continue; + */ return None; } @@ -46,6 +76,8 @@ impl FrameAllocator for BumpAllocator { let page_virt = A::phys_to_virt(page_phys); A::write_bytes(page_virt, 0, count.data() * A::PAGE_SIZE); self.offset += count.data() * A::PAGE_SIZE; + + self.abs_offset = page_phys; return Some(page_phys); } offset -= area.size; From 99fcfc5faeeb0e9a79d5968fb74ba5e4cb2234f5 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 11 Apr 2024 22:02:44 +0200 Subject: [PATCH 102/120] Skip insufficiently large bump alloc regions. --- src/allocator/frame/bump.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 85334b4af6..a911647159 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -62,14 +62,11 @@ impl FrameAllocator for BumpAllocator { for area in self.areas.iter() { if offset < area.size { if area.size - offset < count.data() * A::PAGE_SIZE { - /* // The area may be too small for this alloc request. In that case, skip to the // next area. self.offset += area.size - offset; offset = 0; continue; - */ - return None; } let page_phys = area.base.add(offset); From 7c0995ee24a5b991d108e25c0439cda7ac2fdd43 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Thu, 11 Jul 2024 23:41:22 +0300 Subject: [PATCH 103/120] Revived tests Also fixed debug build failure on i686, and reformatted the code --- src/allocator/frame/buddy.rs | 37 ++++------ src/allocator/frame/bump.rs | 11 +-- src/allocator/frame/mod.rs | 10 ++- src/arch/aarch64.rs | 26 +++---- src/arch/emulate.rs | 137 +++++++++++++++++++++-------------- src/arch/mod.rs | 31 +++----- src/arch/riscv64/sv39.rs | 19 ++--- src/arch/riscv64/sv48.rs | 26 +++---- src/arch/x86.rs | 10 +-- src/arch/x86_64.rs | 13 +--- src/lib.rs | 6 +- src/main.rs | 90 ++++++++++++----------- src/page/entry.rs | 11 ++- src/page/flags.rs | 16 ++-- src/page/flush.rs | 24 +++--- src/page/mapper.rs | 102 ++++++++++++++++++-------- src/page/mod.rs | 8 +- src/page/table.rs | 22 +++--- 18 files changed, 306 insertions(+), 293 deletions(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 7aad111abf..564004303a 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -1,16 +1,7 @@ -use core::{ - marker::PhantomData, - mem, -}; +use core::{marker::PhantomData, mem}; use crate::{ - Arch, - BumpAllocator, - FrameAllocator, - FrameCount, - FrameUsage, - PhysicalAddress, - VirtualAddress, + Arch, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, PhysicalAddress, VirtualAddress, }; #[repr(transparent)] @@ -94,7 +85,7 @@ impl BuddyAllocator { // Allocate buddy table let table_phys = bump_allocator.allocate_one()?; let table_virt = A::phys_to_virt(table_phys); - for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { let virt = table_virt.add(i * mem::size_of::>()); A::write(virt, BuddyEntry::::empty()); } @@ -117,7 +108,7 @@ impl BuddyAllocator { area.size -= offset; offset = 0; } - for i in 0 .. (A::PAGE_SIZE / mem::size_of::>()) { + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { let virt = table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); let inserted = if area.base.add(area.size) == { entry.base } { @@ -147,7 +138,7 @@ impl BuddyAllocator { //TODO: sort areas? // Allocate buddy maps - for i in 0 .. Self::BUDDY_ENTRIES { + for i in 0..Self::BUDDY_ENTRIES { let virt = table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); @@ -186,13 +177,15 @@ impl FrameAllocator for BuddyAllocator { return None; } - for entry_i in 0 .. Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(entry_i * mem::size_of::>()); + for entry_i in 0..Self::BUDDY_ENTRIES { + let virt = self + .table_virt + .add(entry_i * mem::size_of::>()); let mut entry = A::read::>(virt); let mut free_page = entry.skip; let mut free_count = 0; - for page in entry.skip .. entry.pages() { + for page in entry.skip..entry.pages() { let usage = entry.usage(page)?; if usage.0 == 0 { free_count += 1; @@ -207,7 +200,7 @@ impl FrameAllocator for BuddyAllocator { } if free_count == count.data() { - for page in free_page .. free_page + free_count { + for page in free_page..free_page + free_count { // Update usage let mut usage = entry.usage(page)?; usage.0 += 1; @@ -243,7 +236,7 @@ impl FrameAllocator for BuddyAllocator { } let size = count.data() * A::PAGE_SIZE; - for i in 0 .. Self::BUDDY_ENTRIES { + for i in 0..Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); @@ -269,7 +262,9 @@ impl FrameAllocator for BuddyAllocator { entry.used -= 1; } - entry.set_usage(page, usage).expect("failed to set usage during free"); + entry + .set_usage(page, usage) + .expect("failed to set usage during free"); } // Write updated entry @@ -283,7 +278,7 @@ impl FrameAllocator for BuddyAllocator { unsafe fn usage(&self) -> FrameUsage { let mut total = 0; let mut used = 0; - for i in 0 .. Self::BUDDY_ENTRIES { + for i in 0..Self::BUDDY_ENTRIES { let virt = self.table_virt.add(i * mem::size_of::>()); let entry = A::read::>(virt); total += entry.size >> A::PAGE_SHIFT; diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index a911647159..51bee174ca 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -1,13 +1,6 @@ use core::marker::PhantomData; -use crate::{ - Arch, - FrameAllocator, - FrameCount, - FrameUsage, - MemoryArea, - PhysicalAddress, -}; +use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress}; pub struct BumpAllocator { areas: &'static [MemoryArea], @@ -17,7 +10,7 @@ pub struct BumpAllocator { } impl BumpAllocator { - pub fn new(areas: &'static [MemoryArea], offset: usize) -> Self { + pub fn new(areas: &'static [MemoryArea], _offset: usize) -> Self { Self { areas, offset: 0, diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index 99c9d3215c..626ff41714 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -1,7 +1,6 @@ use crate::PhysicalAddress; -pub use self::buddy::*; -pub use self::bump::*; +pub use self::{buddy::*, bump::*}; mod buddy; mod bump; @@ -30,7 +29,7 @@ impl FrameUsage { pub fn new(used: FrameCount, total: FrameCount) -> Self { Self { used, total } } - + pub fn used(&self) -> FrameCount { self.used } @@ -60,7 +59,10 @@ pub trait FrameAllocator { unsafe fn usage(&self) -> FrameUsage; } -impl FrameAllocator for &mut T where T: FrameAllocator { +impl FrameAllocator for &mut T +where + T: FrameAllocator, +{ unsafe fn allocate(&mut self, count: FrameCount) -> Option { T::allocate(self, count) } diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index d4ebf2185a..3a811a0184 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct AArch64Arch; @@ -18,12 +12,10 @@ impl Arch for AArch64Arch { //TODO const ENTRY_ADDRESS_SHIFT: usize = 52; - const ENTRY_FLAG_DEFAULT_PAGE: usize - = Self::ENTRY_FLAG_PRESENT + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Page flag | 1 << 10 // Access flag - | Self::ENTRY_FLAG_NO_GLOBAL - ; + | Self::ENTRY_FLAG_NO_GLOBAL; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Table flag @@ -58,12 +50,14 @@ impl Arch for AArch64Arch { #[inline(always)] unsafe fn invalidate_all() { - asm!(" + asm!( + " dsb ishst tlbi vmalle1is dsb ish isb - "); + " + ); } #[inline(always)] @@ -72,7 +66,7 @@ impl Arch for AArch64Arch { match table_kind { TableKind::User => { asm!("mrs {0}, ttbr0_el1", out(reg) address); - }, + } TableKind::Kernel => { asm!("mrs {0}, ttbr1_el1", out(reg) address); } @@ -85,7 +79,7 @@ impl Arch for AArch64Arch { match table_kind { TableKind::User => { asm!("msr ttbr0_el1, {0}", in(reg) address.data()); - }, + } TableKind::Kernel => { asm!("msr ttbr1_el1, {0}", in(reg) address.data()); } @@ -101,8 +95,8 @@ impl Arch for AArch64Arch { #[cfg(test)] mod tests { - use crate::Arch; use super::AArch64Arch; + use crate::Arch; #[test] fn constants() { diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index a73bb084c8..88a200769a 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -1,18 +1,9 @@ -use core::{ - marker::PhantomData, - mem, - ptr, -}; +use core::{marker::PhantomData, mem, ptr}; use std::collections::BTreeMap; use crate::{ - MEGABYTE, - Arch, - MemoryArea, - PageEntry, - PhysicalAddress, - VirtualAddress, - arch::x86_64::X8664Arch, + arch::x86_64::X8664Arch, page::PageFlags, Arch, MemoryArea, PageEntry, PhysicalAddress, + TableKind, VirtualAddress, MEGABYTE, }; #[derive(Clone, Copy)] @@ -35,6 +26,9 @@ impl Arch for EmulateArch { const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET; + const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; + const ENTRY_FLAG_NO_GLOBAL: usize = X8664Arch::ENTRY_FLAG_NO_GLOBAL; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); @@ -43,7 +37,10 @@ impl Arch for EmulateArch { let pml4 = 0; let pdp = pml4 + Self::PAGE_SIZE; let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT; - machine.write_phys::(PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), pdp | flags); + machine.write_phys::( + PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), + pdp | flags, + ); // PDP link to PD let pd = pdp + Self::PAGE_SIZE; @@ -56,13 +53,16 @@ impl Arch for EmulateArch { // PT links to frames for i in 0..Self::PAGE_ENTRIES { let page = i * Self::PAGE_SIZE; - machine.write_phys::(PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), page | flags); + machine.write_phys::( + PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), + page | flags, + ); } MACHINE = Some(machine); // Set table to pml4 - EmulateArch::set_table(PhysicalAddress::new(pml4)); + EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4)); &MEMORY_AREAS } @@ -93,12 +93,12 @@ impl Arch for EmulateArch { } #[inline(always)] - unsafe fn table() -> PhysicalAddress { + unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { MACHINE.as_mut().unwrap().get_table() } #[inline(always)] - unsafe fn set_table(address: PhysicalAddress) { + unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { MACHINE.as_mut().unwrap().set_table(address); } fn virt_is_valid(_address: VirtualAddress) -> bool { @@ -111,12 +111,12 @@ const MEMORY_SIZE: usize = 64 * MEGABYTE; static MEMORY_AREAS: [MemoryArea; 2] = [ MemoryArea { base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted - size: MEMORY_SIZE/2 - EmulateArch::PAGE_SIZE * 4, + size: MEMORY_SIZE / 2 - EmulateArch::PAGE_SIZE * 4, }, // Second area for debugging MemoryArea { - base: PhysicalAddress::new(MEMORY_SIZE/2), - size: MEMORY_SIZE/2, + base: PhysicalAddress::new(MEMORY_SIZE / 2), + size: MEMORY_SIZE / 2, }, ]; @@ -142,11 +142,13 @@ impl Machine { fn read_phys(&self, phys: PhysicalAddress) -> T { let size = mem::size_of::(); if phys.add(size).data() <= self.memory.len() { - unsafe { - ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) - } + unsafe { ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) } } else { - panic!("read_phys: 0x{:X} size 0x{:X} outside of memory", phys.data(), size); + panic!( + "read_phys: 0x{:X} size 0x{:X} outside of memory", + phys.data(), + size + ); } } @@ -157,29 +159,34 @@ impl Machine { ptr::write(self.memory.as_mut_ptr().add(phys.data()) as *mut T, value); } } else { - panic!("write_phys: 0x{:X} size 0x{:X} outside of memory", phys.data(), size); + panic!( + "write_phys: 0x{:X} size 0x{:X} outside of memory", + phys.data(), + size + ); } } fn write_phys_bytes(&mut self, phys: PhysicalAddress, value: u8, count: usize) { if phys.add(count).data() <= self.memory.len() { unsafe { - ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()) as *mut u8, value, count); + ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()), value, count); } } else { - panic!("write_phys_bytes: 0x{:X} count 0x{:X} outside of memory", phys.data(), count); + panic!( + "write_phys_bytes: 0x{:X} count 0x{:X} outside of memory", + phys.data(), + count + ); } } - fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, usize)> { + fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags)> { let virt_data = virt.data(); let page = virt_data & A::PAGE_ADDRESS_MASK; let offset = virt_data & A::PAGE_OFFSET_MASK; let entry = self.map.get(&VirtualAddress::new(page))?; - Some(( - entry.address().add(offset), - entry.flags(), - )) + Some((entry.address().ok()?.add(offset), entry.flags())) } fn read(&self, virt: VirtualAddress) -> T { @@ -187,7 +194,10 @@ impl Machine { let virt_data = virt.data(); let size = mem::size_of::(); if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("read: 0x{:X} size 0x{:X} passes page boundary", virt_data, size); + panic!( + "read: 0x{:X} size 0x{:X} passes page boundary", + virt_data, size + ); } if let Some((phys, _flags)) = self.translate(virt) { @@ -202,11 +212,14 @@ impl Machine { let virt_data = virt.data(); let size = mem::size_of::(); if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("write: 0x{:X} size 0x{:X} passes page boundary", virt_data, size); + panic!( + "write: 0x{:X} size 0x{:X} passes page boundary", + virt_data, size + ); } if let Some((phys, flags)) = self.translate(virt) { - if flags & A::ENTRY_FLAG_READWRITE != 0 { + if flags.has_write() { self.write_phys(phys, value); } else { panic!("write: 0x{:X} size 0x{:X} not writable", virt_data, size); @@ -219,18 +232,28 @@ impl Machine { fn write_bytes(&mut self, virt: VirtualAddress, value: u8, count: usize) { //TODO: allow writing past page boundaries let virt_data = virt.data(); - if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK) { - panic!("write_bytes: 0x{:X} count 0x{:X} passes page boundary", virt_data, count); + if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK) + { + panic!( + "write_bytes: 0x{:X} count 0x{:X} passes page boundary", + virt_data, count + ); } if let Some((phys, flags)) = self.translate(virt) { - if flags & A::ENTRY_FLAG_READWRITE != 0 { + if flags.has_write() { self.write_phys_bytes(phys, value, count); } else { - panic!("write_bytes: 0x{:X} count 0x{:X} not writable", virt_data, count); + panic!( + "write_bytes: 0x{:X} count 0x{:X} not writable", + virt_data, count + ); } } else { - panic!("write_bytes: 0x{:X} count 0x{:X} not present", virt_data, count); + panic!( + "write_bytes: 0x{:X} count 0x{:X} not present", + virt_data, count + ); } } @@ -247,37 +270,45 @@ impl Machine { for i4 in 0..A::PAGE_ENTRIES { let e3 = self.read_phys::(PhysicalAddress::new(a4 + i4 * A::PAGE_ENTRY_SIZE)); let f3 = e3 & A::ENTRY_FLAGS_MASK; - if f3 & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f3 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page directory pointer let a3 = e3 & A::ENTRY_ADDRESS_MASK; for i3 in 0..A::PAGE_ENTRIES { - let e2 = self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); + let e2 = + self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); let f2 = e2 & A::ENTRY_FLAGS_MASK; - if f2 & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f2 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page directory let a2 = e2 & A::ENTRY_ADDRESS_MASK; for i2 in 0..A::PAGE_ENTRIES { - let e1 = self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); + let e1 = + self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); let f1 = e1 & A::ENTRY_FLAGS_MASK; - if f1 & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f1 & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page table let a1 = e1 & A::ENTRY_ADDRESS_MASK; for i1 in 0..A::PAGE_ENTRIES { - let e = self.read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); + let e = self + .read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); let f = e & A::ENTRY_FLAGS_MASK; - if f & A::ENTRY_FLAG_PRESENT == 0 { continue; } + if f & A::ENTRY_FLAG_PRESENT == 0 { + continue; + } // Page - let page = - (i4 << 39) | - (i3 << 30) | - (i2 << 21) | - (i1 << 12); + let page = (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12); //println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); - self.map.insert(VirtualAddress::new(page), PageEntry::new(e)); + self.map + .insert(VirtualAddress::new(page), PageEntry::new(e)); } } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f22eaae406..f6f5ec9c53 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,35 +1,25 @@ use core::ptr; -use crate::{ - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; //TODO: Support having all page tables compile on all architectures +#[cfg(all(feature = "std", target_pointer_width = "64"))] +pub use self::emulate::EmulateArch; #[cfg(target_pointer_width = "32")] -pub use self::{ - x86::X86Arch, -}; +pub use self::x86::X86Arch; #[cfg(target_pointer_width = "64")] pub use self::{ aarch64::AArch64Arch, - riscv64::{ - RiscV64Sv39Arch, - RiscV64Sv48Arch - }, + riscv64::{RiscV64Sv39Arch, RiscV64Sv48Arch}, x86_64::X8664Arch, }; -#[cfg(all(feature = "std", target_pointer_width = "64"))] -pub use self::emulate::EmulateArch; #[cfg(target_pointer_width = "64")] mod aarch64; -#[cfg(target_pointer_width = "64")] -mod riscv64; #[cfg(all(feature = "std", target_pointer_width = "64"))] mod emulate; +#[cfg(target_pointer_width = "64")] +mod riscv64; #[cfg(target_pointer_width = "32")] mod x86; #[cfg(target_pointer_width = "64")] @@ -62,11 +52,12 @@ pub trait Arch: Clone + Copy { const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT); const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT; const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; - const PAGE_NEGATIVE_MASK: usize = !((Self::PAGE_ADDRESS_SIZE as u64) - 1) as usize; + const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1) as usize; const ENTRY_ADDRESS_SIZE: u64 = 1 << Self::ENTRY_ADDRESS_SHIFT; - const ENTRY_ADDRESS_MASK: usize = (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; - const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK as usize; + const ENTRY_ADDRESS_MASK: usize = + (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; + const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; unsafe fn init() -> &'static [MemoryArea]; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index ced66efa3b..0980428754 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct RiscV64Sv39Arch; @@ -22,9 +16,7 @@ impl Arch for RiscV64Sv39Arch { = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag ; - const ENTRY_FLAG_DEFAULT_TABLE: usize - = Self::ENTRY_FLAG_PRESENT - ; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; @@ -51,14 +43,13 @@ impl Arch for RiscV64Sv39Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT // Convert from PPN + (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN ) } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - let satp = - (8 << 60) | // Sv39 MODE + let satp = (8 << 60) | // Sv39 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); } @@ -73,8 +64,8 @@ impl Arch for RiscV64Sv39Arch { #[cfg(test)] mod tests { - use crate::Arch; use super::RiscV64Sv39Arch; + use crate::Arch; #[test] fn constants() { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 1246842bfd..b1142317f0 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct RiscV64Sv48Arch; @@ -22,9 +16,7 @@ impl Arch for RiscV64Sv48Arch { = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag ; - const ENTRY_FLAG_DEFAULT_TABLE: usize - = Self::ENTRY_FLAG_PRESENT - ; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; @@ -51,14 +43,13 @@ impl Arch for RiscV64Sv48Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT // Convert from PPN + (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN ) } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - let satp = - (9 << 60) | // Sv48 MODE + let satp = (9 << 60) | // Sv48 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); } @@ -67,15 +58,14 @@ impl Arch for RiscV64Sv48Arch { let mask = 0xFFFF_8000_0000_0000; let masked = address.data() & mask; - masked == mask - || masked == 0 + masked == mask || masked == 0 } } #[cfg(test)] mod tests { - use crate::Arch; use super::RiscV64Sv48Arch; + use crate::Arch; #[test] fn constants() { @@ -104,7 +94,9 @@ mod tests { assert!(RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); } fn no(address: usize) { - assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address))); + assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new( + address + ))); } yes(0xFFFF_8000_1337_1337); diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 424a1332b0..8dcd238f3e 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -1,13 +1,7 @@ //TODO: USE PAE use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct X86Arch; @@ -61,8 +55,8 @@ impl Arch for X86Arch { #[cfg(test)] mod tests { - use crate::Arch; use super::{VirtualAddress, X86Arch}; + use crate::Arch; #[test] fn constants() { diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 29e8d14002..113f66f036 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -1,12 +1,6 @@ use core::arch::asm; -use crate::{ - Arch, - MemoryArea, - PhysicalAddress, - TableKind, - VirtualAddress, -}; +use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy, Debug)] pub struct X8664Arch; @@ -65,15 +59,14 @@ impl VirtualAddress { pub fn is_canonical(self) -> bool { let masked = self.data() & 0xFFFF_8000_0000_0000; // TODO: 5-level paging - masked == 0xFFFF_8000_0000_0000 - || masked == 0 + masked == 0xFFFF_8000_0000_0000 || masked == 0 } } #[cfg(test)] mod tests { - use crate::Arch; use super::{VirtualAddress, X8664Arch}; + use crate::Arch; #[test] fn constants() { diff --git a/src/lib.rs b/src/lib.rs index 259c4eefbe..b0615774d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,7 @@ #![cfg_attr(not(feature = "std"), no_std)] #![feature(doc_cfg)] -pub use crate::{ - allocator::*, - arch::*, - page::*, -}; +pub use crate::{allocator::*, arch::*, page::*}; mod allocator; mod arch; diff --git a/src/main.rs b/src/main.rs index a501837db4..86f5c67b60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,24 +1,11 @@ #![cfg(target_pointer_width = "64")] use rmm::{ - KILOBYTE, - MEGABYTE, - GIGABYTE, - TERABYTE, - Arch, - BuddyAllocator, - BumpAllocator, - EmulateArch, - FrameAllocator, - FrameCount, - MemoryArea, - PageFlags, - PageFlushAll, - PageMapper, - PageTable, - PhysicalAddress, - VirtualAddress, + Arch, BuddyAllocator, BumpAllocator, EmulateArch, Flusher, FrameAllocator, FrameCount, + MemoryArea, PageFlags, PageFlushAll, PageMapper, PageTable, PhysicalAddress, TableKind, + VirtualAddress, GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE, }; +use std::marker::PhantomData; pub fn format_size(size: usize) -> String { if size >= 2 * TERABYTE { @@ -34,6 +21,7 @@ pub fn format_size(size: usize) -> String { } } +#[allow(dead_code)] unsafe fn dump_tables(table: PageTable) { let level = table.level(); for i in 0..A::PAGE_ENTRIES { @@ -41,7 +29,11 @@ unsafe fn dump_tables(table: PageTable) { if let Some(entry) = table.entry(i) { if entry.present() { let base = table.entry_base(i).unwrap(); - println!("0x{:X}: 0x{:X}", base.data(), entry.address().data()); + println!( + "0x{:X}: 0x{:X}", + base.data(), + entry.address().unwrap().data() + ); } } } else { @@ -179,18 +171,15 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { { // Map all physical areas at PHYS_OFFSET - let mut mapper = PageMapper::::create( - &mut bump_allocator - ).expect("failed to create Mapper"); + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); for area in areas.iter() { for i in 0..area.size / A::PAGE_SIZE { let phys = area.base.add(i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); - let flush = mapper.map_phys( - virt, - phys, - PageFlags::::new().write(true) - ).expect("failed to map page to frame"); + let flush = mapper + .map_phys(virt, phys, PageFlags::::new().write(true)) + .expect("failed to map page to frame"); flush.ignore(); // Not the active table } } @@ -229,35 +218,39 @@ unsafe fn new_tables(areas: &'static [MemoryArea]) { } } - let mut mapper = PageMapper::::current( - &mut allocator - ); - let flush_all = PageFlushAll::new(); + let mut mapper = PageMapper::::current(TableKind::Kernel, &mut allocator); + let mut flush_all = PageFlushAll::new(); for i in 0..16 { let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); - let flush = mapper.map( - virt, - PageFlags::::new().user(true).write(true) - ).expect("failed to map page"); + let flush = mapper + .map(virt, PageFlags::::new().user(true).write(true)) + .expect("failed to map page"); flush_all.consume(flush); } flush_all.flush(); - let flush_all = PageFlushAll::new(); + let mut flush_all = PageFlushAll::new(); for i in 0..16 { let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); - let flush = mapper.unmap( - virt, - ).expect("failed to unmap page"); + let flush = mapper.unmap(virt, false).expect("failed to unmap page"); flush_all.consume(flush); } flush_all.flush(); let usage = allocator.usage(); println!("Allocator usage:"); - println!(" Used: {}", format_size(usage.used().data() * A::PAGE_SIZE)); - println!(" Free: {}", format_size(usage.free().data() * A::PAGE_SIZE)); - println!(" Total: {}", format_size(usage.total().data() * A::PAGE_SIZE)); + println!( + " Used: {}", + format_size(usage.used().data() * A::PAGE_SIZE) + ); + println!( + " Free: {}", + format_size(usage.free().data() * A::PAGE_SIZE) + ); + println!( + " Total: {}", + format_size(usage.total().data() * A::PAGE_SIZE) + ); } unsafe fn inner() { @@ -270,19 +263,28 @@ unsafe fn inner() { //dump_tables(PageTable::::top()); - for i in &[1, 2, 4, 8, 16, 32] { let phys = PhysicalAddress::new(i * MEGABYTE); let virt = A::phys_to_virt(phys); // Test read - println!("0x{:X} (0x{:X}) = 0x{:X}", virt.data(), phys.data(), A::read::(virt)); + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); // Test write A::write::(virt, 0x5A); // Test read - println!("0x{:X} (0x{:X}) = 0x{:X}", virt.data(), phys.data(), A::read::(virt)); + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); } } diff --git a/src/page/entry.rs b/src/page/entry.rs index 800482cc2f..34a83d448f 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -1,10 +1,6 @@ use core::marker::PhantomData; -use crate::{ - Arch, - PageFlags, - PhysicalAddress, -}; +use crate::{Arch, PageFlags, PhysicalAddress}; #[derive(Clone, Copy, Debug)] pub struct PageEntry { @@ -15,7 +11,10 @@ pub struct PageEntry { impl PageEntry { #[inline(always)] pub fn new(data: usize) -> Self { - Self { data, phantom: PhantomData } + Self { + data, + phantom: PhantomData, + } } #[inline(always)] diff --git a/src/page/flags.rs b/src/page/flags.rs index b65d31344e..bffe12643e 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -14,10 +14,10 @@ impl PageFlags { unsafe { Self::from_data( // Flags set to present, kernel space, read-only, no-execute by default - A::ENTRY_FLAG_DEFAULT_PAGE | - A::ENTRY_FLAG_READONLY | - A::ENTRY_FLAG_NO_EXEC | - A::ENTRY_FLAG_NO_GLOBAL, + A::ENTRY_FLAG_DEFAULT_PAGE + | A::ENTRY_FLAG_READONLY + | A::ENTRY_FLAG_NO_EXEC + | A::ENTRY_FLAG_NO_GLOBAL, ) } } @@ -27,10 +27,10 @@ impl PageFlags { unsafe { Self::from_data( // Flags set to present, kernel space, read-only, no-execute by default - A::ENTRY_FLAG_DEFAULT_TABLE | - A::ENTRY_FLAG_READONLY | - A::ENTRY_FLAG_NO_EXEC | - A::ENTRY_FLAG_NO_GLOBAL, + A::ENTRY_FLAG_DEFAULT_TABLE + | A::ENTRY_FLAG_READONLY + | A::ENTRY_FLAG_NO_EXEC + | A::ENTRY_FLAG_NO_GLOBAL, ) } } diff --git a/src/page/flush.rs b/src/page/flush.rs index 1114c33bc8..371301b901 100644 --- a/src/page/flush.rs +++ b/src/page/flush.rs @@ -1,12 +1,6 @@ -use core::{ - marker::PhantomData, - mem, -}; +use core::{marker::PhantomData, mem}; -use crate::{ - Arch, - VirtualAddress, -}; +use crate::{Arch, VirtualAddress}; pub trait Flusher { fn consume(&mut self, flush: PageFlush); @@ -27,7 +21,9 @@ impl PageFlush { } pub fn flush(self) { - unsafe { A::invalidate(self.virt); } + unsafe { + A::invalidate(self.virt); + } } pub unsafe fn ignore(self) { @@ -41,7 +37,7 @@ pub struct PageFlushAll { phantom: PhantomData A>, } -impl PageFlushAll { +impl PageFlushAll { pub fn new() -> Self { Self { phantom: PhantomData, @@ -56,12 +52,16 @@ impl PageFlushAll { } impl Drop for PageFlushAll { fn drop(&mut self) { - unsafe { A::invalidate_all(); } + unsafe { + A::invalidate_all(); + } } } impl Flusher for PageFlushAll { fn consume(&mut self, flush: PageFlush) { - unsafe { flush.ignore(); } + unsafe { + flush.ignore(); + } } } impl + ?Sized> Flusher for &mut T { diff --git a/src/page/mapper.rs b/src/page/mapper.rs index f75c01a9af..28bea52791 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -1,14 +1,7 @@ use core::marker::PhantomData; use crate::{ - Arch, - FrameAllocator, - PageEntry, - PageFlags, - PageFlush, - PageTable, - PhysicalAddress, - TableKind, + Arch, FrameAllocator, PageEntry, PageFlags, PageFlush, PageTable, PhysicalAddress, TableKind, VirtualAddress, }; @@ -50,13 +43,7 @@ impl PageMapper { pub fn table(&self) -> PageTable { // SAFETY: The only way to initialize a PageMapper is via new(), and we assume it upholds // all necessary invariants for this to be safe. - unsafe { - PageTable::new( - VirtualAddress::new(0), - self.table_addr, - A::PAGE_LEVELS - 1 - ) - } + unsafe { PageTable::new(VirtualAddress::new(0), self.table_addr, A::PAGE_LEVELS - 1) } } pub fn allocator(&self) -> &F { @@ -67,7 +54,11 @@ impl PageMapper { &mut self.allocator } - pub unsafe fn remap_with_full(&mut self, virt: VirtualAddress, f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags)) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + pub unsafe fn remap_with_full( + &mut self, + virt: VirtualAddress, + f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags), + ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { self.visit(virt, |p1, i| { let old_entry = p1.entry(i)?; let old_phys = old_entry.address().ok()?; @@ -77,21 +68,41 @@ impl PageMapper { let new_entry = PageEntry::new(new_phys.data() | new_flags.data()); p1.set_entry(i, new_entry); Some((old_flags, old_phys, PageFlush::new(virt))) - }).flatten() + }) + .flatten() } - pub unsafe fn remap_with(&mut self, virt: VirtualAddress, map_flags: impl FnOnce(PageFlags) -> PageFlags) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { - self.remap_with_full(virt, |same_phys, old_flags| (same_phys, map_flags(old_flags))) + pub unsafe fn remap_with( + &mut self, + virt: VirtualAddress, + map_flags: impl FnOnce(PageFlags) -> PageFlags, + ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { + self.remap_with_full(virt, |same_phys, old_flags| { + (same_phys, map_flags(old_flags)) + }) } - pub unsafe fn remap(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + pub unsafe fn remap( + &mut self, + virt: VirtualAddress, + flags: PageFlags, + ) -> Option> { self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) } - pub unsafe fn map(&mut self, virt: VirtualAddress, flags: PageFlags) -> Option> { + pub unsafe fn map( + &mut self, + virt: VirtualAddress, + flags: PageFlags, + ) -> Option> { let phys = self.allocator.allocate_one()?; self.map_phys(virt, phys, flags) } - pub unsafe fn map_phys(&mut self, virt: VirtualAddress, phys: PhysicalAddress, flags: PageFlags) -> Option> { + pub unsafe fn map_phys( + &mut self, + virt: VirtualAddress, + phys: PhysicalAddress, + flags: PageFlags, + ) -> Option> { //TODO: verify virt and phys are aligned //TODO: verify flags have correct bits let entry = PageEntry::new(phys.data() | flags.data()); @@ -109,7 +120,13 @@ impl PageMapper { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { A::ENTRY_FLAG_USER } else { 0 }; + let flags = A::ENTRY_FLAG_READWRITE + | A::ENTRY_FLAG_DEFAULT_TABLE + | if virt.kind() == TableKind::User { + A::ENTRY_FLAG_USER + } else { + 0 + }; table.set_entry(i, PageEntry::new(next_phys.data() | flags)); table.next(i)? } @@ -118,11 +135,19 @@ impl PageMapper { } } } - pub unsafe fn map_linearly(&mut self, phys: PhysicalAddress, flags: PageFlags) -> Option<(VirtualAddress, PageFlush)> { + pub unsafe fn map_linearly( + &mut self, + phys: PhysicalAddress, + flags: PageFlags, + ) -> Option<(VirtualAddress, PageFlush)> { let virt = A::phys_to_virt(phys); self.map_phys(virt, phys, flags).map(|flush| (virt, flush)) } - fn visit(&self, virt: VirtualAddress, f: impl FnOnce(&mut PageTable, usize) -> T) -> Option { + fn visit( + &self, + virt: VirtualAddress, + f: impl FnOnce(&mut PageTable, usize) -> T, + ) -> Option { let mut table = self.table(); unsafe { loop { @@ -140,20 +165,35 @@ impl PageMapper { Some((entry.address().ok()?, entry.flags())) } - pub unsafe fn unmap(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option> { + pub unsafe fn unmap( + &mut self, + virt: VirtualAddress, + unmap_parents: bool, + ) -> Option> { let (old, _, flush) = self.unmap_phys(virt, unmap_parents)?; self.allocator.free_one(old); Some(flush) } - pub unsafe fn unmap_phys(&mut self, virt: VirtualAddress, unmap_parents: bool) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { + pub unsafe fn unmap_phys( + &mut self, + virt: VirtualAddress, + unmap_parents: bool, + ) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { //TODO: verify virt is aligned let mut table = self.table(); let level = table.level(); - unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator).map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator) + .map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) } } -unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable, initial_level: usize, unmap_parents: bool, allocator: &mut impl FrameAllocator) -> Option<(PhysicalAddress, PageFlags)> { +unsafe fn unmap_phys_inner( + virt: VirtualAddress, + table: &mut PageTable, + initial_level: usize, + unmap_parents: bool, + allocator: &mut impl FrameAllocator, +) -> Option<(PhysicalAddress, PageFlags)> { let i = table.index_of(virt)?; if table.level() == 0 { @@ -172,7 +212,9 @@ unsafe fn unmap_phys_inner(virt: VirtualAddress, table: &mut PageTable< if unmap_parents { // TODO: Use a counter? This would reduce the remaining number of available bits, but could be // faster (benchmark is needed). - let is_still_populated = (0..A::PAGE_ENTRIES).map(|j| subtable.entry(j).expect("must be within bounds")).any(|e| e.present()); + let is_still_populated = (0..A::PAGE_ENTRIES) + .map(|j| subtable.entry(j).expect("must be within bounds")) + .any(|e| e.present()); if !is_still_populated { allocator.free_one(subtable.phys()); diff --git a/src/page/mod.rs b/src/page/mod.rs index bcf360b322..c4c4f00241 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -1,10 +1,4 @@ -pub use self::{ - entry::*, - flags::*, - flush::*, - mapper::*, - table::*, -}; +pub use self::{entry::*, flags::*, flush::*, mapper::*, table::*}; mod entry; mod flags; diff --git a/src/page/table.rs b/src/page/table.rs index 3c003cb7ec..323a808c00 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -1,12 +1,7 @@ use core::marker::PhantomData; -use crate::{ - Arch, - PhysicalAddress, - TableKind, - VirtualAddress, -}; use super::PageEntry; +use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress}; pub struct PageTable { base: VirtualAddress, @@ -17,14 +12,19 @@ pub struct PageTable { impl PageTable { pub unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self { - Self { base, phys, level, phantom: PhantomData } + Self { + base, + phys, + level, + phantom: PhantomData, + } } pub unsafe fn top(table_kind: TableKind) -> Self { Self::new( VirtualAddress::new(0), A::table(table_kind), - A::PAGE_LEVELS - 1 + A::PAGE_LEVELS - 1, ) } @@ -85,7 +85,11 @@ impl PageTable { // Canonicalize address first let address = VirtualAddress::new(address.data() & A::PAGE_ADDRESS_MASK); let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; - let level_mask = (A::PAGE_ENTRIES << level_shift) - 1; + // Intentionally wraps around at last-level table to get all-ones mask on architectures + // where addressable physical address space covers entire usized space (e.g. x86) + let level_mask = A::PAGE_ENTRIES + .wrapping_shl(level_shift as u32) + .wrapping_sub(1); if address >= self.base && address <= self.base.add(level_mask) { Some((address.data() >> level_shift) & A::PAGE_ENTRY_MASK) } else { From 5f9a587968fc914c0dacd45c4a89a8a72c074cde Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 8 Sep 2024 20:48:05 +0200 Subject: [PATCH 104/120] Convert repr(packed) to repr(C, packed). --- src/allocator/frame/buddy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index 564004303a..b4c224f957 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -7,7 +7,7 @@ use crate::{ #[repr(transparent)] struct BuddyUsage(u8); -#[repr(packed)] +#[repr(C, packed)] struct BuddyEntry { base: PhysicalAddress, size: usize, From d95ea6598fa3b14ae876acafb5df2c59f73867f2 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Fri, 14 Jun 2024 11:36:47 +0200 Subject: [PATCH 105/120] Fix bump allocator. --- src/allocator/frame/bump.rs | 83 ++++++++++++++----------------------- src/lib.rs | 1 + 2 files changed, 32 insertions(+), 52 deletions(-) diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 51bee174ca..792bc7c9e3 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -2,77 +2,59 @@ use core::marker::PhantomData; use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress}; +#[derive(Debug)] pub struct BumpAllocator { - areas: &'static [MemoryArea], - offset: usize, - abs_offset: PhysicalAddress, + orig_areas: (&'static [MemoryArea], usize), + cur_areas: (&'static [MemoryArea], usize), _marker: PhantomData A>, } impl BumpAllocator { - pub fn new(areas: &'static [MemoryArea], _offset: usize) -> Self { + pub fn new(mut areas: &'static [MemoryArea], mut offset: usize) -> Self { + while let Some(first) = areas.first() && first.size <= offset { + offset -= first.size; + areas = &areas[1..]; + } + Self { - areas, - offset: 0, - abs_offset: areas[0].base, + orig_areas: (areas, offset), + cur_areas: (areas, offset), _marker: PhantomData, } } pub fn areas(&self) -> &'static [MemoryArea] { - self.areas + todo!() } /// Returns one semifree and the fully free areas. The offset is the number of bytes after /// which the first area is free. pub fn free_areas(&self) -> (&'static [MemoryArea], usize) { - let mut areas = self.areas; - let mut offset = self.offset; - - loop { - let Some(area) = areas.first() else { - return (&[], 0); - }; - - if offset > area.size { - areas = &areas[1..]; - offset -= area.size; - } else { - return (areas, offset); - } - } + self.cur_areas } pub fn abs_offset(&self) -> PhysicalAddress { - self.abs_offset + let (areas, off) = self.cur_areas; + areas.first().map_or(PhysicalAddress::new(0), |a| a.base.add(off)) } - pub fn offset(&self) -> usize { - self.offset + (unsafe { self.usage().total().data() - self.usage().free().data() }) * A::PAGE_SIZE } } impl FrameAllocator for BumpAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option { - let mut offset = self.offset; - for area in self.areas.iter() { - if offset < area.size { - if area.size - offset < count.data() * A::PAGE_SIZE { - // The area may be too small for this alloc request. In that case, skip to the - // next area. - self.offset += area.size - offset; - offset = 0; - continue; - } + let req_size = count.data() * A::PAGE_SIZE; - let page_phys = area.base.add(offset); - let page_virt = A::phys_to_virt(page_phys); - A::write_bytes(page_virt, 0, count.data() * A::PAGE_SIZE); - self.offset += count.data() * A::PAGE_SIZE; - - self.abs_offset = page_phys; - return Some(page_phys); + let block = loop { + let area = self.cur_areas.0.first()?; + let off = self.cur_areas.1; + if area.size - off <= req_size { + self.cur_areas = (&self.cur_areas.0[1..], 0); } - offset -= area.size; - } - None + self.cur_areas.1 += req_size; + + break area.base.add(off); + }; + A::write_bytes(A::phys_to_virt(block), 0, req_size); + Some(block) } unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { @@ -80,11 +62,8 @@ impl FrameAllocator for BumpAllocator { } unsafe fn usage(&self) -> FrameUsage { - let mut total = 0; - for area in self.areas.iter() { - total += area.size >> A::PAGE_SHIFT; - } - let used = self.offset >> A::PAGE_SHIFT; - FrameUsage::new(FrameCount::new(used), FrameCount::new(total)) + let total = self.orig_areas.0.iter().map(|a| a.size).sum::() - self.orig_areas.1; + let free = self.cur_areas.0.iter().map(|a| a.size).sum::() - self.cur_areas.1; + FrameUsage::new(FrameCount::new(total - free), FrameCount::new(total)) } } diff --git a/src/lib.rs b/src/lib.rs index b0615774d7..e4dc8ff060 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ #![cfg_attr(not(feature = "std"), no_std)] #![feature(doc_cfg)] +#![feature(let_chains)] pub use crate::{allocator::*, arch::*, page::*}; From b7d3acf6069bc33ae4275829ffb7dcd2c9b63e83 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 07:42:21 +0300 Subject: [PATCH 106/120] Separate leaf page and directory pages USER flag This is required for RISC-V. Privileged spec says: > For non-leaf PTEs, the D, A, and U bits are reserved for future standard use. > Until their use is defined by a standard extension, they must be cleared by software > for forward compatibility. QEMU fails address translation if it sees any of these flags set on non-leaf page entry. --- src/arch/aarch64.rs | 2 +- src/arch/emulate.rs | 2 +- src/arch/mod.rs | 3 ++- src/arch/riscv64/sv39.rs | 3 ++- src/arch/riscv64/sv48.rs | 3 ++- src/arch/x86.rs | 2 +- src/arch/x86_64.rs | 2 +- src/page/flags.rs | 4 ++-- src/page/mapper.rs | 2 +- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 3a811a0184..106c102b6e 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -24,7 +24,7 @@ impl Arch for AArch64Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 1 << 7; const ENTRY_FLAG_READWRITE: usize = 0; - const ENTRY_FLAG_USER: usize = 1 << 6; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 6; // This sets both userspace and privileged execute never //TODO: Separate the two? const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 88a200769a..c44d72a72d 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -20,7 +20,7 @@ impl Arch for EmulateArch { const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_READONLY: usize = X8664Arch::ENTRY_FLAG_READONLY; const ENTRY_FLAG_READWRITE: usize = X8664Arch::ENTRY_FLAG_READWRITE; - const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; + const ENTRY_FLAG_PAGE_USER: usize = X8664Arch::ENTRY_FLAG_PAGE_USER; const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; const ENTRY_FLAG_EXEC: usize = X8664Arch::ENTRY_FLAG_EXEC; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f6f5ec9c53..0a3828106d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -36,7 +36,8 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_PRESENT: usize; const ENTRY_FLAG_READONLY: usize; const ENTRY_FLAG_READWRITE: usize; - const ENTRY_FLAG_USER: usize; + const ENTRY_FLAG_PAGE_USER: usize; // Leaf table user page flag + const ENTRY_FLAG_TABLE_USER: usize = Self::ENTRY_FLAG_PAGE_USER; // Directory user page table flag const ENTRY_FLAG_NO_EXEC: usize; const ENTRY_FLAG_EXEC: usize; const ENTRY_FLAG_GLOBAL: usize; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 0980428754..72b99a1e27 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -20,7 +20,8 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; - const ENTRY_FLAG_USER: usize = 1 << 4; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; + const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index b1142317f0..196aee28e3 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -20,7 +20,8 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; - const ENTRY_FLAG_USER: usize = 1 << 4; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; + const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 8dcd238f3e..176aef175d 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -17,7 +17,7 @@ impl Arch for X86Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; - const ENTRY_FLAG_USER: usize = 1 << 2; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_GLOBAL: usize = 0; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 113f66f036..ccde251a7c 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -16,7 +16,7 @@ impl Arch for X8664Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; - const ENTRY_FLAG_USER: usize = 1 << 2; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_GLOBAL: usize = 0; diff --git a/src/page/flags.rs b/src/page/flags.rs index bffe12643e..ca03d1ce0f 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -72,12 +72,12 @@ impl PageFlags { #[must_use] #[inline(always)] pub fn user(self, value: bool) -> Self { - self.custom_flag(A::ENTRY_FLAG_USER, value) + self.custom_flag(A::ENTRY_FLAG_PAGE_USER, value) } #[inline(always)] pub fn has_user(&self) -> bool { - self.has_flag(A::ENTRY_FLAG_USER) + self.has_flag(A::ENTRY_FLAG_PAGE_USER) } #[must_use] diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 28bea52791..5d2584ee63 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -123,7 +123,7 @@ impl PageMapper { let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { - A::ENTRY_FLAG_USER + A::ENTRY_FLAG_TABLE_USER } else { 0 }; From a96ea6f5f38a5f6c71b50857edb0854a34757678 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 08:04:24 +0300 Subject: [PATCH 107/120] Allow physical address bits be anywhere in PTE entry Before this commit, RMM assumed base physical address was presented in PTE as is, i.e. physical page address was shifted exactly to PAGE_SHIFT, so physical address can be extracted from PTE by simply masking off some bits and can be placed in PTE by simple addition/OR. This is not the case for RISC-V which has 4Kb base page so 12 bits PAGE_SHIFT, yet physical page address is only shifted 10 bits in PTE. This commit removes this assumption. NOTE: This commit changes meaning of constants: * ENTRY_ADDRESS_SIZE from "total physical size in bytes" to "total physical size in PAGES" * ENTRY_ADDRESS_MASK from "mask of physical bits in PTE" to "mask of physical bits starting at bit 0" --- src/arch/aarch64.rs | 6 +++--- src/arch/emulate.rs | 11 +++++++---- src/arch/mod.rs | 10 +++++----- src/arch/riscv64/sv39.rs | 23 ++++++++++++----------- src/arch/riscv64/sv48.rs | 15 ++++++++------- src/arch/x86.rs | 6 +++--- src/arch/x86_64.rs | 6 +++--- src/page/entry.rs | 13 +++++++++++-- src/page/mapper.rs | 10 +++++----- src/page/table.rs | 2 +- 10 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 106c102b6e..5999d60a44 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -11,7 +11,7 @@ impl Arch for AArch64Arch { const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 //TODO - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 40; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Page flag | 1 << 10 // Access flag @@ -110,8 +110,8 @@ mod tests { assert_eq!(AArch64Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(AArch64Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); - assert_eq!(AArch64Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(AArch64Arch::ENTRY_ADDRESS_SIZE, 0x0000_0100_0000_0000); + assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x0000_00FF_FFFF_FFFF); assert_eq!(AArch64Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index c44d72a72d..d228161df0 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -29,6 +29,8 @@ impl Arch for EmulateArch { const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; const ENTRY_FLAG_NO_GLOBAL: usize = X8664Arch::ENTRY_FLAG_NO_GLOBAL; + const ENTRY_ADDRESS_WIDTH: usize = X8664Arch::ENTRY_ADDRESS_WIDTH; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); @@ -275,7 +277,7 @@ impl Machine { } // Page directory pointer - let a3 = e3 & A::ENTRY_ADDRESS_MASK; + let a3 = ((e3 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; for i3 in 0..A::PAGE_ENTRIES { let e2 = self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); @@ -285,7 +287,7 @@ impl Machine { } // Page directory - let a2 = e2 & A::ENTRY_ADDRESS_MASK; + let a2 = ((e2 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; for i2 in 0..A::PAGE_ENTRIES { let e1 = self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); @@ -295,7 +297,8 @@ impl Machine { } // Page table - let a1 = e1 & A::ENTRY_ADDRESS_MASK; + let a1 = + ((e1 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; for i1 in 0..A::PAGE_ENTRIES { let e = self .read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); @@ -308,7 +311,7 @@ impl Machine { let page = (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12); //println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); self.map - .insert(VirtualAddress::new(page), PageEntry::new(e)); + .insert(VirtualAddress::new(page), PageEntry::from_data(e)); } } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 0a3828106d..1d40b3109c 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -30,7 +30,8 @@ pub trait Arch: Clone + Copy { const PAGE_ENTRY_SHIFT: usize; const PAGE_LEVELS: usize; - const ENTRY_ADDRESS_SHIFT: usize; + const ENTRY_ADDRESS_WIDTH: usize; // Number of bits of physical address in PTE + const ENTRY_ADDRESS_SHIFT: usize = Self::PAGE_SHIFT; // Offset of physical address in PTE const ENTRY_FLAG_DEFAULT_PAGE: usize; const ENTRY_FLAG_DEFAULT_TABLE: usize; const ENTRY_FLAG_PRESENT: usize; @@ -55,10 +56,9 @@ pub trait Arch: Clone + Copy { const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1) as usize; - const ENTRY_ADDRESS_SIZE: u64 = 1 << Self::ENTRY_ADDRESS_SHIFT; - const ENTRY_ADDRESS_MASK: usize = - (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; - const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; + const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_WIDTH; // size of addressable physical memory, in pages + const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - 1; // Mask of physical address, starting at 0th bit + const ENTRY_FLAGS_MASK: usize = !(Self::ENTRY_ADDRESS_MASK << Self::ENTRY_ADDRESS_SHIFT); unsafe fn init() -> &'static [MemoryArea]; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 72b99a1e27..f13e649ee4 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -10,8 +10,9 @@ impl Arch for RiscV64Sv39Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 3; // L0, L1, L2 - //TODO - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 44; + const ENTRY_ADDRESS_SHIFT: usize = 10; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag @@ -27,7 +28,7 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_GLOBAL: usize = 1 << 5; const ENTRY_FLAG_NO_GLOBAL: usize = 0; - const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; + const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; unsafe fn init() -> &'static [MemoryArea] { unimplemented!("RiscV64Sv39Arch::init unimplemented"); @@ -44,7 +45,7 @@ impl Arch for RiscV64Sv39Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN ) } @@ -56,10 +57,10 @@ impl Arch for RiscV64Sv39Arch { } fn virt_is_valid(address: VirtualAddress) -> bool { - const MASK: usize = 0xFFFF_FFC0_0000_0000; - let masked = address.data() & MASK; + let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); + let masked = address.data() & mask; - masked == MASK || masked == 0 + masked == mask || masked == 0 } } @@ -80,11 +81,11 @@ mod tests { assert_eq!(RiscV64Sv39Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(RiscV64Sv39Arch::PAGE_NEGATIVE_MASK, 0xFFFF_FF80_0000_0000); - assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); - assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE, 0x0000_1000_0000_0000); + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x0000_0FFF_FFFF_FFFF); + assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFC0_0000_0000_03FF); - assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_FFC0_0000_0000); } #[test] fn is_canonical() { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 196aee28e3..a63f887afd 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -10,8 +10,9 @@ impl Arch for RiscV64Sv48Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 - //TODO - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 44; + const ENTRY_ADDRESS_SHIFT: usize = 10; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag @@ -44,7 +45,7 @@ impl Arch for RiscV64Sv48Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN ) } @@ -56,7 +57,7 @@ impl Arch for RiscV64Sv48Arch { } fn virt_is_valid(address: VirtualAddress) -> bool { // RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64. - let mask = 0xFFFF_8000_0000_0000; + let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); let masked = address.data() & mask; masked == mask || masked == 0 @@ -80,9 +81,9 @@ mod tests { assert_eq!(RiscV64Sv48Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(RiscV64Sv48Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); - assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); - assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE, 0x0000_1000_0000_0000); + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x0000_0FFF_FFFF_FFFF); + assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFC0_0000_0000_03FF); assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 176aef175d..3c4ac924f7 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -11,7 +11,7 @@ impl Arch for X86Arch { const PAGE_ENTRY_SHIFT: usize = 10; // 1024 entries, 4 bytes each const PAGE_LEVELS: usize = 2; // PD, PT - const ENTRY_ADDRESS_SHIFT: usize = 32; + const ENTRY_ADDRESS_WIDTH: usize = 20; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; @@ -70,8 +70,8 @@ mod tests { assert_eq!(X86Arch::PAGE_ENTRY_MASK, 0x3FF); assert_eq!(X86Arch::PAGE_NEGATIVE_MASK, 0x0000_0000_0000); - assert_eq!(X86Arch::ENTRY_ADDRESS_SIZE, 0x0000_0001_0000_0000); - assert_eq!(X86Arch::ENTRY_ADDRESS_MASK, 0xFFFF_F000); + assert_eq!(X86Arch::ENTRY_ADDRESS_SIZE, 0x0000_0000_0010_0000); + assert_eq!(X86Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF); assert_eq!(X86Arch::ENTRY_FLAGS_MASK, 0x0000_0FFF); assert_eq!(X86Arch::PHYS_OFFSET, 0x8000_0000); diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index ccde251a7c..316323adc3 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -10,7 +10,7 @@ impl Arch for X8664Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 40; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; @@ -80,8 +80,8 @@ mod tests { assert_eq!(X8664Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(X8664Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); - assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0000_0100_0000_0000); + assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x0000_00FF_FFFF_FFFF); assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); assert_eq!(X8664Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); diff --git a/src/page/entry.rs b/src/page/entry.rs index 34a83d448f..ac75f162b0 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -10,7 +10,14 @@ pub struct PageEntry { impl PageEntry { #[inline(always)] - pub fn new(data: usize) -> Self { + pub fn new(address: usize, flags: usize) -> Self { + let data = (((address >> A::PAGE_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::ENTRY_ADDRESS_SHIFT) + | flags; + Self::from_data(data) + } + + #[inline(always)] + pub fn from_data(data: usize) -> Self { Self { data, phantom: PhantomData, @@ -24,7 +31,9 @@ impl PageEntry { #[inline(always)] pub fn address(&self) -> Result { - let addr = PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK); + let addr = PhysicalAddress( + ((self.data >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT, + ); if self.present() { Ok(addr) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 5d2584ee63..842d5b1125 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -65,7 +65,7 @@ impl PageMapper { let old_flags = old_entry.flags(); let (new_phys, new_flags) = f(old_phys, old_flags); // TODO: Higher-level PageEntry::new interface? - let new_entry = PageEntry::new(new_phys.data() | new_flags.data()); + let new_entry = PageEntry::new(new_phys.data(), new_flags.data()); p1.set_entry(i, new_entry); Some((old_flags, old_phys, PageFlush::new(virt))) }) @@ -105,7 +105,7 @@ impl PageMapper { ) -> Option> { //TODO: verify virt and phys are aligned //TODO: verify flags have correct bits - let entry = PageEntry::new(phys.data() | flags.data()); + let entry = PageEntry::new(phys.data(), flags.data()); let mut table = self.table(); loop { let i = table.index_of(virt)?; @@ -127,7 +127,7 @@ impl PageMapper { } else { 0 }; - table.set_entry(i, PageEntry::new(next_phys.data() | flags)); + table.set_entry(i, PageEntry::new(next_phys.data(), flags)); table.next(i)? } }; @@ -198,7 +198,7 @@ unsafe fn unmap_phys_inner( if table.level() == 0 { let entry_opt = table.entry(i); - table.set_entry(i, PageEntry::new(0)); + table.set_entry(i, PageEntry::new(0, 0)); let entry = entry_opt?; Some((entry.address().ok()?, entry.flags())) @@ -218,7 +218,7 @@ unsafe fn unmap_phys_inner( if !is_still_populated { allocator.free_one(subtable.phys()); - table.set_entry(i, PageEntry::new(0)); + table.set_entry(i, PageEntry::new(0, 0)); } } diff --git a/src/page/table.rs b/src/page/table.rs index 323a808c00..70d281543d 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -72,7 +72,7 @@ impl PageTable { pub unsafe fn entry(&self, i: usize) -> Option> { let addr = self.entry_virt(i)?; - Some(PageEntry::new(A::read::(addr))) + Some(PageEntry::from_data(A::read::(addr))) } pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { From f51cd00f00f445b364f2dde5825acb3e15ad6d7e Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 08:58:01 +0300 Subject: [PATCH 108/120] Remove read/write flags from common PDE flags. RISC-V convention marks PDE with no read/write/execute, so we can't have none of this flags set there. Remove their setting from PDE handling code and instead set them as appropriate in arch-specific defaults. Also enable both readonly and readwrite flags to be non-zero (as long as their intersection completely masks both of them), as required for RISC-V PTE handling. --- src/arch/aarch64.rs | 1 + src/arch/riscv64/sv39.rs | 9 +++------ src/arch/riscv64/sv48.rs | 9 +++------ src/arch/x86.rs | 2 +- src/arch/x86_64.rs | 2 +- src/page/flags.rs | 18 ++++++++++-------- src/page/mapper.rs | 3 +-- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 5999d60a44..82e4983183 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -18,6 +18,7 @@ impl Arch for AArch64Arch { | Self::ENTRY_FLAG_NO_GLOBAL; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT + | Self::ENTRY_FLAG_READWRITE | 1 << 1 // Table flag | 1 << 10 // Access flag ; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index f13e649ee4..607f4a2b1e 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -13,14 +13,11 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_ADDRESS_WIDTH: usize = 44; const ENTRY_ADDRESS_SHIFT: usize = 10; - const ENTRY_FLAG_DEFAULT_PAGE: usize - = Self::ENTRY_FLAG_PRESENT - | 1 << 1 // Read flag - ; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; - const ENTRY_FLAG_READONLY: usize = 0; - const ENTRY_FLAG_READWRITE: usize = 1 << 2; + const ENTRY_FLAG_READONLY: usize = 1 << 1; + const ENTRY_FLAG_READWRITE: usize = 3 << 1; const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index a63f887afd..e80e52a000 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -13,14 +13,11 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_ADDRESS_WIDTH: usize = 44; const ENTRY_ADDRESS_SHIFT: usize = 10; - const ENTRY_FLAG_DEFAULT_PAGE: usize - = Self::ENTRY_FLAG_PRESENT - | 1 << 1 // Read flag - ; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; - const ENTRY_FLAG_READONLY: usize = 0; - const ENTRY_FLAG_READWRITE: usize = 1 << 2; + const ENTRY_FLAG_READONLY: usize = 1 << 1; + const ENTRY_FLAG_READWRITE: usize = 3 << 1; const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 3c4ac924f7..b0e333a342 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -13,7 +13,7 @@ impl Arch for X86Arch { const ENTRY_ADDRESS_WIDTH: usize = 20; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; - const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 316323adc3..25fa84691d 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -12,7 +12,7 @@ impl Arch for X8664Arch { const ENTRY_ADDRESS_WIDTH: usize = 40; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; - const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; diff --git a/src/page/flags.rs b/src/page/flags.rs index ca03d1ce0f..9de75484ef 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -27,10 +27,7 @@ impl PageFlags { unsafe { Self::from_data( // Flags set to present, kernel space, read-only, no-execute by default - A::ENTRY_FLAG_DEFAULT_TABLE - | A::ENTRY_FLAG_READONLY - | A::ENTRY_FLAG_NO_EXEC - | A::ENTRY_FLAG_NO_GLOBAL, + A::ENTRY_FLAG_DEFAULT_TABLE | A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_NO_GLOBAL, ) } } @@ -83,14 +80,19 @@ impl PageFlags { #[must_use] #[inline(always)] pub fn write(self, value: bool) -> Self { - // Architecture may use readonly or readwrite, support either - self.custom_flag(A::ENTRY_FLAG_READONLY, !value) - .custom_flag(A::ENTRY_FLAG_READWRITE, value) + // Architecture may use readonly or readwrite, or both, support either + if value { + self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false) + .custom_flag(A::ENTRY_FLAG_READWRITE, true) + } else { + self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false) + .custom_flag(A::ENTRY_FLAG_READONLY, true) + } } #[inline(always)] pub fn has_write(&self) -> bool { - // Architecture may use readonly or readwrite, support either + // Architecture may use readonly or readwrite, or both, support either self.data & (A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE) == A::ENTRY_FLAG_READWRITE } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 842d5b1125..a6aa726be8 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -120,8 +120,7 @@ impl PageMapper { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - let flags = A::ENTRY_FLAG_READWRITE - | A::ENTRY_FLAG_DEFAULT_TABLE + let flags = A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { A::ENTRY_FLAG_TABLE_USER } else { From 192dd8283fcc45ebfa94500695bb3fabdc089887 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 09:01:23 +0300 Subject: [PATCH 109/120] RISC-V: implement TLB flush --- src/arch/riscv64/sv39.rs | 11 ++++++++--- src/arch/riscv64/sv48.rs | 12 +++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 607f4a2b1e..81dd8583f1 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -32,9 +32,13 @@ impl Arch for RiscV64Sv39Arch { } #[inline(always)] - unsafe fn invalidate(_address: VirtualAddress) { - //TODO: can one address be invalidated? - Self::invalidate_all(); + unsafe fn invalidate(address: VirtualAddress) { + asm!("sfence.vma {}", in(reg) address.data()); + } + + #[inline(always)] + unsafe fn invalidate_all() { + asm!("sfence.vma"); } #[inline(always)] @@ -51,6 +55,7 @@ impl Arch for RiscV64Sv39Arch { let satp = (8 << 60) | // Sv39 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); } fn virt_is_valid(address: VirtualAddress) -> bool { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index e80e52a000..b6796ab3d8 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -32,9 +32,13 @@ impl Arch for RiscV64Sv48Arch { } #[inline(always)] - unsafe fn invalidate(_address: VirtualAddress) { - //TODO: can one address be invalidated? - Self::invalidate_all(); + unsafe fn invalidate(address: VirtualAddress) { + asm!("sfence.vma {}", in(reg) address.data()); + } + + #[inline(always)] + unsafe fn invalidate_all() { + asm!("sfence.vma"); } #[inline(always)] @@ -51,7 +55,9 @@ impl Arch for RiscV64Sv48Arch { let satp = (9 << 60) | // Sv48 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); } + fn virt_is_valid(address: VirtualAddress) -> bool { // RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64. let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); From ed8bcfca1f7427d8acca966a8c6ca36ce887467b Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 28 Jun 2024 06:30:25 +0300 Subject: [PATCH 110/120] Add write combining page flags where applicable --- src/arch/aarch64.rs | 1 + src/arch/emulate.rs | 2 ++ src/arch/mod.rs | 1 + src/arch/riscv64/sv39.rs | 1 + src/arch/riscv64/sv48.rs | 1 + src/arch/x86.rs | 1 + src/arch/x86_64.rs | 1 + src/page/flags.rs | 6 ++++++ 8 files changed, 14 insertions(+) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 82e4983183..687294d3b5 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -32,6 +32,7 @@ impl Arch for AArch64Arch { const ENTRY_FLAG_EXEC: usize = 0; const ENTRY_FLAG_GLOBAL: usize = 0; const ENTRY_FLAG_NO_GLOBAL: usize = 1 << 11; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index d228161df0..3e381db676 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -31,6 +31,8 @@ impl Arch for EmulateArch { const ENTRY_ADDRESS_WIDTH: usize = X8664Arch::ENTRY_ADDRESS_WIDTH; + const ENTRY_FLAG_WRITE_COMBINING: usize = X8664Arch::ENTRY_FLAG_WRITE_COMBINING; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 1d40b3109c..cd68f09f2c 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -43,6 +43,7 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_EXEC: usize; const ENTRY_FLAG_GLOBAL: usize; const ENTRY_FLAG_NO_GLOBAL: usize; + const ENTRY_FLAG_WRITE_COMBINING: usize; const PHYS_OFFSET: usize; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 81dd8583f1..31ade20c1e 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -24,6 +24,7 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index b6796ab3d8..7e80ad0c3a 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -24,6 +24,7 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index b0e333a342..7e078a2fc0 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -23,6 +23,7 @@ impl Arch for X86Arch { const ENTRY_FLAG_NO_GLOBAL: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; // NOT AVAILABLE UNLESS PAE IS USED! const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 1 << 7; const PHYS_OFFSET: usize = 0x8000_0000; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 25fa84691d..3b0bacff0d 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -22,6 +22,7 @@ impl Arch for X8664Arch { const ENTRY_FLAG_NO_GLOBAL: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 1 << 7; const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1) as usize; // PML4 slot 256 and onwards diff --git a/src/page/flags.rs b/src/page/flags.rs index 9de75484ef..3d90691d60 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -56,6 +56,12 @@ impl PageFlags { self } + #[must_use] + #[inline(always)] + pub fn write_combining(self, value: bool) -> Self { + self.custom_flag(A::ENTRY_FLAG_WRITE_COMBINING, value) + } + #[inline(always)] pub fn has_flag(&self, flag: usize) -> bool { self.data & flag == flag From 91ccf4e6aa904af4fc46d112b406a27a551ef9f6 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Tue, 22 Oct 2024 22:14:39 +0300 Subject: [PATCH 111/120] Fix a bump allocator error --- src/allocator/frame/bump.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 792bc7c9e3..ee897feb80 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -46,8 +46,9 @@ impl FrameAllocator for BumpAllocator { let block = loop { let area = self.cur_areas.0.first()?; let off = self.cur_areas.1; - if area.size - off <= req_size { + if area.size - off < req_size { self.cur_areas = (&self.cur_areas.0[1..], 0); + continue; } self.cur_areas.1 += req_size; @@ -64,6 +65,6 @@ impl FrameAllocator for BumpAllocator { unsafe fn usage(&self) -> FrameUsage { let total = self.orig_areas.0.iter().map(|a| a.size).sum::() - self.orig_areas.1; let free = self.cur_areas.0.iter().map(|a| a.size).sum::() - self.cur_areas.1; - FrameUsage::new(FrameCount::new(total - free), FrameCount::new(total)) + FrameUsage::new(FrameCount::new((total - free) / A::PAGE_SIZE), FrameCount::new(total / A::PAGE_SIZE)) } } From 08c00a434d451c3b4edc5b1f75b26189bf4e06a2 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 10 Sep 2025 15:37:17 +0200 Subject: [PATCH 112/120] Update to the 2024 edition On newer rustc versions let_chain is only allowed with the 2024 edition. --- Cargo.toml | 2 +- src/allocator/frame/buddy.rs | 358 ++++++++++++++++++----------------- src/allocator/frame/bump.rs | 41 ++-- src/allocator/frame/mod.rs | 16 +- src/arch/aarch64.rs | 46 +++-- src/arch/emulate.rs | 84 ++++---- src/arch/mod.rs | 12 +- src/arch/riscv64/sv39.rs | 28 ++- src/arch/riscv64/sv48.rs | 28 ++- src/arch/x86_64.rs | 16 +- src/main.rs | 350 ++++++++++++++++++---------------- src/page/mapper.rs | 187 ++++++++++-------- src/page/table.rs | 74 +++++--- 13 files changed, 679 insertions(+), 563 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index adafa2a7dc..d5755aae45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "rmm" version = "0.1.0" authors = ["Jeremy Soller "] -edition = "2018" +edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/allocator/frame/buddy.rs b/src/allocator/frame/buddy.rs index b4c224f957..def1c87a84 100644 --- a/src/allocator/frame/buddy.rs +++ b/src/allocator/frame/buddy.rs @@ -54,22 +54,28 @@ impl BuddyEntry { } unsafe fn usage_addr(&self, page: usize) -> Option { - if page < self.pages() { - let phys = self.base.add(page * mem::size_of::()); - Some(A::phys_to_virt(phys)) - } else { - None + unsafe { + if page < self.pages() { + let phys = self.base.add(page * mem::size_of::()); + Some(A::phys_to_virt(phys)) + } else { + None + } } } unsafe fn usage(&self, page: usize) -> Option { - let addr = self.usage_addr(page)?; - Some(A::read(addr)) + unsafe { + let addr = self.usage_addr(page)?; + Some(A::read(addr)) + } } unsafe fn set_usage(&self, page: usize, usage: BuddyUsage) -> Option<()> { - let addr = self.usage_addr(page)?; - Some(A::write(addr, usage)) + unsafe { + let addr = self.usage_addr(page)?; + Some(A::write(addr, usage)) + } } } @@ -82,208 +88,216 @@ impl BuddyAllocator { const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::>(); pub unsafe fn new(mut bump_allocator: BumpAllocator) -> Option { - // Allocate buddy table - let table_phys = bump_allocator.allocate_one()?; - let table_virt = A::phys_to_virt(table_phys); - for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { - let virt = table_virt.add(i * mem::size_of::>()); - A::write(virt, BuddyEntry::::empty()); - } - - let allocator = Self { - table_virt, - phantom: PhantomData, - }; - - // Add areas to buddy table, combining areas when possible, and skipping frames used - // by the bump allocator - let mut offset = bump_allocator.offset(); - for old_area in bump_allocator.areas().iter() { - let mut area = old_area.clone(); - if offset >= area.size { - offset -= area.size; - continue; - } else if offset > 0 { - area.base = area.base.add(offset); - area.size -= offset; - offset = 0; - } + unsafe { + // Allocate buddy table + let table_phys = bump_allocator.allocate_one()?; + let table_virt = A::phys_to_virt(table_phys); for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { + let virt = table_virt.add(i * mem::size_of::>()); + A::write(virt, BuddyEntry::::empty()); + } + + let allocator = Self { + table_virt, + phantom: PhantomData, + }; + + // Add areas to buddy table, combining areas when possible, and skipping frames used + // by the bump allocator + let mut offset = bump_allocator.offset(); + for old_area in bump_allocator.areas().iter() { + let mut area = old_area.clone(); + if offset >= area.size { + offset -= area.size; + continue; + } else if offset > 0 { + area.base = area.base.add(offset); + area.size -= offset; + offset = 0; + } + for i in 0..(A::PAGE_SIZE / mem::size_of::>()) { + let virt = table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); + let inserted = if area.base.add(area.size) == { entry.base } { + // Combine entry at start + entry.base = area.base; + entry.size += area.size; + true + } else if area.base == entry.base.add(entry.size) { + // Combine entry at end + entry.size += area.size; + true + } else if entry.size == 0 { + // Create new entry + entry.base = area.base; + entry.size = area.size; + true + } else { + false + }; + if inserted { + A::write(virt, entry); + break; + } + } + } + + //TODO: sort areas? + + // Allocate buddy maps + for i in 0..Self::BUDDY_ENTRIES { let virt = table_virt.add(i * mem::size_of::>()); let mut entry = A::read::>(virt); - let inserted = if area.base.add(area.size) == { entry.base } { - // Combine entry at start - entry.base = area.base; - entry.size += area.size; - true - } else if area.base == entry.base.add(entry.size) { - // Combine entry at end - entry.size += area.size; - true - } else if entry.size == 0 { - // Create new entry - entry.base = area.base; - entry.size = area.size; - true - } else { - false - }; - if inserted { - A::write(virt, entry); - break; - } - } - } - //TODO: sort areas? + // Only set up entries that have enough space for their own usage map + let usage_pages = entry.usage_pages(); + if entry.pages() > usage_pages { + // Mark all usage bytes as unused + let usage_start = entry.usage_addr(0)?; + for page in 0..usage_pages { + A::write_bytes(usage_start.add(page << A::PAGE_SHIFT), 0, A::PAGE_SIZE); + } - // Allocate buddy maps - for i in 0..Self::BUDDY_ENTRIES { - let virt = table_virt.add(i * mem::size_of::>()); - let mut entry = A::read::>(virt); - - // Only set up entries that have enough space for their own usage map - let usage_pages = entry.usage_pages(); - if entry.pages() > usage_pages { - // Mark all usage bytes as unused - let usage_start = entry.usage_addr(0)?; - for page in 0..usage_pages { - A::write_bytes(usage_start.add(page << A::PAGE_SHIFT), 0, A::PAGE_SIZE); + // Mark bytes used for usage as used + for page in 0..usage_pages { + entry.set_usage(page, BuddyUsage(1))?; + } } - // Mark bytes used for usage as used - for page in 0..usage_pages { - entry.set_usage(page, BuddyUsage(1))?; - } + // Skip the pages used for usage + entry.skip = usage_pages; + + // Set used pages to pages used for usage + entry.used = usage_pages; + + // Write updated entry + A::write(virt, entry); } - // Skip the pages used for usage - entry.skip = usage_pages; - - // Set used pages to pages used for usage - entry.used = usage_pages; - - // Write updated entry - A::write(virt, entry); + Some(allocator) } - - Some(allocator) } } impl FrameAllocator for BuddyAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option { - if self.table_virt.data() == 0 { - return None; - } + unsafe { + if self.table_virt.data() == 0 { + return None; + } - for entry_i in 0..Self::BUDDY_ENTRIES { - let virt = self - .table_virt - .add(entry_i * mem::size_of::>()); - let mut entry = A::read::>(virt); + for entry_i in 0..Self::BUDDY_ENTRIES { + let virt = self + .table_virt + .add(entry_i * mem::size_of::>()); + let mut entry = A::read::>(virt); - let mut free_page = entry.skip; - let mut free_count = 0; - for page in entry.skip..entry.pages() { - let usage = entry.usage(page)?; - if usage.0 == 0 { - free_count += 1; + let mut free_page = entry.skip; + let mut free_count = 0; + for page in entry.skip..entry.pages() { + let usage = entry.usage(page)?; + if usage.0 == 0 { + free_count += 1; - if free_count == count.data() { - break; + if free_count == count.data() { + break; + } + } else { + free_page = page + 1; + free_count = 0; } - } else { - free_page = page + 1; - free_count = 0; + } + + if free_count == count.data() { + for page in free_page..free_page + free_count { + // Update usage + let mut usage = entry.usage(page)?; + usage.0 += 1; + entry.set_usage(page, usage); + + // Zero page + let page_phys = entry.base.add(page << A::PAGE_SHIFT); + let page_virt = A::phys_to_virt(page_phys); + A::write_bytes(page_virt, 0, A::PAGE_SIZE); + } + + // Update skip if necessary + if entry.skip == free_page { + entry.skip = free_page + free_count; + } + + // Update used page count + entry.used += free_count; + + // Write updated entry + A::write(virt, entry); + + return Some(entry.base.add(free_page << A::PAGE_SHIFT)); } } - if free_count == count.data() { - for page in free_page..free_page + free_count { - // Update usage - let mut usage = entry.usage(page)?; - usage.0 += 1; - entry.set_usage(page, usage); - - // Zero page - let page_phys = entry.base.add(page << A::PAGE_SHIFT); - let page_virt = A::phys_to_virt(page_phys); - A::write_bytes(page_virt, 0, A::PAGE_SIZE); - } - - // Update skip if necessary - if entry.skip == free_page { - entry.skip = free_page + free_count; - } - - // Update used page count - entry.used += free_count; - - // Write updated entry - A::write(virt, entry); - - return Some(entry.base.add(free_page << A::PAGE_SHIFT)); - } + None } - - None } unsafe fn free(&mut self, base: PhysicalAddress, count: FrameCount) { - if self.table_virt.data() == 0 { - return; - } + unsafe { + if self.table_virt.data() == 0 { + return; + } - let size = count.data() * A::PAGE_SIZE; - for i in 0..Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(i * mem::size_of::>()); - let mut entry = A::read::>(virt); + let size = count.data() * A::PAGE_SIZE; + for i in 0..Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::>()); + let mut entry = A::read::>(virt); - if base >= { entry.base } && base.add(size) <= entry.base.add(entry.size) { - let start_page = (base.data() - { entry.base }.data()) >> A::PAGE_SHIFT; - for page in start_page..start_page + count.data() { - let mut usage = entry.usage(page).expect("failed to get usage during free"); + if base >= { entry.base } && base.add(size) <= entry.base.add(entry.size) { + let start_page = (base.data() - { entry.base }.data()) >> A::PAGE_SHIFT; + for page in start_page..start_page + count.data() { + let mut usage = entry.usage(page).expect("failed to get usage during free"); - if usage.0 > 0 { - usage.0 -= 1; - } else { - panic!("tried to free already free frame"); - } - - // If page was freed - if usage.0 == 0 { - // Update skip if necessary - if page < entry.skip { - entry.skip = page; + if usage.0 > 0 { + usage.0 -= 1; + } else { + panic!("tried to free already free frame"); } - // Update used page count - entry.used -= 1; + // If page was freed + if usage.0 == 0 { + // Update skip if necessary + if page < entry.skip { + entry.skip = page; + } + + // Update used page count + entry.used -= 1; + } + + entry + .set_usage(page, usage) + .expect("failed to set usage during free"); } - entry - .set_usage(page, usage) - .expect("failed to set usage during free"); + // Write updated entry + A::write(virt, entry); + + return; } - - // Write updated entry - A::write(virt, entry); - - return; } } } unsafe fn usage(&self) -> FrameUsage { - let mut total = 0; - let mut used = 0; - for i in 0..Self::BUDDY_ENTRIES { - let virt = self.table_virt.add(i * mem::size_of::>()); - let entry = A::read::>(virt); - total += entry.size >> A::PAGE_SHIFT; - used += entry.used; + unsafe { + let mut total = 0; + let mut used = 0; + for i in 0..Self::BUDDY_ENTRIES { + let virt = self.table_virt.add(i * mem::size_of::>()); + let entry = A::read::>(virt); + total += entry.size >> A::PAGE_SHIFT; + used += entry.used; + } + FrameUsage::new(FrameCount::new(used), FrameCount::new(total)) } - FrameUsage::new(FrameCount::new(used), FrameCount::new(total)) } } diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index ee897feb80..9985feff3a 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -11,7 +11,9 @@ pub struct BumpAllocator { impl BumpAllocator { pub fn new(mut areas: &'static [MemoryArea], mut offset: usize) -> Self { - while let Some(first) = areas.first() && first.size <= offset { + while let Some(first) = areas.first() + && first.size <= offset + { offset -= first.size; areas = &areas[1..]; } @@ -32,7 +34,9 @@ impl BumpAllocator { } pub fn abs_offset(&self) -> PhysicalAddress { let (areas, off) = self.cur_areas; - areas.first().map_or(PhysicalAddress::new(0), |a| a.base.add(off)) + areas + .first() + .map_or(PhysicalAddress::new(0), |a| a.base.add(off)) } pub fn offset(&self) -> usize { (unsafe { self.usage().total().data() - self.usage().free().data() }) * A::PAGE_SIZE @@ -41,21 +45,23 @@ impl BumpAllocator { impl FrameAllocator for BumpAllocator { unsafe fn allocate(&mut self, count: FrameCount) -> Option { - let req_size = count.data() * A::PAGE_SIZE; + unsafe { + let req_size = count.data() * A::PAGE_SIZE; - let block = loop { - let area = self.cur_areas.0.first()?; - let off = self.cur_areas.1; - if area.size - off < req_size { - self.cur_areas = (&self.cur_areas.0[1..], 0); - continue; - } - self.cur_areas.1 += req_size; + let block = loop { + let area = self.cur_areas.0.first()?; + let off = self.cur_areas.1; + if area.size - off < req_size { + self.cur_areas = (&self.cur_areas.0[1..], 0); + continue; + } + self.cur_areas.1 += req_size; - break area.base.add(off); - }; - A::write_bytes(A::phys_to_virt(block), 0, req_size); - Some(block) + break area.base.add(off); + }; + A::write_bytes(A::phys_to_virt(block), 0, req_size); + Some(block) + } } unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { @@ -65,6 +71,9 @@ impl FrameAllocator for BumpAllocator { unsafe fn usage(&self) -> FrameUsage { let total = self.orig_areas.0.iter().map(|a| a.size).sum::() - self.orig_areas.1; let free = self.cur_areas.0.iter().map(|a| a.size).sum::() - self.cur_areas.1; - FrameUsage::new(FrameCount::new((total - free) / A::PAGE_SIZE), FrameCount::new(total / A::PAGE_SIZE)) + FrameUsage::new( + FrameCount::new((total - free) / A::PAGE_SIZE), + FrameCount::new(total / A::PAGE_SIZE), + ) } } diff --git a/src/allocator/frame/mod.rs b/src/allocator/frame/mod.rs index 626ff41714..1260def01b 100644 --- a/src/allocator/frame/mod.rs +++ b/src/allocator/frame/mod.rs @@ -49,11 +49,13 @@ pub trait FrameAllocator { unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount); unsafe fn allocate_one(&mut self) -> Option { - self.allocate(FrameCount::new(1)) + unsafe { self.allocate(FrameCount::new(1)) } } unsafe fn free_one(&mut self, address: PhysicalAddress) { - self.free(address, FrameCount::new(1)); + unsafe { + self.free(address, FrameCount::new(1)); + } } unsafe fn usage(&self) -> FrameUsage; @@ -64,18 +66,18 @@ where T: FrameAllocator, { unsafe fn allocate(&mut self, count: FrameCount) -> Option { - T::allocate(self, count) + unsafe { T::allocate(self, count) } } unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) { - T::free(self, address, count) + unsafe { T::free(self, address, count) } } unsafe fn allocate_one(&mut self) -> Option { - T::allocate_one(self) + unsafe { T::allocate_one(self) } } unsafe fn free_one(&mut self, address: PhysicalAddress) { - T::free_one(self, address) + unsafe { T::free_one(self, address) } } unsafe fn usage(&self) -> FrameUsage { - T::usage(self) + unsafe { T::usage(self) } } } diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 687294d3b5..118ac956ea 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -42,51 +42,59 @@ impl Arch for AArch64Arch { #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - asm!(" + unsafe { + asm!(" dsb ishst tlbi vaae1is, {} dsb ish isb ", in(reg) (address.data() >> Self::PAGE_SHIFT)); + } } #[inline(always)] unsafe fn invalidate_all() { - asm!( - " + unsafe { + asm!( + " dsb ishst tlbi vmalle1is dsb ish isb " - ); + ); + } } #[inline(always)] unsafe fn table(table_kind: TableKind) -> PhysicalAddress { - let address: usize; - match table_kind { - TableKind::User => { - asm!("mrs {0}, ttbr0_el1", out(reg) address); - } - TableKind::Kernel => { - asm!("mrs {0}, ttbr1_el1", out(reg) address); + unsafe { + let address: usize; + match table_kind { + TableKind::User => { + asm!("mrs {0}, ttbr0_el1", out(reg) address); + } + TableKind::Kernel => { + asm!("mrs {0}, ttbr1_el1", out(reg) address); + } } + PhysicalAddress::new(address) } - PhysicalAddress::new(address) } #[inline(always)] unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress) { - match table_kind { - TableKind::User => { - asm!("msr ttbr0_el1, {0}", in(reg) address.data()); - } - TableKind::Kernel => { - asm!("msr ttbr1_el1, {0}", in(reg) address.data()); + unsafe { + match table_kind { + TableKind::User => { + asm!("msr ttbr0_el1, {0}", in(reg) address.data()); + } + TableKind::Kernel => { + asm!("msr ttbr1_el1, {0}", in(reg) address.data()); + } } + Self::invalidate_all(); } - Self::invalidate_all(); } fn virt_is_valid(_address: VirtualAddress) -> bool { diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 3e381db676..f6f33cd171 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -34,76 +34,84 @@ impl Arch for EmulateArch { const ENTRY_FLAG_WRITE_COMBINING: usize = X8664Arch::ENTRY_FLAG_WRITE_COMBINING; unsafe fn init() -> &'static [MemoryArea] { - // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) - let mut machine = Machine::new(MEMORY_SIZE); + unsafe { + // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) + let mut machine = Machine::new(MEMORY_SIZE); - // PML4 index 256 (PHYS_OFFSET) link to PDP - let pml4 = 0; - let pdp = pml4 + Self::PAGE_SIZE; - let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT; - machine.write_phys::( - PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), - pdp | flags, - ); - - // PDP link to PD - let pd = pdp + Self::PAGE_SIZE; - machine.write_phys::(PhysicalAddress::new(pdp), pd | flags); - - // PD link to PT - let pt = pd + Self::PAGE_SIZE; - machine.write_phys::(PhysicalAddress::new(pd), pt | flags); - - // PT links to frames - for i in 0..Self::PAGE_ENTRIES { - let page = i * Self::PAGE_SIZE; + // PML4 index 256 (PHYS_OFFSET) link to PDP + let pml4 = 0; + let pdp = pml4 + Self::PAGE_SIZE; + let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT; machine.write_phys::( - PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), - page | flags, + PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE), + pdp | flags, ); + + // PDP link to PD + let pd = pdp + Self::PAGE_SIZE; + machine.write_phys::(PhysicalAddress::new(pdp), pd | flags); + + // PD link to PT + let pt = pd + Self::PAGE_SIZE; + machine.write_phys::(PhysicalAddress::new(pd), pt | flags); + + // PT links to frames + for i in 0..Self::PAGE_ENTRIES { + let page = i * Self::PAGE_SIZE; + machine.write_phys::( + PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE), + page | flags, + ); + } + + MACHINE = Some(machine); + + // Set table to pml4 + EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4)); + + &MEMORY_AREAS } - - MACHINE = Some(machine); - - // Set table to pml4 - EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4)); - - &MEMORY_AREAS } #[inline(always)] unsafe fn read(address: VirtualAddress) -> T { - MACHINE.as_ref().unwrap().read(address) + unsafe { MACHINE.as_ref().unwrap().read(address) } } #[inline(always)] unsafe fn write(address: VirtualAddress, value: T) { - MACHINE.as_mut().unwrap().write(address, value) + unsafe { MACHINE.as_mut().unwrap().write(address, value) } } #[inline(always)] unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { - MACHINE.as_mut().unwrap().write_bytes(address, value, count) + unsafe { MACHINE.as_mut().unwrap().write_bytes(address, value, count) } } #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - MACHINE.as_mut().unwrap().invalidate(address); + unsafe { + MACHINE.as_mut().unwrap().invalidate(address); + } } #[inline(always)] unsafe fn invalidate_all() { - MACHINE.as_mut().unwrap().invalidate_all(); + unsafe { + MACHINE.as_mut().unwrap().invalidate_all(); + } } #[inline(always)] unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { - MACHINE.as_mut().unwrap().get_table() + unsafe { MACHINE.as_mut().unwrap().get_table() } } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - MACHINE.as_mut().unwrap().set_table(address); + unsafe { + MACHINE.as_mut().unwrap().set_table(address); + } } fn virt_is_valid(_address: VirtualAddress) -> bool { // TODO: Don't see why an emulated arch would have any problems with canonicalness... diff --git a/src/arch/mod.rs b/src/arch/mod.rs index cd68f09f2c..a5f92a80bc 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -65,25 +65,27 @@ pub trait Arch: Clone + Copy { #[inline(always)] unsafe fn read(address: VirtualAddress) -> T { - ptr::read(address.data() as *const T) + unsafe { ptr::read(address.data() as *const T) } } #[inline(always)] unsafe fn write(address: VirtualAddress, value: T) { - ptr::write(address.data() as *mut T, value) + unsafe { ptr::write(address.data() as *mut T, value) } } #[inline(always)] unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { - ptr::write_bytes(address.data() as *mut u8, value, count) + unsafe { ptr::write_bytes(address.data() as *mut u8, value, count) } } unsafe fn invalidate(address: VirtualAddress); #[inline(always)] unsafe fn invalidate_all() { - //TODO: this stub only works on x86_64, maybe make the arch implement this? - Self::set_table(TableKind::User, Self::table(TableKind::User)); + unsafe { + //TODO: this stub only works on x86_64, maybe make the arch implement this? + Self::set_table(TableKind::User, Self::table(TableKind::User)); + } } unsafe fn table(table_kind: TableKind) -> PhysicalAddress; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 31ade20c1e..5948d88da6 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -34,29 +34,37 @@ impl Arch for RiscV64Sv39Arch { #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - asm!("sfence.vma {}", in(reg) address.data()); + unsafe { + asm!("sfence.vma {}", in(reg) address.data()); + } } #[inline(always)] unsafe fn invalidate_all() { - asm!("sfence.vma"); + unsafe { + asm!("sfence.vma"); + } } #[inline(always)] unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { - let satp: usize; - asm!("csrr {0}, satp", out(reg) satp); - PhysicalAddress::new( - (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN - ) + unsafe { + let satp: usize; + asm!("csrr {0}, satp", out(reg) satp); + PhysicalAddress::new( + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN + ) + } } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - let satp = (8 << 60) | // Sv39 MODE + unsafe { + let satp = (8 << 60) | // Sv39 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) - asm!("csrw satp, {0}", in(reg) satp); - Self::invalidate_all(); + asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); + } } fn virt_is_valid(address: VirtualAddress) -> bool { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 7e80ad0c3a..8d6e2faf0c 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -34,29 +34,37 @@ impl Arch for RiscV64Sv48Arch { #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - asm!("sfence.vma {}", in(reg) address.data()); + unsafe { + asm!("sfence.vma {}", in(reg) address.data()); + } } #[inline(always)] unsafe fn invalidate_all() { - asm!("sfence.vma"); + unsafe { + asm!("sfence.vma"); + } } #[inline(always)] unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { - let satp: usize; - asm!("csrr {0}, satp", out(reg) satp); - PhysicalAddress::new( - (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN - ) + unsafe { + let satp: usize; + asm!("csrr {0}, satp", out(reg) satp); + PhysicalAddress::new( + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN + ) + } } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - let satp = (9 << 60) | // Sv48 MODE + unsafe { + let satp = (9 << 60) | // Sv48 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) - asm!("csrw satp, {0}", in(reg) satp); - Self::invalidate_all(); + asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); + } } fn virt_is_valid(address: VirtualAddress) -> bool { diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 3b0bacff0d..6bc5022388 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -32,19 +32,25 @@ impl Arch for X8664Arch { #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - asm!("invlpg [{0}]", in(reg) address.data()); + unsafe { + asm!("invlpg [{0}]", in(reg) address.data()); + } } #[inline(always)] unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { - let address: usize; - asm!("mov {0}, cr3", out(reg) address); - PhysicalAddress::new(address) + unsafe { + let address: usize; + asm!("mov {0}, cr3", out(reg) address); + PhysicalAddress::new(address) + } } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - asm!("mov cr3, {0}", in(reg) address.data()); + unsafe { + asm!("mov cr3, {0}", in(reg) address.data()); + } } fn virt_is_valid(address: VirtualAddress) -> bool { diff --git a/src/main.rs b/src/main.rs index 86f5c67b60..a63c24f305 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,22 +23,24 @@ pub fn format_size(size: usize) -> String { #[allow(dead_code)] unsafe fn dump_tables(table: PageTable) { - let level = table.level(); - for i in 0..A::PAGE_ENTRIES { - if level == 0 { - if let Some(entry) = table.entry(i) { - if entry.present() { - let base = table.entry_base(i).unwrap(); - println!( - "0x{:X}: 0x{:X}", - base.data(), - entry.address().unwrap().data() - ); + unsafe { + let level = table.level(); + for i in 0..A::PAGE_ENTRIES { + if level == 0 { + if let Some(entry) = table.entry(i) { + if entry.present() { + let base = table.entry_base(i).unwrap(); + println!( + "0x{:X}: 0x{:X}", + base.data(), + entry.address().unwrap().data() + ); + } + } + } else { + if let Some(next) = table.next(i) { + dump_tables(next); } - } - } else { - if let Some(next) = table.next(i) { - dump_tables(next); } } } @@ -64,21 +66,25 @@ impl SlabNode { } pub unsafe fn insert(&mut self, phys: PhysicalAddress) { - let virt = A::phys_to_virt(phys); - A::write(virt, self.next); - self.next = phys; - self.count += 1; + unsafe { + let virt = A::phys_to_virt(phys); + A::write(virt, self.next); + self.next = phys; + self.count += 1; + } } pub unsafe fn remove(&mut self) -> Option { - if self.count > 0 { - let phys = self.next; - let virt = A::phys_to_virt(phys); - self.next = A::read(virt); - self.count -= 1; - Some(phys) - } else { - None + unsafe { + if self.count > 0 { + let phys = self.next; + let virt = A::phys_to_virt(phys); + self.next = A::read(virt); + self.count -= 1; + Some(phys) + } else { + None + } } } } @@ -91,57 +97,63 @@ pub struct SlabAllocator { impl SlabAllocator { pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Self { - let mut allocator = Self { - nodes: [ - SlabNode::empty(), - SlabNode::empty(), - SlabNode::empty(), - SlabNode::empty(), - ], - phantom: PhantomData, - }; + unsafe { + let mut allocator = Self { + nodes: [ + SlabNode::empty(), + SlabNode::empty(), + SlabNode::empty(), + SlabNode::empty(), + ], + phantom: PhantomData, + }; - // Add unused areas to free lists - let mut area_offset = offset; - for area in areas.iter() { - if area_offset < area.size { - let area_base = area.base.add(area_offset); - let area_size = area.size - area_offset; - allocator.free(area_base, area_size); - area_offset = 0; - } else { - area_offset -= area.size; + // Add unused areas to free lists + let mut area_offset = offset; + for area in areas.iter() { + if area_offset < area.size { + let area_base = area.base.add(area_offset); + let area_size = area.size - area_offset; + allocator.free(area_base, area_size); + area_offset = 0; + } else { + area_offset -= area.size; + } } - } - allocator + allocator + } } pub unsafe fn allocate(&mut self, size: usize) -> Option { - for level in 0..A::PAGE_LEVELS - 1 { - let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; - let level_size = 1 << level_shift; - if size <= level_size { - if let Some(base) = self.nodes[level].remove() { - self.free(base.add(size), level_size - size); - return Some(base); + unsafe { + for level in 0..A::PAGE_LEVELS - 1 { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + if size <= level_size { + if let Some(base) = self.nodes[level].remove() { + self.free(base.add(size), level_size - size); + return Some(base); + } } } + None } - None } //TODO: This causes fragmentation, since neighbors are not identified //TODO: remainders less than PAGE_SIZE will be lost pub unsafe fn free(&mut self, mut base: PhysicalAddress, mut size: usize) { - for level in (0..A::PAGE_LEVELS - 1).rev() { - let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; - let level_size = 1 << level_shift; - while size >= level_size { - println!("Add {:X} {}", base.data(), format_size(level_size)); - self.nodes[level].insert(base); - base = base.add(level_size); - size -= level_size; + unsafe { + for level in (0..A::PAGE_LEVELS - 1).rev() { + let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT; + let level_size = 1 << level_shift; + while size >= level_size { + println!("Add {:X} {}", base.data(), format_size(level_size)); + self.nodes[level].insert(base); + base = base.add(level_size); + size -= level_size; + } } } } @@ -158,133 +170,137 @@ impl SlabAllocator { } unsafe fn new_tables(areas: &'static [MemoryArea]) { - // First, calculate how much memory we have - let mut size = 0; - for area in areas.iter() { - size += area.size; - } - - println!("Memory: {}", format_size(size)); - - // Create a basic allocator for the first pages - let mut bump_allocator = BumpAllocator::::new(areas, 0); - - { - // Map all physical areas at PHYS_OFFSET - let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) - .expect("failed to create Mapper"); + unsafe { + // First, calculate how much memory we have + let mut size = 0; for area in areas.iter() { - for i in 0..area.size / A::PAGE_SIZE { - let phys = area.base.add(i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flush = mapper - .map_phys(virt, phys, PageFlags::::new().write(true)) - .expect("failed to map page to frame"); - flush.ignore(); // Not the active table - } + size += area.size; } - // Use the new table - mapper.make_current(); - } + println!("Memory: {}", format_size(size)); - // Create the physical memory map - let offset = bump_allocator.offset(); - println!("Permanently used: {}", format_size(offset)); + // Create a basic allocator for the first pages + let mut bump_allocator = BumpAllocator::::new(areas, 0); - let mut allocator = BuddyAllocator::::new(bump_allocator).unwrap(); - - for i in 0..16 { { - let phys_opt = allocator.allocate_one(); - println!("page {}: {:X?}", i, phys_opt); - if i % 3 == 0 { - if let Some(phys) = phys_opt { - println!("free {}: {:X?}", i, phys_opt); - allocator.free_one(phys); + // Map all physical areas at PHYS_OFFSET + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); + for area in areas.iter() { + for i in 0..area.size / A::PAGE_SIZE { + let phys = area.base.add(i * A::PAGE_SIZE); + let virt = A::phys_to_virt(phys); + let flush = mapper + .map_phys(virt, phys, PageFlags::::new().write(true)) + .expect("failed to map page to frame"); + flush.ignore(); // Not the active table + } + } + + // Use the new table + mapper.make_current(); + } + + // Create the physical memory map + let offset = bump_allocator.offset(); + println!("Permanently used: {}", format_size(offset)); + + let mut allocator = BuddyAllocator::::new(bump_allocator).unwrap(); + + for i in 0..16 { + { + let phys_opt = allocator.allocate_one(); + println!("page {}: {:X?}", i, phys_opt); + if i % 3 == 0 { + if let Some(phys) = phys_opt { + println!("free {}: {:X?}", i, phys_opt); + allocator.free_one(phys); + } + } + } + + { + let phys_opt = allocator.allocate(FrameCount::new(16)); + println!("page*16 {}: {:X?}", i, phys_opt); + if i % 2 == 0 { + if let Some(phys) = phys_opt { + println!("free*16 {}: {:X?}", i, phys_opt); + allocator.free(phys, FrameCount::new(16)); + } } } } - { - let phys_opt = allocator.allocate(FrameCount::new(16)); - println!("page*16 {}: {:X?}", i, phys_opt); - if i % 2 == 0 { - if let Some(phys) = phys_opt { - println!("free*16 {}: {:X?}", i, phys_opt); - allocator.free(phys, FrameCount::new(16)); - } - } + let mut mapper = PageMapper::::current(TableKind::Kernel, &mut allocator); + let mut flush_all = PageFlushAll::new(); + for i in 0..16 { + let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); + let flush = mapper + .map(virt, PageFlags::::new().user(true).write(true)) + .expect("failed to map page"); + flush_all.consume(flush); } - } + flush_all.flush(); - let mut mapper = PageMapper::::current(TableKind::Kernel, &mut allocator); - let mut flush_all = PageFlushAll::new(); - for i in 0..16 { - let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); - let flush = mapper - .map(virt, PageFlags::::new().user(true).write(true)) - .expect("failed to map page"); - flush_all.consume(flush); - } - flush_all.flush(); + let mut flush_all = PageFlushAll::new(); + for i in 0..16 { + let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); + let flush = mapper.unmap(virt, false).expect("failed to unmap page"); + flush_all.consume(flush); + } + flush_all.flush(); - let mut flush_all = PageFlushAll::new(); - for i in 0..16 { - let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE); - let flush = mapper.unmap(virt, false).expect("failed to unmap page"); - flush_all.consume(flush); + let usage = allocator.usage(); + println!("Allocator usage:"); + println!( + " Used: {}", + format_size(usage.used().data() * A::PAGE_SIZE) + ); + println!( + " Free: {}", + format_size(usage.free().data() * A::PAGE_SIZE) + ); + println!( + " Total: {}", + format_size(usage.total().data() * A::PAGE_SIZE) + ); } - flush_all.flush(); - - let usage = allocator.usage(); - println!("Allocator usage:"); - println!( - " Used: {}", - format_size(usage.used().data() * A::PAGE_SIZE) - ); - println!( - " Free: {}", - format_size(usage.free().data() * A::PAGE_SIZE) - ); - println!( - " Total: {}", - format_size(usage.total().data() * A::PAGE_SIZE) - ); } unsafe fn inner() { - let areas = A::init(); + unsafe { + let areas = A::init(); - // Debug table - //dump_tables(PageTable::::top()); + // Debug table + //dump_tables(PageTable::::top()); - new_tables::(areas); + new_tables::(areas); - //dump_tables(PageTable::::top()); + //dump_tables(PageTable::::top()); - for i in &[1, 2, 4, 8, 16, 32] { - let phys = PhysicalAddress::new(i * MEGABYTE); - let virt = A::phys_to_virt(phys); + for i in &[1, 2, 4, 8, 16, 32] { + let phys = PhysicalAddress::new(i * MEGABYTE); + let virt = A::phys_to_virt(phys); - // Test read - println!( - "0x{:X} (0x{:X}) = 0x{:X}", - virt.data(), - phys.data(), - A::read::(virt) - ); + // Test read + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); - // Test write - A::write::(virt, 0x5A); + // Test write + A::write::(virt, 0x5A); - // Test read - println!( - "0x{:X} (0x{:X}) = 0x{:X}", - virt.data(), - phys.data(), - A::read::(virt) - ); + // Test read + println!( + "0x{:X} (0x{:X}) = 0x{:X}", + virt.data(), + phys.data(), + A::read::(virt) + ); + } } } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index a6aa726be8..27e2b936e7 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -23,13 +23,17 @@ impl PageMapper { } pub unsafe fn create(table_kind: TableKind, mut allocator: F) -> Option { - let table_addr = allocator.allocate_one()?; - Some(Self::new(table_kind, table_addr, allocator)) + unsafe { + let table_addr = allocator.allocate_one()?; + Some(Self::new(table_kind, table_addr, allocator)) + } } pub unsafe fn current(table_kind: TableKind, allocator: F) -> Self { - let table_addr = A::table(table_kind); - Self::new(table_kind, table_addr, allocator) + unsafe { + let table_addr = A::table(table_kind); + Self::new(table_kind, table_addr, allocator) + } } pub fn is_current(&self) -> bool { @@ -37,7 +41,9 @@ impl PageMapper { } pub unsafe fn make_current(&self) { - A::set_table(self.table_kind, self.table_addr); + unsafe { + A::set_table(self.table_kind, self.table_addr); + } } pub fn table(&self) -> PageTable { @@ -59,33 +65,37 @@ impl PageMapper { virt: VirtualAddress, f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags), ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { - self.visit(virt, |p1, i| { - let old_entry = p1.entry(i)?; - let old_phys = old_entry.address().ok()?; - let old_flags = old_entry.flags(); - let (new_phys, new_flags) = f(old_phys, old_flags); - // TODO: Higher-level PageEntry::new interface? - let new_entry = PageEntry::new(new_phys.data(), new_flags.data()); - p1.set_entry(i, new_entry); - Some((old_flags, old_phys, PageFlush::new(virt))) - }) - .flatten() + unsafe { + self.visit(virt, |p1, i| { + let old_entry = p1.entry(i)?; + let old_phys = old_entry.address().ok()?; + let old_flags = old_entry.flags(); + let (new_phys, new_flags) = f(old_phys, old_flags); + // TODO: Higher-level PageEntry::new interface? + let new_entry = PageEntry::new(new_phys.data(), new_flags.data()); + p1.set_entry(i, new_entry); + Some((old_flags, old_phys, PageFlush::new(virt))) + }) + .flatten() + } } pub unsafe fn remap_with( &mut self, virt: VirtualAddress, map_flags: impl FnOnce(PageFlags) -> PageFlags, ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { - self.remap_with_full(virt, |same_phys, old_flags| { - (same_phys, map_flags(old_flags)) - }) + unsafe { + self.remap_with_full(virt, |same_phys, old_flags| { + (same_phys, map_flags(old_flags)) + }) + } } pub unsafe fn remap( &mut self, virt: VirtualAddress, flags: PageFlags, ) -> Option> { - self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) + unsafe { self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) } } pub unsafe fn map( @@ -93,8 +103,10 @@ impl PageMapper { virt: VirtualAddress, flags: PageFlags, ) -> Option> { - let phys = self.allocator.allocate_one()?; - self.map_phys(virt, phys, flags) + unsafe { + let phys = self.allocator.allocate_one()?; + self.map_phys(virt, phys, flags) + } } pub unsafe fn map_phys( @@ -103,34 +115,36 @@ impl PageMapper { phys: PhysicalAddress, flags: PageFlags, ) -> Option> { - //TODO: verify virt and phys are aligned - //TODO: verify flags have correct bits - let entry = PageEntry::new(phys.data(), flags.data()); - let mut table = self.table(); - loop { - let i = table.index_of(virt)?; - if table.level() == 0 { - //TODO: check for overwriting entry - table.set_entry(i, entry); - return Some(PageFlush::new(virt)); - } else { - let next_opt = table.next(i); - let next = match next_opt { - Some(some) => some, - None => { - let next_phys = self.allocator.allocate_one()?; - //TODO: correct flags? - let flags = A::ENTRY_FLAG_DEFAULT_TABLE - | if virt.kind() == TableKind::User { - A::ENTRY_FLAG_TABLE_USER - } else { - 0 - }; - table.set_entry(i, PageEntry::new(next_phys.data(), flags)); - table.next(i)? - } - }; - table = next; + unsafe { + //TODO: verify virt and phys are aligned + //TODO: verify flags have correct bits + let entry = PageEntry::new(phys.data(), flags.data()); + let mut table = self.table(); + loop { + let i = table.index_of(virt)?; + if table.level() == 0 { + //TODO: check for overwriting entry + table.set_entry(i, entry); + return Some(PageFlush::new(virt)); + } else { + let next_opt = table.next(i); + let next = match next_opt { + Some(some) => some, + None => { + let next_phys = self.allocator.allocate_one()?; + //TODO: correct flags? + let flags = A::ENTRY_FLAG_DEFAULT_TABLE + | if virt.kind() == TableKind::User { + A::ENTRY_FLAG_TABLE_USER + } else { + 0 + }; + table.set_entry(i, PageEntry::new(next_phys.data(), flags)); + table.next(i)? + } + }; + table = next; + } } } } @@ -139,8 +153,10 @@ impl PageMapper { phys: PhysicalAddress, flags: PageFlags, ) -> Option<(VirtualAddress, PageFlush)> { - let virt = A::phys_to_virt(phys); - self.map_phys(virt, phys, flags).map(|flush| (virt, flush)) + unsafe { + let virt = A::phys_to_virt(phys); + self.map_phys(virt, phys, flags).map(|flush| (virt, flush)) + } } fn visit( &self, @@ -169,9 +185,11 @@ impl PageMapper { virt: VirtualAddress, unmap_parents: bool, ) -> Option> { - let (old, _, flush) = self.unmap_phys(virt, unmap_parents)?; - self.allocator.free_one(old); - Some(flush) + unsafe { + let (old, _, flush) = self.unmap_phys(virt, unmap_parents)?; + self.allocator.free_one(old); + Some(flush) + } } pub unsafe fn unmap_phys( @@ -179,11 +197,13 @@ impl PageMapper { virt: VirtualAddress, unmap_parents: bool, ) -> Option<(PhysicalAddress, PageFlags, PageFlush)> { - //TODO: verify virt is aligned - let mut table = self.table(); - let level = table.level(); - unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator) - .map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + unsafe { + //TODO: verify virt is aligned + let mut table = self.table(); + let level = table.level(); + unmap_phys_inner(virt, &mut table, level, unmap_parents, &mut self.allocator) + .map(|(pa, pf)| (pa, pf, PageFlush::new(virt))) + } } } unsafe fn unmap_phys_inner( @@ -193,35 +213,38 @@ unsafe fn unmap_phys_inner( unmap_parents: bool, allocator: &mut impl FrameAllocator, ) -> Option<(PhysicalAddress, PageFlags)> { - let i = table.index_of(virt)?; + unsafe { + let i = table.index_of(virt)?; - if table.level() == 0 { - let entry_opt = table.entry(i); - table.set_entry(i, PageEntry::new(0, 0)); - let entry = entry_opt?; + if table.level() == 0 { + let entry_opt = table.entry(i); + table.set_entry(i, PageEntry::new(0, 0)); + let entry = entry_opt?; - Some((entry.address().ok()?, entry.flags())) - } else { - let mut subtable = table.next(i)?; + Some((entry.address().ok()?, entry.flags())) + } else { + let mut subtable = table.next(i)?; - let res = unmap_phys_inner(virt, &mut subtable, initial_level, unmap_parents, allocator)?; + let res = + unmap_phys_inner(virt, &mut subtable, initial_level, unmap_parents, allocator)?; - //TODO: This is a bad idea for architectures where the kernel mappings are done in the process tables, - // as these mappings may become out of sync - if unmap_parents { - // TODO: Use a counter? This would reduce the remaining number of available bits, but could be - // faster (benchmark is needed). - let is_still_populated = (0..A::PAGE_ENTRIES) - .map(|j| subtable.entry(j).expect("must be within bounds")) - .any(|e| e.present()); + //TODO: This is a bad idea for architectures where the kernel mappings are done in the process tables, + // as these mappings may become out of sync + if unmap_parents { + // TODO: Use a counter? This would reduce the remaining number of available bits, but could be + // faster (benchmark is needed). + let is_still_populated = (0..A::PAGE_ENTRIES) + .map(|j| subtable.entry(j).expect("must be within bounds")) + .any(|e| e.present()); - if !is_still_populated { - allocator.free_one(subtable.phys()); - table.set_entry(i, PageEntry::new(0, 0)); + if !is_still_populated { + allocator.free_one(subtable.phys()); + table.set_entry(i, PageEntry::new(0, 0)); + } } - } - Some(res) + Some(res) + } } } impl core::fmt::Debug for PageMapper { diff --git a/src/page/table.rs b/src/page/table.rs index 70d281543d..c6d377a28e 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -21,11 +21,13 @@ impl PageTable { } pub unsafe fn top(table_kind: TableKind) -> Self { - Self::new( - VirtualAddress::new(0), - A::table(table_kind), - A::PAGE_LEVELS - 1, - ) + unsafe { + Self::new( + VirtualAddress::new(0), + A::table(table_kind), + A::PAGE_LEVELS - 1, + ) + } } pub fn base(&self) -> VirtualAddress { @@ -41,16 +43,18 @@ impl PageTable { } pub unsafe fn virt(&self) -> VirtualAddress { - A::phys_to_virt(self.phys) + unsafe { + A::phys_to_virt(self.phys) - // Recursive mapping - // let mut addr = 0xFFFF_FFFF_FFFF_F000; - // for level in (self.level + 1 .. A::PAGE_LEVELS).rev() { - // let index = (self.base.0 >> (level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) & A::PAGE_ENTRY_MASK; - // addr <<= A::PAGE_ENTRY_SHIFT; - // addr |= index << A::PAGE_SHIFT; - // } - // VirtualAddress::new(addr) + // Recursive mapping + // let mut addr = 0xFFFF_FFFF_FFFF_F000; + // for level in (self.level + 1 .. A::PAGE_LEVELS).rev() { + // let index = (self.base.0 >> (level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT)) & A::PAGE_ENTRY_MASK; + // addr <<= A::PAGE_ENTRY_SHIFT; + // addr |= index << A::PAGE_SHIFT; + // } + // VirtualAddress::new(addr) + } } pub fn entry_base(&self, i: usize) -> Option { @@ -63,22 +67,28 @@ impl PageTable { } pub unsafe fn entry_virt(&self, i: usize) -> Option { - if i < A::PAGE_ENTRIES { - Some(self.virt().add(i * A::PAGE_ENTRY_SIZE)) - } else { - None + unsafe { + if i < A::PAGE_ENTRIES { + Some(self.virt().add(i * A::PAGE_ENTRY_SIZE)) + } else { + None + } } } pub unsafe fn entry(&self, i: usize) -> Option> { - let addr = self.entry_virt(i)?; - Some(PageEntry::from_data(A::read::(addr))) + unsafe { + let addr = self.entry_virt(i)?; + Some(PageEntry::from_data(A::read::(addr))) + } } pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { - let addr = self.entry_virt(i)?; - A::write::(addr, entry.data()); - Some(()) + unsafe { + let addr = self.entry_virt(i)?; + A::write::(addr, entry.data()); + Some(()) + } } pub unsafe fn index_of(&self, address: VirtualAddress) -> Option { @@ -98,14 +108,16 @@ impl PageTable { } pub unsafe fn next(&self, i: usize) -> Option { - if self.level == 0 { - return None; - } + unsafe { + if self.level == 0 { + return None; + } - Some(PageTable::new( - self.entry_base(i)?, - self.entry(i)?.address().ok()?, - self.level - 1, - )) + Some(PageTable::new( + self.entry_base(i)?, + self.entry(i)?.address().ok()?, + self.level - 1, + )) + } } } From e371f47f51ad3bdc53c9cbb1150b4d660d186efe Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 25 Oct 2025 19:59:27 +0200 Subject: [PATCH 113/120] Remove let_chain feature gate It is now stable. --- src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e4dc8ff060..b0615774d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ #![cfg_attr(not(feature = "std"), no_std)] #![feature(doc_cfg)] -#![feature(let_chains)] pub use crate::{allocator::*, arch::*, page::*}; From 5b5bbe5643906ff4e84aaf8b2ad0b241fe98013d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 25 Oct 2025 20:01:28 +0200 Subject: [PATCH 114/120] Remove VirtualAddress::is_canonical This way the last remaining feature gate can be removed. It also matches how other architectures handle this functionality. --- src/arch/x86_64.rs | 13 +++---------- src/lib.rs | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 6bc5022388..d8b44357cc 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -57,14 +57,7 @@ impl Arch for X8664Arch { // On x86_64, an address is valid if and only if it is canonical. It may still point to // unmapped memory, but will always be valid once translated via the page table has // suceeded. - address.is_canonical() - } -} - -impl VirtualAddress { - #[doc(cfg(target_arch = "x86_64"))] - pub fn is_canonical(self) -> bool { - let masked = self.data() & 0xFFFF_8000_0000_0000; + let masked = address.data() & 0xFFFF_8000_0000_0000; // TODO: 5-level paging masked == 0xFFFF_8000_0000_0000 || masked == 0 } @@ -96,10 +89,10 @@ mod tests { #[test] fn is_canonical() { fn yes(address: usize) { - assert!(VirtualAddress::new(address).is_canonical()); + assert!(X8664Arch::virt_is_valid(VirtualAddress::new(address))); } fn no(address: usize) { - assert!(!VirtualAddress::new(address).is_canonical()); + assert!(!X8664Arch::virt_is_valid(VirtualAddress::new(address))); } yes(0xFFFF_8000_1337_1337); diff --git a/src/lib.rs b/src/lib.rs index b0615774d7..0cc0beb2d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,4 @@ #![cfg_attr(not(feature = "std"), no_std)] -#![feature(doc_cfg)] pub use crate::{allocator::*, arch::*, page::*}; From e6d42dda1918b891f0584acf8d1f4711cba3ff8f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:55:42 +0100 Subject: [PATCH 115/120] Fix compilation of EmulateArch --- src/arch/emulate.rs | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index f6f33cd171..5e8bdb3c2a 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -1,5 +1,4 @@ -use core::{marker::PhantomData, mem, ptr}; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, marker::PhantomData, mem, ptr, sync::Mutex}; use crate::{ arch::x86_64::X8664Arch, page::PageFlags, Arch, MemoryArea, PageEntry, PhysicalAddress, @@ -64,7 +63,7 @@ impl Arch for EmulateArch { ); } - MACHINE = Some(machine); + *MACHINE.lock().unwrap() = Some(machine); // Set table to pml4 EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4)); @@ -75,43 +74,52 @@ impl Arch for EmulateArch { #[inline(always)] unsafe fn read(address: VirtualAddress) -> T { - unsafe { MACHINE.as_ref().unwrap().read(address) } + MACHINE.lock().unwrap().as_ref().unwrap().read(address) } #[inline(always)] unsafe fn write(address: VirtualAddress, value: T) { - unsafe { MACHINE.as_mut().unwrap().write(address, value) } + MACHINE + .lock() + .unwrap() + .as_mut() + .unwrap() + .write(address, value) } #[inline(always)] unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) { - unsafe { MACHINE.as_mut().unwrap().write_bytes(address, value, count) } + MACHINE + .lock() + .unwrap() + .as_mut() + .unwrap() + .write_bytes(address, value, count) } #[inline(always)] unsafe fn invalidate(address: VirtualAddress) { - unsafe { - MACHINE.as_mut().unwrap().invalidate(address); - } + MACHINE + .lock() + .unwrap() + .as_mut() + .unwrap() + .invalidate(address); } #[inline(always)] unsafe fn invalidate_all() { - unsafe { - MACHINE.as_mut().unwrap().invalidate_all(); - } + MACHINE.lock().unwrap().as_mut().unwrap().invalidate_all(); } #[inline(always)] unsafe fn table(_table_kind: TableKind) -> PhysicalAddress { - unsafe { MACHINE.as_mut().unwrap().get_table() } + MACHINE.lock().unwrap().as_mut().unwrap().get_table() } #[inline(always)] unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) { - unsafe { - MACHINE.as_mut().unwrap().set_table(address); - } + MACHINE.lock().unwrap().as_mut().unwrap().set_table(address); } fn virt_is_valid(_address: VirtualAddress) -> bool { // TODO: Don't see why an emulated arch would have any problems with canonicalness... @@ -132,7 +140,7 @@ static MEMORY_AREAS: [MemoryArea; 2] = [ }, ]; -static mut MACHINE: Option> = None; +static MACHINE: Mutex>> = Mutex::new(None); struct Machine { memory: Box<[u8]>, From 075d52f1534dbbdeed0bf49822b19d06deda4a74 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:58:43 +0100 Subject: [PATCH 116/120] Fix example binary --- src/allocator/frame/bump.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allocator/frame/bump.rs b/src/allocator/frame/bump.rs index 9985feff3a..9043b181ae 100644 --- a/src/allocator/frame/bump.rs +++ b/src/allocator/frame/bump.rs @@ -25,7 +25,7 @@ impl BumpAllocator { } } pub fn areas(&self) -> &'static [MemoryArea] { - todo!() + self.orig_areas.0 } /// Returns one semifree and the fully free areas. The offset is the number of bytes after /// which the first area is free. From 409a1c02f5a6ff965093d27f7c9dc69e351436a5 Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Wed, 12 Nov 2025 07:36:54 -0700 Subject: [PATCH 117/120] Derive Hash for PhysicalAddress --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0cc0beb2d7..521692b6d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,7 @@ pub enum TableKind { } /// Physical memory address -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct PhysicalAddress(usize); From 5b149b7fe46df57cee02edce1b412d8be4906843 Mon Sep 17 00:00:00 2001 From: aarch <126242-aarch@users.noreply.gitlab.redox-os.org> Date: Tue, 9 Dec 2025 13:28:44 +0000 Subject: [PATCH 118/120] Set RISCV MMU marker flags by default --- src/arch/riscv64/sv39.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 5948d88da6..79d1a94484 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -5,6 +5,9 @@ use crate::{Arch, MemoryArea, PhysicalAddress, TableKind, VirtualAddress}; #[derive(Clone, Copy)] pub struct RiscV64Sv39Arch; +pub const ACCESSED: usize = 1 << 6; +pub const DIRTY: usize = 1 << 7; + impl Arch for RiscV64Sv39Arch { const PAGE_SHIFT: usize = 12; // 4096 bytes const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each @@ -13,7 +16,8 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_ADDRESS_WIDTH: usize = 44; const ENTRY_ADDRESS_SHIFT: usize = 10; - const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY; + const ENTRY_FLAG_DEFAULT_PAGE: usize = + Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY | ACCESSED | DIRTY; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 1 << 1; From f299c885e64112c05ab21c8e020e7f07a4b6a348 Mon Sep 17 00:00:00 2001 From: Anhad Singh Date: Thu, 19 Feb 2026 19:32:10 +1100 Subject: [PATCH 119/120] feat(lib): implement `Debug` for `PhysicalAddress` Make it consistent with the debug output for `VirtualAddress` Signed-off-by: Anhad Singh --- src/lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 521692b6d3..bb6e38cd7f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,7 @@ pub enum TableKind { } /// Physical memory address -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] #[repr(transparent)] pub struct PhysicalAddress(usize); @@ -44,6 +44,12 @@ impl PhysicalAddress { } } +impl core::fmt::Debug for PhysicalAddress { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[phys {:#0x}]", self.data()) + } +} + /// Virtual memory address #[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] #[repr(transparent)] @@ -74,6 +80,7 @@ impl VirtualAddress { } } } + impl core::fmt::Debug for VirtualAddress { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "[virt {:#0x}]", self.data()) From d82ba37de8d1801cd1bf47f211ca1ddcfcf8201c Mon Sep 17 00:00:00 2001 From: Anhad Singh Date: Thu, 19 Feb 2026 19:39:16 +1100 Subject: [PATCH 120/120] feat(mapper/remap_with_full): conditional remap This commit modifies the transform function (`f`) argument of `remap_with_full` to return an `Option`. This allows the caller to skip remaps based on the previous frame and page flags. It can alternatively be done by first translating the address and then remapping based on that but that would mean we have to walk the page tables twice :| Signed-off-by: Anhad Singh --- src/page/mapper.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 27e2b936e7..b97d21543f 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -63,14 +63,16 @@ impl PageMapper { pub unsafe fn remap_with_full( &mut self, virt: VirtualAddress, - f: impl FnOnce(PhysicalAddress, PageFlags) -> (PhysicalAddress, PageFlags), + f: impl FnOnce(PhysicalAddress, PageFlags) -> Option<(PhysicalAddress, PageFlags)>, ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { unsafe { self.visit(virt, |p1, i| { let old_entry = p1.entry(i)?; let old_phys = old_entry.address().ok()?; let old_flags = old_entry.flags(); - let (new_phys, new_flags) = f(old_phys, old_flags); + let Some((new_phys, new_flags)) = f(old_phys, old_flags) else { + return None; + }; // TODO: Higher-level PageEntry::new interface? let new_entry = PageEntry::new(new_phys.data(), new_flags.data()); p1.set_entry(i, new_entry); @@ -86,7 +88,7 @@ impl PageMapper { ) -> Option<(PageFlags, PhysicalAddress, PageFlush)> { unsafe { self.remap_with_full(virt, |same_phys, old_flags| { - (same_phys, map_flags(old_flags)) + Some((same_phys, map_flags(old_flags))) }) } }