From b7d3acf6069bc33ae4275829ffb7dcd2c9b63e83 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 07:42:21 +0300 Subject: [PATCH 1/5] Separate leaf page and directory pages USER flag This is required for RISC-V. Privileged spec says: > For non-leaf PTEs, the D, A, and U bits are reserved for future standard use. > Until their use is defined by a standard extension, they must be cleared by software > for forward compatibility. QEMU fails address translation if it sees any of these flags set on non-leaf page entry. --- src/arch/aarch64.rs | 2 +- src/arch/emulate.rs | 2 +- src/arch/mod.rs | 3 ++- src/arch/riscv64/sv39.rs | 3 ++- src/arch/riscv64/sv48.rs | 3 ++- src/arch/x86.rs | 2 +- src/arch/x86_64.rs | 2 +- src/page/flags.rs | 4 ++-- src/page/mapper.rs | 2 +- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 3a811a0184..106c102b6e 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -24,7 +24,7 @@ impl Arch for AArch64Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 1 << 7; const ENTRY_FLAG_READWRITE: usize = 0; - const ENTRY_FLAG_USER: usize = 1 << 6; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 6; // This sets both userspace and privileged execute never //TODO: Separate the two? const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53; diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index 88a200769a..c44d72a72d 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -20,7 +20,7 @@ impl Arch for EmulateArch { const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_READONLY: usize = X8664Arch::ENTRY_FLAG_READONLY; const ENTRY_FLAG_READWRITE: usize = X8664Arch::ENTRY_FLAG_READWRITE; - const ENTRY_FLAG_USER: usize = X8664Arch::ENTRY_FLAG_USER; + const ENTRY_FLAG_PAGE_USER: usize = X8664Arch::ENTRY_FLAG_PAGE_USER; const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC; const ENTRY_FLAG_EXEC: usize = X8664Arch::ENTRY_FLAG_EXEC; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f6f5ec9c53..0a3828106d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -36,7 +36,8 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_PRESENT: usize; const ENTRY_FLAG_READONLY: usize; const ENTRY_FLAG_READWRITE: usize; - const ENTRY_FLAG_USER: usize; + const ENTRY_FLAG_PAGE_USER: usize; // Leaf table user page flag + const ENTRY_FLAG_TABLE_USER: usize = Self::ENTRY_FLAG_PAGE_USER; // Directory user page table flag const ENTRY_FLAG_NO_EXEC: usize; const ENTRY_FLAG_EXEC: usize; const ENTRY_FLAG_GLOBAL: usize; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 0980428754..72b99a1e27 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -20,7 +20,8 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; - const ENTRY_FLAG_USER: usize = 1 << 4; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; + const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index b1142317f0..196aee28e3 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -20,7 +20,8 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 2; - const ENTRY_FLAG_USER: usize = 1 << 4; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; + const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 8dcd238f3e..176aef175d 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -17,7 +17,7 @@ impl Arch for X86Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; - const ENTRY_FLAG_USER: usize = 1 << 2; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_GLOBAL: usize = 0; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 113f66f036..ccde251a7c 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -16,7 +16,7 @@ impl Arch for X8664Arch { const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; - const ENTRY_FLAG_USER: usize = 1 << 2; + const ENTRY_FLAG_PAGE_USER: usize = 1 << 2; // Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7; const ENTRY_FLAG_GLOBAL: usize = 1 << 8; const ENTRY_FLAG_NO_GLOBAL: usize = 0; diff --git a/src/page/flags.rs b/src/page/flags.rs index bffe12643e..ca03d1ce0f 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -72,12 +72,12 @@ impl PageFlags { #[must_use] #[inline(always)] pub fn user(self, value: bool) -> Self { - self.custom_flag(A::ENTRY_FLAG_USER, value) + self.custom_flag(A::ENTRY_FLAG_PAGE_USER, value) } #[inline(always)] pub fn has_user(&self) -> bool { - self.has_flag(A::ENTRY_FLAG_USER) + self.has_flag(A::ENTRY_FLAG_PAGE_USER) } #[must_use] diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 28bea52791..5d2584ee63 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -123,7 +123,7 @@ impl PageMapper { let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { - A::ENTRY_FLAG_USER + A::ENTRY_FLAG_TABLE_USER } else { 0 }; From a96ea6f5f38a5f6c71b50857edb0854a34757678 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 08:04:24 +0300 Subject: [PATCH 2/5] Allow physical address bits be anywhere in PTE entry Before this commit, RMM assumed base physical address was presented in PTE as is, i.e. physical page address was shifted exactly to PAGE_SHIFT, so physical address can be extracted from PTE by simply masking off some bits and can be placed in PTE by simple addition/OR. This is not the case for RISC-V which has 4Kb base page so 12 bits PAGE_SHIFT, yet physical page address is only shifted 10 bits in PTE. This commit removes this assumption. NOTE: This commit changes meaning of constants: * ENTRY_ADDRESS_SIZE from "total physical size in bytes" to "total physical size in PAGES" * ENTRY_ADDRESS_MASK from "mask of physical bits in PTE" to "mask of physical bits starting at bit 0" --- src/arch/aarch64.rs | 6 +++--- src/arch/emulate.rs | 11 +++++++---- src/arch/mod.rs | 10 +++++----- src/arch/riscv64/sv39.rs | 23 ++++++++++++----------- src/arch/riscv64/sv48.rs | 15 ++++++++------- src/arch/x86.rs | 6 +++--- src/arch/x86_64.rs | 6 +++--- src/page/entry.rs | 13 +++++++++++-- src/page/mapper.rs | 10 +++++----- src/page/table.rs | 2 +- 10 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 106c102b6e..5999d60a44 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -11,7 +11,7 @@ impl Arch for AArch64Arch { const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 //TODO - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 40; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Page flag | 1 << 10 // Access flag @@ -110,8 +110,8 @@ mod tests { assert_eq!(AArch64Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(AArch64Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); - assert_eq!(AArch64Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(AArch64Arch::ENTRY_ADDRESS_SIZE, 0x0000_0100_0000_0000); + assert_eq!(AArch64Arch::ENTRY_ADDRESS_MASK, 0x0000_00FF_FFFF_FFFF); assert_eq!(AArch64Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); assert_eq!(AArch64Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index c44d72a72d..d228161df0 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -29,6 +29,8 @@ impl Arch for EmulateArch { const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL; const ENTRY_FLAG_NO_GLOBAL: usize = X8664Arch::ENTRY_FLAG_NO_GLOBAL; + const ENTRY_ADDRESS_WIDTH: usize = X8664Arch::ENTRY_ADDRESS_WIDTH; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); @@ -275,7 +277,7 @@ impl Machine { } // Page directory pointer - let a3 = e3 & A::ENTRY_ADDRESS_MASK; + let a3 = ((e3 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; for i3 in 0..A::PAGE_ENTRIES { let e2 = self.read_phys::(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE)); @@ -285,7 +287,7 @@ impl Machine { } // Page directory - let a2 = e2 & A::ENTRY_ADDRESS_MASK; + let a2 = ((e2 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; for i2 in 0..A::PAGE_ENTRIES { let e1 = self.read_phys::(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE)); @@ -295,7 +297,8 @@ impl Machine { } // Page table - let a1 = e1 & A::ENTRY_ADDRESS_MASK; + let a1 = + ((e1 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT; for i1 in 0..A::PAGE_ENTRIES { let e = self .read_phys::(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE)); @@ -308,7 +311,7 @@ impl Machine { let page = (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12); //println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f); self.map - .insert(VirtualAddress::new(page), PageEntry::new(e)); + .insert(VirtualAddress::new(page), PageEntry::from_data(e)); } } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 0a3828106d..1d40b3109c 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -30,7 +30,8 @@ pub trait Arch: Clone + Copy { const PAGE_ENTRY_SHIFT: usize; const PAGE_LEVELS: usize; - const ENTRY_ADDRESS_SHIFT: usize; + const ENTRY_ADDRESS_WIDTH: usize; // Number of bits of physical address in PTE + const ENTRY_ADDRESS_SHIFT: usize = Self::PAGE_SHIFT; // Offset of physical address in PTE const ENTRY_FLAG_DEFAULT_PAGE: usize; const ENTRY_FLAG_DEFAULT_TABLE: usize; const ENTRY_FLAG_PRESENT: usize; @@ -55,10 +56,9 @@ pub trait Arch: Clone + Copy { const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1; const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1) as usize; - const ENTRY_ADDRESS_SIZE: u64 = 1 << Self::ENTRY_ADDRESS_SHIFT; - const ENTRY_ADDRESS_MASK: usize = - (Self::ENTRY_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize; - const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK; + const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_WIDTH; // size of addressable physical memory, in pages + const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - 1; // Mask of physical address, starting at 0th bit + const ENTRY_FLAGS_MASK: usize = !(Self::ENTRY_ADDRESS_MASK << Self::ENTRY_ADDRESS_SHIFT); unsafe fn init() -> &'static [MemoryArea]; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 72b99a1e27..f13e649ee4 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -10,8 +10,9 @@ impl Arch for RiscV64Sv39Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 3; // L0, L1, L2 - //TODO - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 44; + const ENTRY_ADDRESS_SHIFT: usize = 10; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag @@ -27,7 +28,7 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_GLOBAL: usize = 1 << 5; const ENTRY_FLAG_NO_GLOBAL: usize = 0; - const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; + const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; unsafe fn init() -> &'static [MemoryArea] { unimplemented!("RiscV64Sv39Arch::init unimplemented"); @@ -44,7 +45,7 @@ impl Arch for RiscV64Sv39Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN ) } @@ -56,10 +57,10 @@ impl Arch for RiscV64Sv39Arch { } fn virt_is_valid(address: VirtualAddress) -> bool { - const MASK: usize = 0xFFFF_FFC0_0000_0000; - let masked = address.data() & MASK; + let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); + let masked = address.data() & mask; - masked == MASK || masked == 0 + masked == mask || masked == 0 } } @@ -80,11 +81,11 @@ mod tests { assert_eq!(RiscV64Sv39Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(RiscV64Sv39Arch::PAGE_NEGATIVE_MASK, 0xFFFF_FF80_0000_0000); - assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); - assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE, 0x0000_1000_0000_0000); + assert_eq!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK, 0x0000_0FFF_FFFF_FFFF); + assert_eq!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK, 0xFFC0_0000_0000_03FF); - assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); + assert_eq!(RiscV64Sv39Arch::PHYS_OFFSET, 0xFFFF_FFC0_0000_0000); } #[test] fn is_canonical() { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index 196aee28e3..a63f887afd 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -10,8 +10,9 @@ impl Arch for RiscV64Sv48Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3 - //TODO - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 44; + const ENTRY_ADDRESS_SHIFT: usize = 10; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | 1 << 1 // Read flag @@ -44,7 +45,7 @@ impl Arch for RiscV64Sv48Arch { let satp: usize; asm!("csrr {0}, satp", out(reg) satp); PhysicalAddress::new( - (satp & 0x0000_0FFF_FFFF_FFFF) << Self::PAGE_SHIFT, // Convert from PPN + (satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN ) } @@ -56,7 +57,7 @@ impl Arch for RiscV64Sv48Arch { } fn virt_is_valid(address: VirtualAddress) -> bool { // RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64. - let mask = 0xFFFF_8000_0000_0000; + let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); let masked = address.data() & mask; masked == mask || masked == 0 @@ -80,9 +81,9 @@ mod tests { assert_eq!(RiscV64Sv48Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(RiscV64Sv48Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); - assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); - assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE, 0x0000_1000_0000_0000); + assert_eq!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK, 0x0000_0FFF_FFFF_FFFF); + assert_eq!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK, 0xFFC0_0000_0000_03FF); assert_eq!(RiscV64Sv48Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); } diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 176aef175d..3c4ac924f7 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -11,7 +11,7 @@ impl Arch for X86Arch { const PAGE_ENTRY_SHIFT: usize = 10; // 1024 entries, 4 bytes each const PAGE_LEVELS: usize = 2; // PD, PT - const ENTRY_ADDRESS_SHIFT: usize = 32; + const ENTRY_ADDRESS_WIDTH: usize = 20; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; @@ -70,8 +70,8 @@ mod tests { assert_eq!(X86Arch::PAGE_ENTRY_MASK, 0x3FF); assert_eq!(X86Arch::PAGE_NEGATIVE_MASK, 0x0000_0000_0000); - assert_eq!(X86Arch::ENTRY_ADDRESS_SIZE, 0x0000_0001_0000_0000); - assert_eq!(X86Arch::ENTRY_ADDRESS_MASK, 0xFFFF_F000); + assert_eq!(X86Arch::ENTRY_ADDRESS_SIZE, 0x0000_0000_0010_0000); + assert_eq!(X86Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF); assert_eq!(X86Arch::ENTRY_FLAGS_MASK, 0x0000_0FFF); assert_eq!(X86Arch::PHYS_OFFSET, 0x8000_0000); diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index ccde251a7c..316323adc3 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -10,7 +10,7 @@ impl Arch for X8664Arch { const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT - const ENTRY_ADDRESS_SHIFT: usize = 52; + const ENTRY_ADDRESS_WIDTH: usize = 40; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; @@ -80,8 +80,8 @@ mod tests { assert_eq!(X8664Arch::PAGE_ENTRY_MASK, 0x1FF); assert_eq!(X8664Arch::PAGE_NEGATIVE_MASK, 0xFFFF_0000_0000_0000); - assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0010_0000_0000_0000); - assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x000F_FFFF_FFFF_F000); + assert_eq!(X8664Arch::ENTRY_ADDRESS_SIZE, 0x0000_0100_0000_0000); + assert_eq!(X8664Arch::ENTRY_ADDRESS_MASK, 0x0000_00FF_FFFF_FFFF); assert_eq!(X8664Arch::ENTRY_FLAGS_MASK, 0xFFF0_0000_0000_0FFF); assert_eq!(X8664Arch::PHYS_OFFSET, 0xFFFF_8000_0000_0000); diff --git a/src/page/entry.rs b/src/page/entry.rs index 34a83d448f..ac75f162b0 100644 --- a/src/page/entry.rs +++ b/src/page/entry.rs @@ -10,7 +10,14 @@ pub struct PageEntry { impl PageEntry { #[inline(always)] - pub fn new(data: usize) -> Self { + pub fn new(address: usize, flags: usize) -> Self { + let data = (((address >> A::PAGE_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::ENTRY_ADDRESS_SHIFT) + | flags; + Self::from_data(data) + } + + #[inline(always)] + pub fn from_data(data: usize) -> Self { Self { data, phantom: PhantomData, @@ -24,7 +31,9 @@ impl PageEntry { #[inline(always)] pub fn address(&self) -> Result { - let addr = PhysicalAddress(self.data & A::ENTRY_ADDRESS_MASK); + let addr = PhysicalAddress( + ((self.data >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT, + ); if self.present() { Ok(addr) diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 5d2584ee63..842d5b1125 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -65,7 +65,7 @@ impl PageMapper { let old_flags = old_entry.flags(); let (new_phys, new_flags) = f(old_phys, old_flags); // TODO: Higher-level PageEntry::new interface? - let new_entry = PageEntry::new(new_phys.data() | new_flags.data()); + let new_entry = PageEntry::new(new_phys.data(), new_flags.data()); p1.set_entry(i, new_entry); Some((old_flags, old_phys, PageFlush::new(virt))) }) @@ -105,7 +105,7 @@ impl PageMapper { ) -> Option> { //TODO: verify virt and phys are aligned //TODO: verify flags have correct bits - let entry = PageEntry::new(phys.data() | flags.data()); + let entry = PageEntry::new(phys.data(), flags.data()); let mut table = self.table(); loop { let i = table.index_of(virt)?; @@ -127,7 +127,7 @@ impl PageMapper { } else { 0 }; - table.set_entry(i, PageEntry::new(next_phys.data() | flags)); + table.set_entry(i, PageEntry::new(next_phys.data(), flags)); table.next(i)? } }; @@ -198,7 +198,7 @@ unsafe fn unmap_phys_inner( if table.level() == 0 { let entry_opt = table.entry(i); - table.set_entry(i, PageEntry::new(0)); + table.set_entry(i, PageEntry::new(0, 0)); let entry = entry_opt?; Some((entry.address().ok()?, entry.flags())) @@ -218,7 +218,7 @@ unsafe fn unmap_phys_inner( if !is_still_populated { allocator.free_one(subtable.phys()); - table.set_entry(i, PageEntry::new(0)); + table.set_entry(i, PageEntry::new(0, 0)); } } diff --git a/src/page/table.rs b/src/page/table.rs index 323a808c00..70d281543d 100644 --- a/src/page/table.rs +++ b/src/page/table.rs @@ -72,7 +72,7 @@ impl PageTable { pub unsafe fn entry(&self, i: usize) -> Option> { let addr = self.entry_virt(i)?; - Some(PageEntry::new(A::read::(addr))) + Some(PageEntry::from_data(A::read::(addr))) } pub unsafe fn set_entry(&mut self, i: usize, entry: PageEntry) -> Option<()> { From f51cd00f00f445b364f2dde5825acb3e15ad6d7e Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 08:58:01 +0300 Subject: [PATCH 3/5] Remove read/write flags from common PDE flags. RISC-V convention marks PDE with no read/write/execute, so we can't have none of this flags set there. Remove their setting from PDE handling code and instead set them as appropriate in arch-specific defaults. Also enable both readonly and readwrite flags to be non-zero (as long as their intersection completely masks both of them), as required for RISC-V PTE handling. --- src/arch/aarch64.rs | 1 + src/arch/riscv64/sv39.rs | 9 +++------ src/arch/riscv64/sv48.rs | 9 +++------ src/arch/x86.rs | 2 +- src/arch/x86_64.rs | 2 +- src/page/flags.rs | 18 ++++++++++-------- src/page/mapper.rs | 3 +-- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 5999d60a44..82e4983183 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -18,6 +18,7 @@ impl Arch for AArch64Arch { | Self::ENTRY_FLAG_NO_GLOBAL; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT + | Self::ENTRY_FLAG_READWRITE | 1 << 1 // Table flag | 1 << 10 // Access flag ; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index f13e649ee4..607f4a2b1e 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -13,14 +13,11 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_ADDRESS_WIDTH: usize = 44; const ENTRY_ADDRESS_SHIFT: usize = 10; - const ENTRY_FLAG_DEFAULT_PAGE: usize - = Self::ENTRY_FLAG_PRESENT - | 1 << 1 // Read flag - ; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; - const ENTRY_FLAG_READONLY: usize = 0; - const ENTRY_FLAG_READWRITE: usize = 1 << 2; + const ENTRY_FLAG_READONLY: usize = 1 << 1; + const ENTRY_FLAG_READWRITE: usize = 3 << 1; const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index a63f887afd..e80e52a000 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -13,14 +13,11 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_ADDRESS_WIDTH: usize = 44; const ENTRY_ADDRESS_SHIFT: usize = 10; - const ENTRY_FLAG_DEFAULT_PAGE: usize - = Self::ENTRY_FLAG_PRESENT - | 1 << 1 // Read flag - ; + const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY; const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; const ENTRY_FLAG_PRESENT: usize = 1 << 0; - const ENTRY_FLAG_READONLY: usize = 0; - const ENTRY_FLAG_READWRITE: usize = 1 << 2; + const ENTRY_FLAG_READONLY: usize = 1 << 1; + const ENTRY_FLAG_READWRITE: usize = 3 << 1; const ENTRY_FLAG_PAGE_USER: usize = 1 << 4; const ENTRY_FLAG_TABLE_USER: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index 3c4ac924f7..b0e333a342 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -13,7 +13,7 @@ impl Arch for X86Arch { const ENTRY_ADDRESS_WIDTH: usize = 20; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; - const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 316323adc3..25fa84691d 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -12,7 +12,7 @@ impl Arch for X8664Arch { const ENTRY_ADDRESS_WIDTH: usize = 40; const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT; - const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; + const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; const ENTRY_FLAG_PRESENT: usize = 1 << 0; const ENTRY_FLAG_READONLY: usize = 0; const ENTRY_FLAG_READWRITE: usize = 1 << 1; diff --git a/src/page/flags.rs b/src/page/flags.rs index ca03d1ce0f..9de75484ef 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -27,10 +27,7 @@ impl PageFlags { unsafe { Self::from_data( // Flags set to present, kernel space, read-only, no-execute by default - A::ENTRY_FLAG_DEFAULT_TABLE - | A::ENTRY_FLAG_READONLY - | A::ENTRY_FLAG_NO_EXEC - | A::ENTRY_FLAG_NO_GLOBAL, + A::ENTRY_FLAG_DEFAULT_TABLE | A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_NO_GLOBAL, ) } } @@ -83,14 +80,19 @@ impl PageFlags { #[must_use] #[inline(always)] pub fn write(self, value: bool) -> Self { - // Architecture may use readonly or readwrite, support either - self.custom_flag(A::ENTRY_FLAG_READONLY, !value) - .custom_flag(A::ENTRY_FLAG_READWRITE, value) + // Architecture may use readonly or readwrite, or both, support either + if value { + self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false) + .custom_flag(A::ENTRY_FLAG_READWRITE, true) + } else { + self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false) + .custom_flag(A::ENTRY_FLAG_READONLY, true) + } } #[inline(always)] pub fn has_write(&self) -> bool { - // Architecture may use readonly or readwrite, support either + // Architecture may use readonly or readwrite, or both, support either self.data & (A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE) == A::ENTRY_FLAG_READWRITE } diff --git a/src/page/mapper.rs b/src/page/mapper.rs index 842d5b1125..a6aa726be8 100644 --- a/src/page/mapper.rs +++ b/src/page/mapper.rs @@ -120,8 +120,7 @@ impl PageMapper { None => { let next_phys = self.allocator.allocate_one()?; //TODO: correct flags? - let flags = A::ENTRY_FLAG_READWRITE - | A::ENTRY_FLAG_DEFAULT_TABLE + let flags = A::ENTRY_FLAG_DEFAULT_TABLE | if virt.kind() == TableKind::User { A::ENTRY_FLAG_TABLE_USER } else { From 192dd8283fcc45ebfa94500695bb3fabdc089887 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 12 Jul 2024 09:01:23 +0300 Subject: [PATCH 4/5] RISC-V: implement TLB flush --- src/arch/riscv64/sv39.rs | 11 ++++++++--- src/arch/riscv64/sv48.rs | 12 +++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 607f4a2b1e..81dd8583f1 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -32,9 +32,13 @@ impl Arch for RiscV64Sv39Arch { } #[inline(always)] - unsafe fn invalidate(_address: VirtualAddress) { - //TODO: can one address be invalidated? - Self::invalidate_all(); + unsafe fn invalidate(address: VirtualAddress) { + asm!("sfence.vma {}", in(reg) address.data()); + } + + #[inline(always)] + unsafe fn invalidate_all() { + asm!("sfence.vma"); } #[inline(always)] @@ -51,6 +55,7 @@ impl Arch for RiscV64Sv39Arch { let satp = (8 << 60) | // Sv39 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); } fn virt_is_valid(address: VirtualAddress) -> bool { diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index e80e52a000..b6796ab3d8 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -32,9 +32,13 @@ impl Arch for RiscV64Sv48Arch { } #[inline(always)] - unsafe fn invalidate(_address: VirtualAddress) { - //TODO: can one address be invalidated? - Self::invalidate_all(); + unsafe fn invalidate(address: VirtualAddress) { + asm!("sfence.vma {}", in(reg) address.data()); + } + + #[inline(always)] + unsafe fn invalidate_all() { + asm!("sfence.vma"); } #[inline(always)] @@ -51,7 +55,9 @@ impl Arch for RiscV64Sv48Arch { let satp = (9 << 60) | // Sv48 MODE (address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment) asm!("csrw satp, {0}", in(reg) satp); + Self::invalidate_all(); } + fn virt_is_valid(address: VirtualAddress) -> bool { // RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64. let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1); From ed8bcfca1f7427d8acca966a8c6ca36ce887467b Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Fri, 28 Jun 2024 06:30:25 +0300 Subject: [PATCH 5/5] Add write combining page flags where applicable --- src/arch/aarch64.rs | 1 + src/arch/emulate.rs | 2 ++ src/arch/mod.rs | 1 + src/arch/riscv64/sv39.rs | 1 + src/arch/riscv64/sv48.rs | 1 + src/arch/x86.rs | 1 + src/arch/x86_64.rs | 1 + src/page/flags.rs | 6 ++++++ 8 files changed, 14 insertions(+) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index 82e4983183..687294d3b5 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -32,6 +32,7 @@ impl Arch for AArch64Arch { const ENTRY_FLAG_EXEC: usize = 0; const ENTRY_FLAG_GLOBAL: usize = 0; const ENTRY_FLAG_NO_GLOBAL: usize = 1 << 11; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/emulate.rs b/src/arch/emulate.rs index d228161df0..3e381db676 100644 --- a/src/arch/emulate.rs +++ b/src/arch/emulate.rs @@ -31,6 +31,8 @@ impl Arch for EmulateArch { const ENTRY_ADDRESS_WIDTH: usize = X8664Arch::ENTRY_ADDRESS_WIDTH; + const ENTRY_FLAG_WRITE_COMBINING: usize = X8664Arch::ENTRY_FLAG_WRITE_COMBINING; + unsafe fn init() -> &'static [MemoryArea] { // Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64) let mut machine = Machine::new(MEMORY_SIZE); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 1d40b3109c..cd68f09f2c 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -43,6 +43,7 @@ pub trait Arch: Clone + Copy { const ENTRY_FLAG_EXEC: usize; const ENTRY_FLAG_GLOBAL: usize; const ENTRY_FLAG_NO_GLOBAL: usize; + const ENTRY_FLAG_WRITE_COMBINING: usize; const PHYS_OFFSET: usize; diff --git a/src/arch/riscv64/sv39.rs b/src/arch/riscv64/sv39.rs index 81dd8583f1..31ade20c1e 100644 --- a/src/arch/riscv64/sv39.rs +++ b/src/arch/riscv64/sv39.rs @@ -24,6 +24,7 @@ impl Arch for RiscV64Sv39Arch { const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; diff --git a/src/arch/riscv64/sv48.rs b/src/arch/riscv64/sv48.rs index b6796ab3d8..7e80ad0c3a 100644 --- a/src/arch/riscv64/sv48.rs +++ b/src/arch/riscv64/sv48.rs @@ -24,6 +24,7 @@ impl Arch for RiscV64Sv48Arch { const ENTRY_FLAG_EXEC: usize = 1 << 3; const ENTRY_FLAG_GLOBAL: usize = 1 << 5; const ENTRY_FLAG_NO_GLOBAL: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 0; const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/x86.rs b/src/arch/x86.rs index b0e333a342..7e078a2fc0 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -23,6 +23,7 @@ impl Arch for X86Arch { const ENTRY_FLAG_NO_GLOBAL: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 0; // NOT AVAILABLE UNLESS PAE IS USED! const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 1 << 7; const PHYS_OFFSET: usize = 0x8000_0000; diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 25fa84691d..3b0bacff0d 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -22,6 +22,7 @@ impl Arch for X8664Arch { const ENTRY_FLAG_NO_GLOBAL: usize = 0; const ENTRY_FLAG_NO_EXEC: usize = 1 << 63; const ENTRY_FLAG_EXEC: usize = 0; + const ENTRY_FLAG_WRITE_COMBINING: usize = 1 << 7; const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1) as usize; // PML4 slot 256 and onwards diff --git a/src/page/flags.rs b/src/page/flags.rs index 9de75484ef..3d90691d60 100644 --- a/src/page/flags.rs +++ b/src/page/flags.rs @@ -56,6 +56,12 @@ impl PageFlags { self } + #[must_use] + #[inline(always)] + pub fn write_combining(self, value: bool) -> Self { + self.custom_flag(A::ENTRY_FLAG_WRITE_COMBINING, value) + } + #[inline(always)] pub fn has_flag(&self, flag: usize) -> bool { self.data & flag == flag