From f581c71c7c5925402fc51e0dc137d92d5b672db4 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 14 Sep 2023 08:38:49 +0200 Subject: [PATCH] Allow code-overwriting optimizations, use for smap. --- src/arch/x86_64/alternative.rs | 105 +++++++++++++++++++++++++++++++++ src/arch/x86_64/cpuid.rs | 7 ++- src/arch/x86_64/macros.rs | 12 +--- src/arch/x86_64/misc.rs | 18 +----- src/arch/x86_64/mod.rs | 2 + src/arch/x86_64/start.rs | 4 ++ 6 files changed, 119 insertions(+), 29 deletions(-) create mode 100644 src/arch/x86_64/alternative.rs diff --git a/src/arch/x86_64/alternative.rs b/src/arch/x86_64/alternative.rs new file mode 100644 index 0000000000..10ed7f69e7 --- /dev/null +++ b/src/arch/x86_64/alternative.rs @@ -0,0 +1,105 @@ +use core::mem::size_of; + +use x86::controlregs::Cr4; + +use crate::context::memory::PageSpan; +use crate::cpuid::has_ext_feat; +use crate::paging::{KernelMapper, Page, PageFlags, PAGE_SIZE, VirtualAddress}; + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct AltReloc { + pub name_start: *const u8, + pub name_len: usize, + pub code_start: *mut u8, + pub origcode_len: usize, + pub altcode_start: *const u8, + pub altcode_len: usize, +} + +#[cold] +pub unsafe fn early_init(bsp: bool) { + let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start(); + let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset; + + assert_eq!(relocs_size % size_of::(), 0); + let relocs = core::slice::from_raw_parts(relocs_offset as *const AltReloc, relocs_size / size_of::()); + + let mut enable_smap = false; + + if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) { + // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any + // userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This + // limits user-memory accesses to the UserSlice wrapper, so that no data outside of + // usercopy functions can be accidentally accessed by the kernel. + x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP); + // Clear CLAC in (the probably unlikely) case the bootloader set it earlier. + x86::bits64::rflags::clac(); + + enable_smap = true; + } + if !bsp { + return; + } + + let mut mapper = KernelMapper::lock(); + for reloc in relocs.iter().copied() { + let name = core::str::from_utf8(core::slice::from_raw_parts(reloc.name_start, reloc.name_len)).expect("invalid feature name"); + let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len); + + let total_length = core::cmp::max(reloc.altcode_len, reloc.origcode_len); + + let dst_pages = PageSpan::between( + Page::containing_address(VirtualAddress::new(reloc.code_start as usize)), + Page::containing_address(VirtualAddress::new((reloc.code_start as usize + reloc.origcode_len).next_multiple_of(PAGE_SIZE))), + ); + for page in dst_pages.pages() { + mapper.remap(page.start_address(), PageFlags::new().write(true).execute(true).global(true)).unwrap().flush(); + } + + let code = core::slice::from_raw_parts_mut(reloc.code_start, total_length); + + log::info!("feature {} current {:x?} altcode {:x?}", name, code, altcode); + + let feature_is_enabled = match name { + "smap" => enable_smap, + //_ => panic!("unknown altcode relocation: {}", name), + _ => true, + }; + + if !feature_is_enabled { + continue; + } + + let (dst, dst_nops) = code.split_at_mut(reloc.altcode_len); + dst.copy_from_slice(altcode); + + // XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X` + // only outputs the (slower) single-byte 0x90 NOP. + + // This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November + // 2020). + const NOPS_TABLE: [&[u8]; 11] = [ + &[0x90], + &[0x66, 0x90], + &[0x0f, 0x1f, 0x00], + &[0x0f, 0x1f, 0x40, 0x00], + &[0x0f, 0x1f, 0x44, 0x00, 0x00], + &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00], + &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00], + &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + ]; + + for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) { + chunk.copy_from_slice(NOPS_TABLE[chunk.len()]); + } + log::info!("feature {} new {:x?} altcode {:x?}", name, code, altcode); + + for page in dst_pages.pages() { + mapper.remap(page.start_address(), PageFlags::new().write(false).execute(true).global(true)).unwrap().flush(); + } + } +} diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 6562b03919..5b2782da4f 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -1,4 +1,4 @@ -use raw_cpuid::{CpuId, CpuIdResult}; +use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures}; pub fn cpuid() -> Option { // CPUID is always available on x86_64 systems. @@ -15,3 +15,8 @@ pub fn cpuid_always() -> CpuId { } }) } +pub fn has_ext_feat(feat: impl FnOnce(ExtendedFeatures) -> bool) -> bool { + cpuid_always() + .get_extended_feature_info() + .map_or(false, feat) +} diff --git a/src/arch/x86_64/macros.rs b/src/arch/x86_64/macros.rs index 6f9c6d01c1..fdba30b17f 100644 --- a/src/arch/x86_64/macros.rs +++ b/src/arch/x86_64/macros.rs @@ -15,16 +15,6 @@ macro_rules! println { ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } - -#[repr(C)] -pub struct AltReloc { - pub code_start: usize, - pub code_end: usize, - pub altcode_start: usize, - pub altcode_end: usize, - pub name_start: usize, - pub name_end: usize, -} macro_rules! expand_bool( ($value:expr) => { concat!($value) @@ -95,7 +85,7 @@ macro_rules! alternative_auto( .quad 70b .quad 71b - 70b .quad 40b - .quad 41b - 40b + .quad 42b - 40b .quad ", $first_digit, "0b .quad ", $first_digit, "1b - ", $first_digit, "0b .popsection diff --git a/src/arch/x86_64/misc.rs b/src/arch/x86_64/misc.rs index 33f3268b50..181efaf357 100644 --- a/src/arch/x86_64/misc.rs +++ b/src/arch/x86_64/misc.rs @@ -1,16 +1,9 @@ use x86::controlregs::Cr4; -use x86::cpuid::ExtendedFeatures; use crate::LogicalCpuId; -use crate::cpuid::cpuid_always; +use crate::cpuid::{cpuid_always, has_ext_feat}; pub unsafe fn init(cpu_id: LogicalCpuId) { - let has_ext_feat = |feat: fn(ExtendedFeatures) -> bool| { - cpuid_always() - .get_extended_feature_info() - .map_or(false, feat) - }; - if has_ext_feat(|feat| feat.has_umip()) { // UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT, // SMSW and STR. KASLR is currently not implemented, but this protects against leaking @@ -23,15 +16,6 @@ pub unsafe fn init(cpu_id: LogicalCpuId) { // obvious reasons. x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP); } - if has_ext_feat(|feat| feat.has_smap()) && cfg!(cpu_feature_always = "smap") { - // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any - // userspace-accessible pages, with the necessary exception of RFLAGS.AC = 1. This limits - // user-memory accesses to the UserSlice wrapper, so that no code outside of usercopy - // functions can be accidentally accessed by the kernel. - x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP); - // Clear CLAC in (the probably unlikely) case the bootloader set it earlier. - x86::bits64::rflags::clac(); - } if let Some(feats) = cpuid_always().get_extended_processor_and_feature_identifiers() && feats.has_rdtscp() { x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into()); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 4e1860efb0..e143f1a08b 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -2,6 +2,8 @@ use crate::Bootstrap; use self::paging::PAGE_SIZE; +pub mod alternative; + #[macro_use] pub mod macros; diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index bdeed30134..dae406a118 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -137,6 +137,8 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Set up IDT idt::init_paging_bsp(); + crate::alternative::early_init(true); + // Set up syscall instruction interrupt::syscall::init(); @@ -235,6 +237,8 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { // Set up IDT for AP idt::init_paging_post_heap(false, cpu_id); + crate::alternative::early_init(false); + // Set up syscall instruction interrupt::syscall::init();