diff --git a/config.toml.example b/config.toml.example index 80c13fcbe2..c7a71487fe 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,5 +1,5 @@ [arch.x86_64.features] -smap = "always" -fsgsbase = "never" +smap = "auto" +fsgsbase = "auto" # vim: ft=toml diff --git a/src/arch/x86_64/alternative.rs b/src/arch/x86_64/alternative.rs index 10ed7f69e7..9eb92875a0 100644 --- a/src/arch/x86_64/alternative.rs +++ b/src/arch/x86_64/alternative.rs @@ -1,9 +1,10 @@ use core::mem::size_of; +use spin::Once; use x86::controlregs::Cr4; use crate::context::memory::PageSpan; -use crate::cpuid::has_ext_feat; +use crate::cpuid::{has_ext_feat, cpuid_always}; use crate::paging::{KernelMapper, Page, PageFlags, PAGE_SIZE, VirtualAddress}; #[repr(C)] @@ -25,7 +26,7 @@ pub unsafe fn early_init(bsp: bool) { assert_eq!(relocs_size % size_of::(), 0); let relocs = core::slice::from_raw_parts(relocs_offset as *const AltReloc, relocs_size / size_of::()); - let mut enable_smap = false; + let mut enable = KcpuFeatures::empty(); if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) { // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any @@ -36,8 +37,18 @@ pub unsafe fn early_init(bsp: bool) { // Clear CLAC in (the probably unlikely) case the bootloader set it earlier. x86::bits64::rflags::clac(); - enable_smap = true; + enable |= KcpuFeatures::SMAP; } + + if cfg!(not(cpu_feature_never = "fsgsbase")) + && let Some(f) = cpuid_always().get_extended_feature_info() + && f.has_fsgsbase() + { + x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE); + + enable |= KcpuFeatures::FSGSBASE; + } + if !bsp { return; } @@ -62,7 +73,8 @@ pub unsafe fn early_init(bsp: bool) { log::info!("feature {} current {:x?} altcode {:x?}", name, code, altcode); let feature_is_enabled = match name { - "smap" => enable_smap, + "smap" => enable.contains(KcpuFeatures::SMAP), + "fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE), //_ => panic!("unknown altcode relocation: {}", name), _ => true, }; @@ -102,4 +114,18 @@ pub unsafe fn early_init(bsp: bool) { mapper.remap(page.start_address(), PageFlags::new().write(false).execute(true).global(true)).unwrap().flush(); } } + FEATURES.call_once(|| enable); +} + +bitflags! { + pub struct KcpuFeatures: usize { + const SMAP = 1; + const FSGSBASE = 2; + } +} + +static FEATURES: Once = Once::new(); + +pub fn features() -> KcpuFeatures { + *FEATURES.get().expect("early_cpu_init was not called") } diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index db76e89354..fa056c72fe 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -13,8 +13,6 @@ use x86::dtables::{self, DescriptorTablePointer}; use x86::segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector}; use x86::task; -use super::cpuid::cpuid; - pub const GDT_NULL: usize = 0; pub const GDT_KERNEL_CODE: usize = 1; pub const GDT_KERNEL_DATA: usize = 2; @@ -200,18 +198,6 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); - let cpu_supports_fsgsbase = cpuid().map_or(false, |cpuid| { - cpuid.get_extended_feature_info().map_or(false, |extended_features| { - extended_features.has_fsgsbase() - }) - }); - - if cfg!(cpu_feature_always = "fsgsbase") { - assert!(cpu_supports_fsgsbase, "running kernel with features not supported by the current CPU"); - - x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE); - } - pcr.percpu = PercpuBlock { cpu_id, switch_internals: Default::default(), diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index dae406a118..977ea50f78 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -289,64 +289,6 @@ macro_rules! inner_pit_unmap( } ); -#[cfg(cpu_feature_never = "fsgsbase")] -macro_rules! save_fsgsbase( - () => { - " - mov ecx, {MSR_FSBASE} - rdmsr - shl rdx, 32 - or rdx, rax - mov r14, rdx - - mov ecx, {MSR_GSBASE} - rdmsr - shl rdx, 32 - or rdx, rax - mov r13, rdx - " - } -); -#[cfg(cpu_feature_always = "fsgsbase")] -macro_rules! save_fsgsbase( - () => { - " - // placeholder: {MSR_FSBASE} {MSR_GSBASE} - rdfsbase r14 - rdgsbase r13 - " - } -); - -#[cfg(cpu_feature_always = "fsgsbase")] -macro_rules! restore_fsgsbase( - () => { - " - wrfsbase r14 - wrgsbase r13 - " - } -); - -#[cfg(cpu_feature_never = "fsgsbase")] -macro_rules! restore_fsgsbase( - () => { - " - mov ecx, {MSR_FSBASE} - mov rdx, r14 - mov eax, edx - shr rdx, 32 - wrmsr - - mov ecx, {MSR_GSBASE} - mov rdx, r13 - mov eax, edx - shr rdx, 32 - wrmsr - " - } -); - #[naked] // TODO: AbiCompatBool pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _is_singlestep: usize) -> ! { @@ -367,7 +309,17 @@ pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _is_singl // Go to usermode swapgs - ", save_fsgsbase!(), " + mov ecx, {MSR_FSBASE} + rdmsr + shl rdx, 32 + or rdx, rax + mov r14, rdx + + mov ecx, {MSR_GSBASE} + rdmsr + shl rdx, 32 + or rdx, rax + mov r13, rdx mov r15, {user_data_seg_selector} mov ds, r15d @@ -378,7 +330,18 @@ pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _is_singl // SS and CS will later be set via sysretq. - restore_fsgsbase!(), " + " + mov ecx, {MSR_FSBASE} + mov rdx, r14 + mov eax, edx + shr rdx, 32 + wrmsr + + mov ecx, {MSR_GSBASE} + mov rdx, r13 + mov eax, edx + shr rdx, 32 + wrmsr // Target instruction pointer mov rcx, rdi diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 33774682aa..8e87502eb6 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -1,4 +1,5 @@ use core::mem; +use core::ptr::{addr_of, addr_of_mut}; use core::sync::atomic::AtomicBool; use alloc::sync::Arc; @@ -10,6 +11,7 @@ use crate::syscall::FloatRegisters; use memoffset::offset_of; use spin::Once; +use x86::msr; /// This must be used by the kernel to ensure that context switches are done atomically /// Compare and exchange this to true when beginning a context switch on any CPU @@ -142,21 +144,50 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { ); { - use x86::{bits64::segmentation::*, msr}; + core::arch::asm!( + alternative!( + feature: "fsgsbase", + then: [" + mov rax, [{next}+{fsbase_off}] + mov rcx, [{next}+{gsbase_off}] - // This is so much shorter in Rust! + rdfsbase rdx + wrfsbase rax + swapgs + rdgsbase rax + wrgsbase rcx + swapgs - if cfg!(cpu_feature_always = "fsgsbase") { - prev.arch.fsbase = rdfsbase() as usize; - wrfsbase(next.arch.fsbase as u64); - swapgs(); - prev.arch.gsbase = rdgsbase() as usize; - wrgsbase(next.arch.gsbase as u64); - swapgs(); - } else { - msr::wrmsr(msr::IA32_FS_BASE, next.arch.fsbase as u64); - msr::wrmsr(msr::IA32_KERNEL_GSBASE, next.arch.gsbase as u64); - } + mov [{prev}+{fsbase_off}], rdx + mov [{prev}+{gsbase_off}], rax + "], + // TODO: Most applications will set FSBASE, but won't touch GSBASE. Maybe avoid + // wrmsr or even the swapgs+rdgsbase+wrgsbase+swapgs sequence if they are already + // equal? + default: [" + mov ecx, {MSR_FSBASE} + mov rdx, [{next}+{fsbase_off}] + mov eax, edx + shr rdx, 32 + wrmsr + + mov ecx, {MSR_KERNEL_GSBASE} + mov rdx, [{next}+{gsbase_off}] + mov eax, edx + shr rdx, 32 + wrmsr + + // {prev} + "] + ), + out("rax") _, + out("rdx") _, + out("ecx") _, prev = in(reg) addr_of_mut!(prev.arch), next = in(reg) addr_of!(next.arch), + MSR_FSBASE = const msr::IA32_FS_BASE, + MSR_KERNEL_GSBASE = const msr::IA32_KERNEL_GSBASE, + gsbase_off = const offset_of!(Context, gsbase), + fsbase_off = const offset_of!(Context, fsbase), + ); } match next.addr_space { diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 71d64dc0d3..ece25f5a29 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -420,28 +420,14 @@ impl ProcScheme { #[cfg(target_arch = "x86_64")] fn read_env_regs(&self, info: &Info) -> Result { + // TODO: Avoid rdmsr if fsgsbase is not enabled, if this is worth optimizing for. let (fsbase, gsbase) = if info.pid == context::context_id() { - #[cfg(cpu_feature_never = "fsgsbase")] unsafe { ( x86::msr::rdmsr(x86::msr::IA32_FS_BASE), x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE), ) } - #[cfg(cpu_feature_always = "fsgsbase")] - unsafe { - use x86::bits64::segmentation::*; - - ( - rdfsbase(), - { - swapgs(); - let gsbase = rdgsbase(); - swapgs(); - gsbase - } - ) - } } else { try_stop_context(info.pid, |context| { Ok((context.arch.fsbase as u64, context.arch.gsbase as u64)) @@ -504,7 +490,6 @@ impl ProcScheme { } if info.pid == context::context_id() { - #[cfg(cpu_feature_never = "fsgsbase")] unsafe { x86::msr::wrmsr(x86::msr::IA32_FS_BASE, regs.fsbase as u64); // We have to write to KERNEL_GSBASE, because when the kernel returns to @@ -518,18 +503,6 @@ impl ProcScheme { } } } - #[cfg(cpu_feature_always = "fsgsbase")] - unsafe { - use x86::bits64::segmentation::*; - - wrfsbase(regs.fsbase); - swapgs(); - wrgsbase(regs.gsbase); - swapgs(); - - // No need to update the current context; with fsgsbase enabled, these - // registers are automatically saved and restored. - } } else { try_stop_context(info.pid, |context| { context.arch.fsbase = regs.fsbase as usize; @@ -562,7 +535,7 @@ impl Scheme for ProcScheme { fn fcntl(&self, id: usize, cmd: usize, arg: usize) -> Result { let mut handles = self.handles.write(); - let mut handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; match cmd { F_SETFL => { handle.info.flags = arg; Ok(0) },