diff --git a/Cargo.lock b/Cargo.lock index 6c52bb13f8..0d6c719864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "autocfg" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 766f1bdc50..9f651a64b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,9 @@ serial_debug = [] system76_ec_debug = [] slab = ["slab_allocator"] +# TODO: Either wait for LLVM 12 and use target_feature, or use another system for cpu features +x86_fsgsbase = [] + [profile.dev] # Kernel doesn't yet work great with debug mode :( opt-level = 3 diff --git a/src/arch/x86_64/consts.rs b/src/arch/x86_64/consts.rs index 2e1b462707..5589106e14 100644 --- a/src/arch/x86_64/consts.rs +++ b/src/arch/x86_64/consts.rs @@ -39,13 +39,6 @@ pub const USER_OFFSET: usize = 0; pub const USER_PML4: usize = (USER_OFFSET & PML4_MASK)/PML4_SIZE; - /// Offset to user TCB - /// Each process has 4096 bytes, at an offset of 4096 * PID - // TODO: Get a real 64-bit offset, and allow loading ELF sections higher up than the current - // limit, iff the processor supports fsgsbase (in which case it is cheap to use 64-bit FS - // offsets). - pub const USER_TCB_OFFSET: usize = 0xB000_0000; - /// Offset to user arguments pub const USER_ARG_OFFSET: usize = USER_OFFSET + PML4_SIZE/2; @@ -69,14 +62,8 @@ /// Size of user sigstack pub const USER_SIGSTACK_SIZE: usize = 256 * 1024; // 256 KB - /// Offset to user TLS - pub const USER_TLS_OFFSET: usize = USER_SIGSTACK_OFFSET + PML4_SIZE; - pub const USER_TLS_PML4: usize = (USER_TLS_OFFSET & PML4_MASK)/PML4_SIZE; - // Maximum TLS allocated to each PID, should be approximately 8 MB - pub const USER_TLS_SIZE: usize = PML4_SIZE / 65536; - /// Offset to user temporary image (used when cloning) - pub const USER_TMP_OFFSET: usize = USER_TLS_OFFSET + PML4_SIZE; + pub const USER_TMP_OFFSET: usize = USER_SIGSTACK_OFFSET + PML4_SIZE; pub const USER_TMP_PML4: usize = (USER_TMP_OFFSET & PML4_MASK)/PML4_SIZE; /// Offset to user temporary heap (used when cloning) @@ -95,10 +82,6 @@ pub const USER_TMP_SIGSTACK_OFFSET: usize = USER_TMP_STACK_OFFSET + PML4_SIZE; pub const USER_TMP_SIGSTACK_PML4: usize = (USER_TMP_SIGSTACK_OFFSET & PML4_MASK)/PML4_SIZE; - /// Offset to user temporary tls (used when cloning) - pub const USER_TMP_TLS_OFFSET: usize = USER_TMP_SIGSTACK_OFFSET + PML4_SIZE; - pub const USER_TMP_TLS_PML4: usize = (USER_TMP_TLS_OFFSET & PML4_MASK)/PML4_SIZE; - /// Offset for usage in other temporary pages - pub const USER_TMP_MISC_OFFSET: usize = USER_TMP_TLS_OFFSET + PML4_SIZE; + pub const USER_TMP_MISC_OFFSET: usize = USER_TMP_SIGSTACK_OFFSET + PML4_SIZE; pub const USER_TMP_MISC_PML4: usize = (USER_TMP_MISC_OFFSET & PML4_MASK)/PML4_SIZE; diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 32553b028b..94ad795baa 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -19,9 +19,8 @@ pub const GDT_KERNEL_KPCR: usize = 3; pub const GDT_USER_CODE32_UNUSED: usize = 4; pub const GDT_USER_DATA: usize = 5; pub const GDT_USER_CODE: usize = 6; -pub const GDT_USER_TLS: usize = 7; -pub const GDT_TSS: usize = 8; -pub const GDT_TSS_HIGH: usize = 9; +pub const GDT_TSS: usize = 7; +pub const GDT_TSS_HIGH: usize = 8; pub const GDT_A_PRESENT: u8 = 1 << 7; pub const GDT_A_RING_0: u8 = 0 << 5; @@ -53,7 +52,7 @@ static mut INIT_GDT: [GdtEntry; 4] = [ ]; #[thread_local] -pub static mut GDT: [GdtEntry; 10] = [ +pub static mut GDT: [GdtEntry; 9] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code @@ -68,8 +67,6 @@ pub static mut GDT: [GdtEntry; 10] = [ GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), // User (64-bit) code GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), - // User TLS - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), // TSS GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0), // TSS must be 16 bytes long, twice the normal size @@ -106,11 +103,6 @@ pub static mut KPCR: ProcessorControlRegion = ProcessorControlRegion { }), }; -pub unsafe fn set_tcb(pid: usize) { - GDT[GDT_USER_TLS].set_offset((crate::USER_TCB_OFFSET + pid * PAGE_SIZE) as u32); - x86::segmentation::load_fs(SegmentSelector::new(GDT_USER_TLS as u16, Ring::Ring3)); -} - #[cfg(feature = "pti")] pub unsafe fn set_tss_stack(stack: usize) { use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK}; @@ -199,8 +191,8 @@ pub unsafe fn init_paging(tcb_offset: usize, stack_offset: usize) { // is called again, making the userspace GS always point to user data. x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0); - // Set the User TLS segment to the offset of the user TCB - set_tcb(0); + // Set the User TLS segment to zero, before we create any contexts and start scheduling. + x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0); // Reload the segment descriptors load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); @@ -214,6 +206,18 @@ pub unsafe fn init_paging(tcb_offset: usize, stack_offset: usize) { // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); + + let has_fsgsbase = raw_cpuid::CpuId::new() + .get_extended_feature_info() + .map_or(false, |extended_features| extended_features.has_fsgsbase()); + + if cfg!(feature = "x86_fsgsbase") { + assert!(has_fsgsbase, "running kernel with features not supported by the current CPU"); + } + + if has_fsgsbase { + x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE); + } } #[derive(Copy, Clone, Debug)] diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index ac67cc858c..877792d26a 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -80,6 +80,11 @@ impl IretRegisters { println!("RSP: {:>016X}", { self.rsp }); println!("SS: {:>016X}", { self.ss }); } + unsafe { + let fsbase = x86::msr::rdmsr(x86::msr::IA32_FS_BASE); + let gsbase = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE); + println!("FSBASE {:>016X}\nGSBASE {:016X}", fsbase, gsbase); + } } } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 88e50734d8..0811d26983 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -45,6 +45,7 @@ pub use ::rmm::X8664Arch as CurrentRmmArch; // Flags pub mod flags { - pub const FLAG_SINGLESTEP: usize = 1 << 8; + pub const SHIFT_SINGLESTEP: usize = 8; + pub const FLAG_SINGLESTEP: usize = 1 << SHIFT_SINGLESTEP; pub const FLAG_INTERRUPTS: usize = 1 << 9; } diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index c0a2ac4ee3..9f94e08319 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -239,45 +239,133 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { crate::kmain_ap(cpu_id); } +#[cfg(not(feature = "pit"))] +macro_rules! inner_pit_unmap( + () => { + " + // unused: {pti_unmap} + " + } +); +#[cfg(feature = "pit")] +macro_rules! inner_pit_unmap( + () => { + " + push rdi + push rsi + push rdx + push rcx + sub rsp, 8 + + call {pti_unmap} + + add rsp, 8 + pop rcx + pop rdx + pop rsi + pop rdi + " + } +); + +#[cfg(not(feature = "x86_fsgsbase"))] +macro_rules! save_fsgsbase( + () => { + " + mov ecx, {MSR_FSBASE} + rdmsr + shl rdx, 32 + or rdx, rax + mov r14, rdx + + mov ecx, {MSR_GSBASE} + rdmsr + shl rdx, 32 + or rdx, rax + mov r13, rdx + " + } +); +#[cfg(feature = "x86_fsgsbase")] +macro_rules! save_fsgsbase( + () => { + " + // placeholder: {MSR_FSBASE} {MSR_GSBASE} + rdfsbase r14 + rdgsbase r13 + " + } +); + +#[cfg(feature = "x86_fsgsbase")] +macro_rules! restore_fsgsbase( + () => { + " + wrfsbase r14 + wrgsbase r13 + " + } +); + +#[cfg(not(feature = "x86_fsgsbase"))] +macro_rules! restore_fsgsbase( + () => { + " + mov ecx, {MSR_FSBASE} + mov rdx, r14 + mov eax, edx + shr rdx, 32 + wrmsr + + mov ecx, {MSR_GSBASE} + mov rdx, r13 + mov eax, edx + shr rdx, 32 + wrmsr + " + } +); + #[naked] #[inline(never)] // TODO: AbiCompatBool -pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _singlestep: u32) -> ! { +pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _is_singlestep: usize) -> ! { // rdi, rsi, rdx, rcx asm!( - " - mov rbx, {flag_interrupts} - test ecx, ecx - jz .after_singlestep_branch - or rbx, {flag_singlestep} + concat!(" + shl rcx, {shift_singlestep} + or rcx, {flag_interrupts} - .after_singlestep_branch: + ", inner_pit_unmap!(), " - // save `ip` (rdi), `sp` (rsi), and `arg` (rdx) in callee-preserved registers, so that - // they are not modified by `pti_unmap` + // Save rdx for later + mov r12, rdx - mov r13, rdi - mov r14, rsi - mov r15, rdx - call {pti_unmap} + // Target RFLAGS + mov r11, rcx // Go to usermode swapgs - mov r8, {user_data_seg_selector} - mov r9, {user_tls_seg_selector} - mov ds, r8d - mov es, r8d - mov fs, r9d - mov gs, r8d - // Target RFLAGS - mov r11, rbx + ", save_fsgsbase!(), " + + mov r15, {user_data_seg_selector} + mov ds, r15d + mov es, r15d + mov fs, r15d + mov gs, r15d + ", + + // SS and CS will later be set via sysretq. + + restore_fsgsbase!(), " + // Target instruction pointer - mov rcx, r13 + mov rcx, rdi // Target stack pointer - mov rsp, r14 + mov rsp, rsi // Target argument - mov rdi, r15 + mov rdi, r12 xor rax, rax xor rbx, rbx @@ -297,20 +385,24 @@ pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _singlest xor r15, r15 fninit - + ", // NOTE: Regarding the sysretq vulnerability, this is safe as we cannot modify RCX, // even though the caller can give us the wrong address. But, it's marked unsafe, so // the caller is responsible for this! (And, the likelihood of rcx being changed in the // middle here, is minimal, unless the attacker already has partial control of kernel // memory.) + " sysretq - ", + "), flag_interrupts = const(FLAG_INTERRUPTS), - flag_singlestep = const(FLAG_SINGLESTEP), + shift_singlestep = const(SHIFT_SINGLESTEP), pti_unmap = sym pti::unmap, user_data_seg_selector = const(gdt::GDT_USER_DATA << 3 | 3), - user_tls_seg_selector = const(gdt::GDT_USER_TLS << 3 | 3), + + MSR_FSBASE = const(x86::msr::IA32_FS_BASE), + MSR_GSBASE = const(x86::msr::IA32_GS_BASE), + options(noreturn), ); } diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index b8959658ef..bebc8a2aba 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -36,6 +36,16 @@ pub struct Context { rbp: usize, /// Stack pointer rsp: usize, + /// FSBASE. + /// + /// NOTE: Same fsgsbase behavior as with gsbase. + pub(crate) fsbase: usize, + /// GSBASE. + /// + /// NOTE: Without fsgsbase, this register will strictly be equal to the register value when + /// running. With fsgsbase, this is neither saved nor restored upon every syscall (there is no + /// need to!), and thus it must be re-read from the register before copying this struct. + pub(crate) gsbase: usize, /// FX valid? loadable: AbiCompatBool, } @@ -60,7 +70,9 @@ impl Context { r14: 0, r15: 0, rbp: 0, - rsp: 0 + rsp: 0, + fsbase: 0, + gsbase: 0, } } @@ -138,6 +150,56 @@ impl Context { } } +macro_rules! load_msr( + ($name:literal, $offset:literal) => { + concat!(" + mov ecx, {", $name, "} + mov rdx, [rsi + {", $offset, "}] + mov eax, edx + shr rdx, 32 + + // MSR <= EDX:EAX + wrmsr + ") + } +); + +// NOTE: RAX is a scratch register and can be set to whatever. There is also no return +// value in switch_to, to it will also never be read. The same goes for RDX, and RCX. +// TODO: Use runtime code patching (perhaps in the bootloader) by pushing alternative code +// sequences into a specialized section, with some macro resembling Linux's `.ALTERNATIVE`. +#[cfg(feature = "x86_fsgsbase")] +macro_rules! switch_fsgsbase( + () => { + " + // placeholder: {MSR_FSBASE} {MSR_KERNELGSBASE} + + rdfsbase rax + mov [rdi + {off_fsbase}], rax + mov rax, [rsi + {off_fsbase}] + wrfsbase rax + + swapgs + rdgsbase rax + mov [rdi + {off_gsbase}], rax + mov rax, [rsi + {off_gsbase}] + wrgsbase rax + swapgs + " + } +); + +#[cfg(not(feature = "x86_fsgsbase"))] +macro_rules! switch_fsgsbase( + () => { + concat!( + load_msr!("MSR_FSBASE", "off_fsbase"), + load_msr!("MSR_KERNELGSBASE", "off_gsbase"), + ) + } +); + + /// Switch to the next context by restoring its stack and registers /// Check disassembly! #[inline(never)] @@ -152,7 +214,7 @@ pub unsafe extern "C" fn switch_to(_prev: &mut Context, _next: &mut Context) { // - we can modify scratch registers, e.g. rax // - we cannot change callee-preserved registers arbitrarily, e.g. rbx, which is why we // store them here in the first place. - " + concat!(" // load `prev.fx` mov rax, [rdi + {off_fx}] @@ -163,26 +225,26 @@ pub unsafe extern "C" fn switch_to(_prev: &mut Context, _next: &mut Context) { mov BYTE PTR [rdi + {off_loadable}], {true} // compare `next.loadable` with true cmp BYTE PTR [rsi + {off_loadable}], {true} - je switch_to.next_is_loadable + je 3f fninit - jmp switch_to.after_fx + jmp 3f - switch_to.next_is_loadable: +2: mov rax, [rsi + {off_fx}] fxrstor64 [rax] - switch_to.after_fx: +3: // Save the current CR3, and load the next CR3 if not identical mov rcx, cr3 mov [rdi + {off_cr3}], rcx mov rax, [rsi + {off_cr3}] cmp rax, rcx - je switch_to.same_cr3 + je 4f mov cr3, rax - switch_to.same_cr3: +4: // Save old registers, and load new ones mov [rdi + {off_rbx}], rbx mov rbx, [rsi + {off_rbx}] @@ -205,6 +267,10 @@ pub unsafe extern "C" fn switch_to(_prev: &mut Context, _next: &mut Context) { mov [rdi + {off_rsp}], rsp mov rsp, [rsi + {off_rsp}] + ", + switch_fsgsbase!(), + " + // push RFLAGS (can only be modified via stack) pushfq // pop RFLAGS into `self.rflags` @@ -222,7 +288,7 @@ pub unsafe extern "C" fn switch_to(_prev: &mut Context, _next: &mut Context) { // Note that switch_finish_hook will be responsible for executing `ret`. jmp {switch_hook} - ", + "), off_fx = const(offset_of!(Cx, fx)), off_cr3 = const(offset_of!(Cx, cr3)), @@ -237,12 +303,17 @@ pub unsafe extern "C" fn switch_to(_prev: &mut Context, _next: &mut Context) { off_rbp = const(offset_of!(Cx, rbp)), off_rsp = const(offset_of!(Cx, rsp)), + off_fsbase = const(offset_of!(Cx, fsbase)), + off_gsbase = const(offset_of!(Cx, gsbase)), + + MSR_FSBASE = const(x86::msr::IA32_FS_BASE), + MSR_KERNELGSBASE = const(x86::msr::IA32_KERNEL_GSBASE), + true = const(AbiCompatBool::True as u8), switch_hook = sym crate::context::switch_finish_hook, options(noreturn), ); } - #[allow(dead_code)] #[repr(packed)] pub struct SignalHandlerStack { diff --git a/src/context/context.rs b/src/context/context.rs index 299e3032af..bf34057d31 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -9,6 +9,7 @@ use core::{ alloc::{GlobalAlloc, Layout}, cmp::Ordering, mem, + ptr::NonNull, }; use spin::RwLock; @@ -20,7 +21,9 @@ use crate::context::memory::{UserGrants, Memory, SharedMemory, Tls}; use crate::ipi::{ipi, IpiKind, IpiTarget}; use crate::scheme::{SchemeNamespace, FileHandle}; use crate::sync::WaitMap; + use crate::syscall::data::SigAction; +use crate::syscall::error::{Result, Error, ENOMEM}; use crate::syscall::flag::{SIG_DFL, SigActionFlags}; /// Unique identifier for a context (i.e. `pid`). @@ -203,9 +206,9 @@ pub struct Context { /// Current system call pub syscall: Option<(usize, usize, usize, usize, usize, usize)>, /// Head buffer to use when system call buffers are not page aligned - pub syscall_head: Box<[u8]>, + pub syscall_head: AlignedBox<[u8; PAGE_SIZE], PAGE_SIZE>, /// Tail buffer to use when system call buffers are not page aligned - pub syscall_tail: Box<[u8]>, + pub syscall_tail: AlignedBox<[u8; PAGE_SIZE], PAGE_SIZE>, /// Context is halting parent pub vfork: bool, /// Context is being waited on @@ -230,8 +233,6 @@ pub struct Context { pub stack: Option, /// User signal stack pub sigstack: Option, - /// User Thread local storage - pub tls: Option, /// User grants pub grants: Arc>, /// The name of the context @@ -253,12 +254,63 @@ pub struct Context { pub ptrace_stop: bool } -impl Context { - pub fn new(id: ContextId) -> Context { - let syscall_head = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(PAGE_SIZE, PAGE_SIZE)) as *mut [u8; PAGE_SIZE]) }; - let syscall_tail = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(PAGE_SIZE, PAGE_SIZE)) as *mut [u8; PAGE_SIZE]) }; +// Necessary because GlobalAlloc::dealloc requires the layout to be the same, and therefore Box +// cannot be used for increased alignment directly. +// TODO: move to common? +pub struct AlignedBox { + inner: Unique, +} +pub unsafe trait ValidForZero {} +unsafe impl ValidForZero for [u8; N] {} - Context { +impl AlignedBox { + const LAYOUT: core::alloc::Layout = { + const fn max(a: usize, b: usize) -> usize { + if a > b { a } else { b } + } + + match core::alloc::Layout::from_size_align(mem::size_of::(), max(mem::align_of::(), ALIGN)) { + Ok(l) => l, + Err(_) => panic!("layout validation failed at compile time"), + } + }; + #[inline(always)] + pub fn try_zeroed() -> Result + where + T: ValidForZero, + { + Ok(unsafe { + let ptr = crate::ALLOCATOR.alloc_zeroed(Self::LAYOUT); + if ptr.is_null() { + return Err(Error::new(ENOMEM))?; + } + Self { + inner: Unique::new_unchecked(ptr.cast()), + } + }) + } +} + +impl core::fmt::Debug for AlignedBox { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[aligned box at {:p}, size {} alignment {}]", self.inner.as_ptr(), mem::size_of::(), mem::align_of::()) + } +} +impl Drop for AlignedBox { + fn drop(&mut self) { + unsafe { + core::ptr::drop_in_place(self.inner.as_ptr()); + crate::ALLOCATOR.dealloc(self.inner.as_ptr().cast(), Self::LAYOUT); + } + } +} + +impl Context { + pub fn new(id: ContextId) -> Result { + let syscall_head = AlignedBox::try_zeroed()?; + let syscall_tail = AlignedBox::try_zeroed()?; + + Ok(Context { id, pgid: id, ppid: ContextId::from(0), @@ -290,7 +342,6 @@ impl Context { image: Vec::new(), stack: None, sigstack: None, - tls: None, grants: Arc::new(RwLock::new(UserGrants::default())), name: Arc::new(RwLock::new(String::new().into_boxed_str())), cwd: Arc::new(RwLock::new(String::new())), @@ -305,7 +356,7 @@ impl Context { ); 128])), regs: None, ptrace_stop: false - } + }) } /// Make a relative path absolute diff --git a/src/context/list.rs b/src/context/list.rs index 90dd566f9a..5ae63cacd9 100644 --- a/src/context/list.rs +++ b/src/context/list.rs @@ -69,7 +69,7 @@ impl ContextList { let id = ContextId::from(self.next_id); self.next_id += 1; - assert!(self.map.insert(id, Arc::new(RwLock::new(Context::new(id)))).is_none()); + assert!(self.map.insert(id, Arc::new(RwLock::new(Context::new(id)?))).is_none()); Ok(self.map.get(&id).expect("Failed to insert new context. ID is out of bounds.")) } diff --git a/src/context/signal.rs b/src/context/signal.rs index 3fe16870c4..ae6b2529e0 100644 --- a/src/context/signal.rs +++ b/src/context/signal.rs @@ -122,7 +122,7 @@ pub extern "C" fn signal_handler(sig: usize) { sp -= mem::size_of::(); *(sp as *mut usize) = restorer; - usermode(handler, sp, sig, u32::from(singlestep)); + usermode(handler, sp, sig, usize::from(singlestep)); } } diff --git a/src/context/switch.rs b/src/context/switch.rs index a823854cd2..9d7c38c1a9 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -173,7 +173,6 @@ pub unsafe fn switch() -> bool { if let Some(ref stack) = to_context.kstack { gdt::set_tss_stack(stack.as_ptr() as usize + stack.len()); } - gdt::set_tcb(to_context.id.into()); } #[cfg(target_arch = "aarch64")] { diff --git a/src/elf.rs b/src/elf.rs index 4914a392c0..2a74ea9226 100644 --- a/src/elf.rs +++ b/src/elf.rs @@ -82,6 +82,12 @@ impl<'a> Elf<'a> { pub fn program_headers(&self) -> usize { self.header.e_phoff as usize } + pub fn program_header_count(&self) -> usize { + self.header.e_phnum as usize + } + pub fn program_headers_size(&self) -> usize { + self.header.e_phentsize as usize + } } pub struct ElfSections<'a> { diff --git a/src/lib.rs b/src/lib.rs index 7f54feb940..6a48b969f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,7 @@ #![feature(concat_idents)] #![feature(const_btree_new)] #![feature(const_maybe_uninit_as_ptr)] +#![feature(const_panic)] #![feature(const_ptr_offset_from)] #![feature(const_raw_ptr_deref)] #![feature(core_intrinsics)] diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 74405b0f43..a42095de08 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -137,6 +137,7 @@ impl SchemeList { //TODO: Only memory: is in the null namespace right now. It should be removed when //anonymous mmap's are implemented self.insert(ns, "memory", |_| Arc::new(MemoryScheme::new())).unwrap(); + self.insert(ns, "thisproc", |_| Arc::new(ProcScheme::restricted())).unwrap(); } /// Initialize a new namespace @@ -168,6 +169,7 @@ impl SchemeList { self.insert(ns, "initfs", |_| Arc::new(InitFsScheme::new())).unwrap(); self.insert(ns, "irq", |scheme_id| Arc::new(IrqScheme::new(scheme_id))).unwrap(); self.insert(ns, "proc", |scheme_id| Arc::new(ProcScheme::new(scheme_id))).unwrap(); + self.insert(ns, "thisproc", |_| Arc::new(ProcScheme::restricted())).unwrap(); self.insert(ns, "serio", |scheme_id| Arc::new(SerioScheme::new(scheme_id))).unwrap(); #[cfg(feature = "live")] { diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 922905a18e..d76a1bb952 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -6,6 +6,7 @@ use crate::{ syscall::{ FloatRegisters, IntRegisters, + EnvRegisters, data::{PtraceEvent, Stat}, error::*, flag::*, @@ -57,6 +58,9 @@ fn try_stop_context(pid: ContextId, mut callback: F) -> Result where F: FnMut(&mut Context) -> Result, { + if pid == context::context_id() { + return Err(Error::new(EBADF)); + } // Stop process let (was_stopped, mut running) = with_context_mut(pid, |context| { let was_stopped = context.ptrace_stop; @@ -88,7 +92,8 @@ where #[derive(Clone, Copy, PartialEq, Eq)] enum RegsKind { Float, - Int + Int, + Env, } #[derive(Clone, Copy, PartialEq, Eq)] enum Operation { @@ -195,6 +200,12 @@ pub static PROC_SCHEME_ID: AtomicSchemeId = AtomicSchemeId::default(); pub struct ProcScheme { next_id: AtomicUsize, handles: RwLock>, + access: Access, +} +#[derive(PartialEq)] +pub enum Access { + OtherProcesses, + Restricted, } impl ProcScheme { @@ -204,6 +215,14 @@ impl ProcScheme { Self { next_id: AtomicUsize::new(0), handles: RwLock::new(BTreeMap::new()), + access: Access::OtherProcesses, + } + } + pub fn restricted() -> Self { + Self { + next_id: AtomicUsize::new(0), + handles: RwLock::new(BTreeMap::new()), + access: Access::Restricted, } } } @@ -211,15 +230,22 @@ impl ProcScheme { impl Scheme for ProcScheme { fn open(&self, path: &str, flags: usize, uid: u32, gid: u32) -> Result { let mut parts = path.splitn(2, '/'); - let pid = parts.next() - .and_then(|s| s.parse().ok()) - .map(ContextId::from) - .ok_or(Error::new(EINVAL))?; + let pid_str = parts.next() + .ok_or(Error::new(ENOENT))?; + + let pid = if pid_str == "current" { + context::context_id() + } else if self.access == Access::Restricted { + return Err(Error::new(EACCES)); + } else { + ContextId::from(pid_str.parse().map_err(|_| Error::new(ENOENT))?) + }; let operation = match parts.next() { Some("mem") => Operation::Memory, Some("regs/float") => Operation::Regs(RegsKind::Float), Some("regs/int") => Operation::Regs(RegsKind::Int), + Some("regs/env") => Operation::Regs(RegsKind::Env), Some("trace") => Operation::Trace, Some("exe") => Operation::Static("exe"), _ => return Err(Error::new(EINVAL)) @@ -382,7 +408,8 @@ impl Scheme for ProcScheme { Operation::Regs(kind) => { union Output { float: FloatRegisters, - int: IntRegisters + int: IntRegisters, + env: EnvRegisters, } let (output, size) = match kind { @@ -406,7 +433,37 @@ impl Scheme for ProcScheme { stack.save(&mut regs); Ok((Output { int: regs }, mem::size_of::())) } - })? + })?, + RegsKind::Env => { + let (fsbase, gsbase) = if info.pid == context::context_id() { + #[cfg(not(feature = "x86_fsgsbase"))] + unsafe { + ( + x86::msr::rdmsr(x86::msr::IA32_FS_BASE), + x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE), + ) + } + #[cfg(feature = "x86_fsgsbase")] + unsafe { + use x86::bits64::segmentation::*; + + ( + rdfsbase(), + { + swapgs(); + let gsbase = rdgsbase(); + swapgs(); + gsbase + } + ) + } + } else { + try_stop_context(info.pid, |context| { + Ok((context.arch.fsbase as u64, context.arch.gsbase as u64)) + })? + }; + (Output { env: EnvRegisters { fsbase, gsbase }}, mem::size_of::()) + } }; let bytes = unsafe { @@ -503,6 +560,9 @@ impl Scheme for ProcScheme { if buf.len() < mem::size_of::() { return Ok(0); } + if (buf.as_ptr() as usize) % mem::align_of::() != 0 { + return Err(Error::new(EINVAL)); + } let regs = unsafe { *(buf as *const _ as *const FloatRegisters) }; @@ -521,6 +581,9 @@ impl Scheme for ProcScheme { if buf.len() < mem::size_of::() { return Ok(0); } + if (buf.as_ptr() as usize) % mem::align_of::() != 0 { + return Err(Error::new(EINVAL)); + } let regs = unsafe { *(buf as *const _ as *const IntRegisters) }; @@ -537,6 +600,57 @@ impl Scheme for ProcScheme { } }) } + RegsKind::Env => { + if buf.len() < mem::size_of::() { + return Ok(0); + } + if (buf.as_ptr() as usize) % mem::align_of::() != 0 { + return Err(Error::new(EINVAL)); + } + let regs = unsafe { + *(buf as *const _ as *const EnvRegisters) + }; + use rmm::{Arch as _, X8664Arch}; + if !(X8664Arch::virt_is_valid(VirtualAddress::new(regs.fsbase as usize)) && X8664Arch::virt_is_valid(VirtualAddress::new(regs.gsbase as usize))) { + return Err(Error::new(EINVAL)); + } + + if info.pid == context::context_id() { + #[cfg(not(feature = "x86_fsgsbase"))] + unsafe { + x86::msr::wrmsr(x86::msr::IA32_FS_BASE, regs.fsbase); + // We have to write to KERNEL_GSBASE, because when the kernel returns to + // userspace, it will have executed SWAPGS first. + x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, regs.gsbase); + + match context::contexts().current().ok_or(Error::new(ESRCH))?.write().arch { + ref mut arch => { + arch.fsbase = regs.fsbase as usize; + arch.gsbase = regs.gsbase as usize; + } + } + } + #[cfg(feature = "x86_fsgsbase")] + unsafe { + use x86::bits64::segmentation::*; + + wrfsbase(regs.fsbase); + swapgs(); + wrgsbase(regs.gsbase); + swapgs(); + + // No need to update the current context; with fsgsbase enabled, these + // registers are automatically saved and restored. + } + } else { + try_stop_context(info.pid, |context| { + context.arch.fsbase = regs.fsbase as usize; + context.arch.gsbase = regs.gsbase as usize; + Ok(()) + })?; + } + Ok(mem::size_of::()) + } }, Operation::Trace => { if buf.len() < mem::size_of::() { @@ -621,6 +735,7 @@ impl Scheme for ProcScheme { Operation::Memory => "mem", Operation::Regs(RegsKind::Float) => "regs/float", Operation::Regs(RegsKind::Int) => "regs/int", + Operation::Regs(RegsKind::Env) => "regs/env", Operation::Trace => "trace", Operation::Static(path) => path, }); diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index b90dd54d6e..54c66e19a3 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -7,6 +7,7 @@ extern crate syscall; pub use self::syscall::{ FloatRegisters, IntRegisters, + EnvRegisters, data, error, flag, diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 850dfebb3c..97d613f0e2 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -27,7 +27,7 @@ use crate::scheme::FileHandle; use crate::start::usermode; use crate::syscall::data::{SigAction, Stat}; use crate::syscall::error::*; -use crate::syscall::flag::{wifcontinued, wifstopped, AT_ENTRY, AT_NULL, AT_PHDR, CloneFlags, +use crate::syscall::flag::{wifcontinued, wifstopped, AT_ENTRY, AT_NULL, AT_PHDR, AT_PHENT, AT_PHNUM, CloneFlags, CLONE_FILES, CLONE_FS, CLONE_SIGHAND, CLONE_STACK, CLONE_VFORK, CLONE_VM, MapFlags, PROT_EXEC, PROT_READ, PROT_WRITE, PTRACE_EVENT_CLONE, PTRACE_STOP_EXIT, SigActionFlags, SIG_BLOCK, SIG_DFL, SIG_SETMASK, SIG_UNBLOCK, @@ -57,7 +57,6 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result { let mut image = vec![]; let mut stack_opt = None; let mut sigstack_opt = None; - let mut tls_opt = None; let grants; let name; let cwd; @@ -202,36 +201,6 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result { sigstack_opt = Some(new_sigstack); } - if let Some(ref tls) = context.tls { - let mut new_tls = context::memory::Tls { - master: tls.master, - file_size: tls.file_size, - mem: context::memory::Memory::new( - VirtualAddress::new(crate::USER_TMP_TLS_OFFSET), - tls.mem.size(), - PageFlags::new().write(true), - true - ), - offset: tls.offset, - }; - - - if flags.contains(CLONE_VM) { - unsafe { - new_tls.load(); - } - } else { - unsafe { - intrinsics::copy(tls.mem.start_address().data() as *const u8, - new_tls.mem.start_address().data() as *mut u8, - tls.mem.size()); - } - } - - new_tls.mem.remap(tls.mem.flags()); - tls_opt = Some(new_tls); - } - if flags.contains(CLONE_VM) { grants = Arc::clone(&context.grants); } else { @@ -352,6 +321,14 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result { context.arch = arch; + // This is needed because these registers may have changed after this context was + // switched to, but before this was called. + #[cfg(all(target_arch = "x86_64", feature = "x86_fsgsbase"))] + unsafe { + context.arch.fsbase = x86::bits64::segmentation::rdfsbase() as usize; + context.arch.gsbase = x86::bits64::segmentation::rdgsbase() as usize; + } + let mut active_utable = unsafe { ActivePageTable::new(TableKind::User) }; let mut active_ktable = unsafe { ActivePageTable::new(TableKind::Kernel) }; @@ -498,15 +475,6 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result { context.sigstack = Some(sigstack); } - // Set up TCB - let tcb_addr = crate::USER_TCB_OFFSET + context.id.into() * PAGE_SIZE; - let mut tcb = context::memory::Memory::new( - VirtualAddress::new(tcb_addr), - PAGE_SIZE, - PageFlags::new().write(true).user(true), - true - ); - #[cfg(target_arch = "aarch64")] { if let Some(stack) = &mut context.kstack { @@ -530,38 +498,6 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result { } } - // Setup user TLS - if let Some(mut tls) = tls_opt { - // Copy TLS mapping - { - let frame = active_utable.p4()[crate::USER_TLS_PML4].pointed_frame().expect("user tls not mapped"); - let flags = active_utable.p4()[crate::USER_TLS_PML4].flags(); - active_utable.with(&mut new_utable, &mut temporary_upage, |mapper| { - mapper.p4_mut()[crate::USER_TLS_PML4].set(frame, flags); - }); - } - - // TODO: Make sure size is not greater than USER_TLS_SIZE - let tls_addr = crate::USER_TLS_OFFSET + context.id.into() * crate::USER_TLS_SIZE; - //println!("{}: Copy TLS: address 0x{:x}, size 0x{:x}", context.id.into(), tls_addr, tls.mem.size()); - tls.mem.move_to(VirtualAddress::new(tls_addr), &mut new_utable, &mut temporary_upage); - unsafe { - *(tcb_addr as *mut usize) = tls.mem.start_address().data() + tls.mem.size(); - } - context.tls = Some(tls); - } else { - //println!("{}: Copy TCB", context.id.into()); - let parent_tcb_addr = crate::USER_TCB_OFFSET + ppid.into() * PAGE_SIZE; - unsafe { - intrinsics::copy(parent_tcb_addr as *const u8, - tcb_addr as *mut u8, - tcb.size()); - } - } - - tcb.move_to(VirtualAddress::new(tcb_addr), &mut new_utable, &mut temporary_upage); - context.image.push(tcb.to_shared()); - context.name = name; context.cwd = cwd; @@ -595,13 +531,11 @@ fn empty(context: &mut context::Context, reaping: bool) { assert!(context.image.is_empty()); assert!(context.stack.is_none()); assert!(context.sigstack.is_none()); - assert!(context.tls.is_none()); } else { - // Unmap previous image, heap, grants, stack, and tls + // Unmap previous image, heap, grants, stack context.image.clear(); drop(context.stack.take()); drop(context.sigstack.take()); - drop(context.tls.take()); } // NOTE: If we do not replace the grants `Arc`, then a strange situation can appear where the @@ -647,11 +581,13 @@ impl Drop for ExecFile { } } +#[allow(clippy::too_many_arguments)] fn fexec_noreturn( setuid: Option, setgid: Option, name: Box, data: Box<[u8]>, + phdr_grant: context::memory::Grant, args: Box<[Box<[u8]>]>, vars: Box<[Box<[u8]>]>, auxv: Box<[usize]>, @@ -674,6 +610,27 @@ fn fexec_noreturn( empty(&mut context, false); + context.grants.write().insert(phdr_grant); + + #[cfg(all(target_arch = "x86_64"))] + { + context.arch.fsbase = 0; + context.arch.gsbase = 0; + + #[cfg(feature = "x86_fsgsbase")] + unsafe { + x86::bits64::segmentation::wrfsbase(0); + x86::bits64::segmentation::swapgs(); + x86::bits64::segmentation::wrgsbase(0); + x86::bits64::segmentation::swapgs(); + } + #[cfg(not(feature = "x86_fsgsbase"))] + unsafe { + x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0); + x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0); + } + } + if let Some(uid) = setuid { context.euid = uid; } @@ -683,20 +640,10 @@ fn fexec_noreturn( } // Map and copy new segments - let mut tls_opt = None; { let elf = elf::Elf::from(&data).unwrap(); entry = elf.entry(); - // Always map TCB - let tcb_addr = crate::USER_TCB_OFFSET + context.id.into() * PAGE_SIZE; - let tcb_mem = context::memory::Memory::new( - VirtualAddress::new(tcb_addr), - PAGE_SIZE, - PageFlags::new().write(true).user(true), - true - ); - for segment in elf.segments() { match segment.p_type { program_header::PT_LOAD => { @@ -730,45 +677,11 @@ fn fexec_noreturn( context.image.push(memory.to_shared()); }, - program_header::PT_TLS => { - let aligned_size = if segment.p_align > 0 { - ((segment.p_memsz + (segment.p_align - 1))/segment.p_align) * segment.p_align - } else { - segment.p_memsz - } as usize; - let rounded_size = ((aligned_size + PAGE_SIZE - 1)/PAGE_SIZE) * PAGE_SIZE; - let rounded_offset = rounded_size - aligned_size; - - // TODO: Make sure size is not greater than USER_TLS_SIZE - let tls_addr = crate::USER_TLS_OFFSET + context.id.into() * crate::USER_TLS_SIZE; - let tls = context::memory::Tls { - master: VirtualAddress::new(segment.p_vaddr as usize), - file_size: segment.p_filesz as usize, - mem: context::memory::Memory::new( - VirtualAddress::new(tls_addr), - rounded_size as usize, - PageFlags::new().write(true).user(true), - true - ), - offset: rounded_offset as usize, - }; - - unsafe { - *(tcb_addr as *mut usize) = tls.mem.start_address().data() + tls.mem.size(); - } - - tls_opt = Some(tls); - }, _ => (), } } - - context.image.push(tcb_mem.to_shared()); } - // Data no longer required, can deallocate - drop(data); - // Map stack context.stack = Some(context::memory::Memory::new( VirtualAddress::new(crate::USER_STACK_OFFSET), @@ -785,14 +698,8 @@ fn fexec_noreturn( true )); - // Map TLS - if let Some(mut tls) = tls_opt { - unsafe { - tls.load(); - } - - context.tls = Some(tls); - } + // Data no longer required, can deallocate + drop(data); let mut push = |arg| { sp -= mem::size_of::(); @@ -904,10 +811,10 @@ fn fexec_noreturn( } // Go to usermode - unsafe { usermode(entry, sp, 0, u32::from(singlestep)) } + unsafe { usermode(entry, sp, 0, usize::from(singlestep)) } } -pub fn fexec_kernel(fd: FileHandle, args: Box<[Box<[u8]>]>, vars: Box<[Box<[u8]>]>, name_override_opt: Option>, auxv: Option>) -> Result { +pub fn fexec_kernel(fd: FileHandle, args: Box<[Box<[u8]>]>, vars: Box<[Box<[u8]>]>, name_override_opt: Option>, auxv: Option<(Vec, context::memory::Grant)>) -> Result { let (uid, gid) = { let contexts = context::contexts(); let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; @@ -1007,17 +914,42 @@ pub fn fexec_kernel(fd: FileHandle, args: Box<[Box<[u8]>]>, vars: Box<[Box<[u8]> // `fexec_kernel` can recurse if an interpreter is found. We get the // auxiliary vector from the first invocation, which is passed via an // argument, or if this is the first one we create it. - let auxv = if let Some(auxv) = auxv { - auxv + let (auxv, phdr_grant) = if let Some((auxv, phdr_grant)) = auxv { + (auxv, phdr_grant) } else { + let phdr_grant = match context::contexts().current().ok_or(Error::new(ESRCH))?.read().grants.write() { + mut grants => { + let size = elf.program_headers_size() * elf.program_header_count(); + let aligned_size = (size + PAGE_SIZE - 1) / PAGE_SIZE * PAGE_SIZE; + + if aligned_size > MAX_PHDRS_SIZE { + return Err(Error::new(ENOMEM)); + } + + let phdrs_region = grants.find_free(aligned_size); + let grant = context::memory::Grant::map(phdrs_region.start_address(), aligned_size, PageFlags::new().write(true).user(true)); + + unsafe { + let dst = core::slice::from_raw_parts_mut(grant.start_address().data() as *mut u8, aligned_size); + dst[..size].copy_from_slice(&data[elf.program_headers()..elf.program_headers() + elf.program_headers_size() * elf.program_header_count()]); + } + + grant + } + + }; let mut auxv = Vec::with_capacity(3); auxv.push(AT_ENTRY); auxv.push(elf.entry()); auxv.push(AT_PHDR); - auxv.push(elf.program_headers()); + auxv.push(phdr_grant.start_address().data()); + auxv.push(AT_PHENT); + auxv.push(elf.program_headers_size()); + auxv.push(AT_PHNUM); + auxv.push(elf.program_header_count()); - auxv + (auxv, phdr_grant) }; // We check the validity of all loadable sections here @@ -1061,21 +993,9 @@ pub fn fexec_kernel(fd: FileHandle, args: Box<[Box<[u8]>]>, vars: Box<[Box<[u8]> args_vec.into_boxed_slice(), vars, Some(name_override), - Some(auxv), + Some((auxv, phdr_grant)), ); }, - program_header::PT_LOAD => { - let voff = segment.p_vaddr as usize % PAGE_SIZE; - let vaddr = segment.p_vaddr as usize - voff; - - // Due to the Userspace and kernel TLS bases being located right above 2GB, - // limit any loadable sections to lower than that. Eventually we will need - // to replace this with a more intelligent TLS address - if vaddr >= 0x8000_0000 { - println!("exec: invalid section address {:X}", segment.p_vaddr); - return Err(Error::new(ENOEXEC)); - } - }, _ => (), } } @@ -1083,8 +1003,9 @@ pub fn fexec_kernel(fd: FileHandle, args: Box<[Box<[u8]>]>, vars: Box<[Box<[u8]> // This is the point of no return, quite literaly. Any checks for validity need // to be done before, and appropriate errors returned. Otherwise, we have nothing // to return to. - fexec_noreturn(setuid, setgid, name.into_boxed_str(), data.into_boxed_slice(), args, vars, auxv.into_boxed_slice()); + fexec_noreturn(setuid, setgid, name.into_boxed_str(), data.into_boxed_slice(), phdr_grant, args, vars, auxv.into_boxed_slice()); } +const MAX_PHDRS_SIZE: usize = PAGE_SIZE; pub fn fexec(fd: FileHandle, arg_ptrs: &[[usize; 2]], var_ptrs: &[[usize; 2]]) -> Result { let mut args = Vec::new(); diff --git a/syscall b/syscall index 841b5f4221..14796acfed 160000 --- a/syscall +++ b/syscall @@ -1 +1 @@ -Subproject commit 841b5f42216782ce2aee6201c55b849ce5d7ab71 +Subproject commit 14796acfed7dd6fc353b18c001d3798c3aed03bf