From 4862977fa545f69c5d13f8008f1cb50e332d590a Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Mon, 18 Mar 2024 20:38:11 +0100 Subject: [PATCH] Optimize syscall fast path without tracing. --- src/arch/aarch64/misc.rs | 8 +--- src/arch/x86/gdt.rs | 8 +--- src/arch/x86_64/gdt.rs | 12 +----- src/arch/x86_64/interrupt/syscall.rs | 57 +++++----------------------- src/arch/x86_shared/idt.rs | 11 ++++-- src/context/arch/x86_64.rs | 4 +- src/context/switch.rs | 15 +++++--- src/percpu.rs | 23 ++++++++++- src/ptrace.rs | 34 ++++++----------- src/syscall/process.rs | 5 +-- 10 files changed, 69 insertions(+), 108 deletions(-) diff --git a/src/arch/aarch64/misc.rs b/src/arch/aarch64/misc.rs index ed9edbdfb2..7f288261c6 100644 --- a/src/arch/aarch64/misc.rs +++ b/src/arch/aarch64/misc.rs @@ -18,13 +18,7 @@ pub unsafe fn init(cpu_id: LogicalCpuId) { let frame = crate::memory::allocate_frames(1).expect("failed to allocate percpu memory"); let virt = RmmA::phys_to_virt(frame.start_address()).data() as *mut PercpuBlock; - virt.write(PercpuBlock { - cpu_id, - switch_internals: crate::context::switch::ContextSwitchPercpu::default(), - current_addrsp: RefCell::new(None), - new_addrsp_tmp: Cell::new(None), - wants_tlb_shootdown: AtomicBool::new(false), - }); + virt.write(PercpuBlock::init(cpu_id)); crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64); } diff --git a/src/arch/x86/gdt.rs b/src/arch/x86/gdt.rs index 5e2ba33182..3c4d0afed3 100644 --- a/src/arch/x86/gdt.rs +++ b/src/arch/x86/gdt.rs @@ -222,13 +222,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); - pcr.percpu = crate::percpu::PercpuBlock { - cpu_id, - switch_internals: Default::default(), - current_addrsp: RefCell::new(None), - new_addrsp_tmp: Cell::new(None), - wants_tlb_shootdown: AtomicBool::new(false), - }; + pcr.percpu = crate::percpu::PercpuBlock::init(cpu_id); crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu); } diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index d52f469255..150890a8cc 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -120,6 +120,7 @@ pub struct ProcessorControlRegion { // to correctly obtain GSBASE, uses SGDT to calculate the PCR offset. pub gdt: [GdtEntry; 8], pub percpu: PercpuBlock, + _rsvd: usize, pub tss: TaskStateSegment, // These two fields are read by the CPU, but not currently modified by the kernel. Instead, the @@ -254,16 +255,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); - pcr.percpu = PercpuBlock { - cpu_id, - switch_internals: Default::default(), - current_addrsp: RefCell::new(None), - new_addrsp_tmp: Cell::new(None), - wants_tlb_shootdown: AtomicBool::new(false), - - #[cfg(feature = "profiling")] - profiling: None, - }; + pcr.percpu = PercpuBlock::init(cpu_id); crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu); } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 390fb0b04a..f533fa6ec3 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -60,24 +60,14 @@ pub unsafe fn init() { msr::wrmsr(msr::IA32_EFER, efer | 1); } -macro_rules! with_interrupt_stack { - (|$stack:ident| $code:block) => {{ - let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None) - .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE))); - - if allowed.unwrap_or(true) { - let $stack = &mut *$stack; - $code - } - - ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); - }}; -} - #[no_mangle] pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) { - with_interrupt_stack!(|stack| { - let scratch = &stack.scratch; + let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None) + .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE))); + + if allowed.unwrap_or(true) { + let scratch = &(*stack).scratch; + syscall::syscall( scratch.rax, scratch.rdi, @@ -85,9 +75,11 @@ pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) scratch.rdx, scratch.r10, scratch.r8, - stack, + &mut *stack, ); - }); + } + + ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); } #[naked] @@ -205,32 +197,3 @@ extern "C" { // TODO: macro? pub fn enter_usermode(); } - -interrupt_stack!(syscall, |stack| { - with_interrupt_stack!(|stack| { - { - let contexts = context::contexts(); - let context = contexts.current(); - if let Some(current) = context { - let current = current.read(); - println!( - "Warning: Context {} used deprecated `int 0x80` construct", - current.name - ); - } else { - println!("Warning: Unknown context used deprecated `int 0x80` construct"); - } - } - - let scratch = &stack.scratch; - syscall::syscall( - scratch.rax, - stack.preserved.rbx, - scratch.rcx, - scratch.rdx, - scratch.rsi, - scratch.rdi, - stack, - ); - }) -}); diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 2c65c19dda..f15a982a28 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -313,10 +313,13 @@ pub unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) { idt.set_reserved_mut(IpiKind::Pit as u8, true); let current_idt = &mut idt.entries; - // Set syscall function - current_idt[0x80].set_func(syscall::syscall); - current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); - idt.set_reserved_mut(0x80, true); + #[cfg(target_arch = "x86")] + { + // Set syscall function + current_idt[0x80].set_func(syscall::syscall); + current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); + idt.set_reserved_mut(0x80, true); + } #[cfg(feature = "profiling")] crate::profiling::maybe_setup_timer(idt, cpu_id); diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 1c08fe387e..669207c284 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -3,10 +3,12 @@ use core::{ sync::atomic::AtomicBool, }; -use crate::{syscall::FloatRegisters, percpu::PercpuBlock}; +use crate::{percpu::PercpuBlock, ptrace, syscall::FloatRegisters}; use core::mem::offset_of; +use alloc::sync::Arc; use spin::Once; +use syscall::PtraceFlags; use x86::msr; /// This must be used by the kernel to ensure that context switches are done atomically diff --git a/src/context/switch.rs b/src/context/switch.rs index d6924dbc4a..a196faf215 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -1,13 +1,11 @@ use core::{cell::Cell, mem, ops::Bound, sync::atomic::Ordering}; +use alloc::sync::Arc; use spinning_top::guard::ArcRwSpinlockWriteGuard; +use syscall::PtraceFlags; use crate::{ - context::{arch, contexts, Context}, - cpu_set::LogicalCpuId, - interrupt, - percpu::PercpuBlock, - time, + context::{arch, contexts, Context}, cpu_set::LogicalCpuId, interrupt, percpu::PercpuBlock, ptrace, time }; use super::{ContextId, Status}; @@ -210,6 +208,13 @@ pub fn switch() -> SwitchResult { _next_guard: next_context_guard, })); + let (ptrace_session, ptrace_flags) = if let Some((session, bp)) = ptrace::sessions().get(&next_context.id).map(|s| (Arc::downgrade(s), s.data.lock().breakpoint)) { + (Some(session), bp.map_or(PtraceFlags::empty(), |f| f.flags)) + } else { (None, PtraceFlags::empty()) }; + + *percpu.ptrace_session.borrow_mut() = ptrace_session; + percpu.ptrace_flags.set(ptrace_flags); + unsafe { arch::switch_to(prev_context, next_context); } diff --git a/src/percpu.rs b/src/percpu.rs index 69e15548ef..15982188b9 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -1,12 +1,14 @@ use core::cell::{Cell, RefCell}; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; -use alloc::sync::Arc; +use alloc::sync::{Arc, Weak}; use rmm::Arch; +use syscall::PtraceFlags; use crate::context::empty_cr3; use crate::context::memory::AddrSpaceWrapper; use crate::cpu_set::MAX_CPU_COUNT; +use crate::ptrace::Session; use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId}; /// The percpu block, that stored all percpu variables. @@ -25,6 +27,9 @@ pub struct PercpuBlock { // first to avoid cache invalidation. #[cfg(feature = "profiling")] pub profiling: Option<&'static crate::profiling::RingBuffer>, + + pub ptrace_flags: Cell, + pub ptrace_session: RefCell>>, } const NULL: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); @@ -115,3 +120,19 @@ pub unsafe fn switch_arch_hook() { crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3()); } } +impl PercpuBlock { + pub fn init(cpu_id: LogicalCpuId) -> Self { + Self { + cpu_id, + switch_internals: Default::default(), + current_addrsp: RefCell::new(None), + new_addrsp_tmp: Cell::new(None), + wants_tlb_shootdown: AtomicBool::new(false), + ptrace_flags: Cell::new(Default::default()), + ptrace_session: RefCell::new(None), + + #[cfg(feature = "profiling")] + profiling: None, + } + } +} diff --git a/src/ptrace.rs b/src/ptrace.rs index 13073e4627..e24c5ca4b7 100644 --- a/src/ptrace.rs +++ b/src/ptrace.rs @@ -3,11 +3,7 @@ //! of the scheme. use crate::{ - context::{self, ContextId}, - event, - scheme::GlobalSchemes, - sync::WaitCondition, - syscall::{data::PtraceEvent, error::*, flag::*, ptrace_event}, + context::{self, ContextId}, event, percpu::PercpuBlock, scheme::GlobalSchemes, sync::WaitCondition, syscall::{data::PtraceEvent, error::*, flag::*, ptrace_event} }; use alloc::{collections::VecDeque, sync::Arc}; @@ -23,7 +19,7 @@ use spin::{Mutex, Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; #[derive(Debug)] pub struct SessionData { - breakpoint: Option, + pub(crate) breakpoint: Option, events: VecDeque, file_id: usize, } @@ -103,7 +99,7 @@ static SESSIONS: Once> = Once::new(); fn init_sessions() -> RwLock { RwLock::new(HashMap::new()) } -fn sessions() -> RwLockReadGuard<'static, SessionMap> { +pub(crate) fn sessions() -> RwLockReadGuard<'static, SessionMap> { SESSIONS.call_once(init_sessions).read() } fn sessions_mut() -> RwLockWriteGuard<'static, SessionMap> { @@ -202,9 +198,9 @@ pub fn send_event(event: PtraceEvent) -> Option<()> { // |_| #[derive(Debug, Clone, Copy)] -struct Breakpoint { +pub(crate) struct Breakpoint { reached: bool, - flags: PtraceFlags, + pub(crate) flags: PtraceFlags, } /// Wait for the tracee to stop, or return immediately if there's an unread @@ -254,25 +250,19 @@ pub fn breakpoint_callback( event: Option, ) -> Option { loop { - let session = { - let contexts = context::contexts(); - let context = contexts.current()?; - let context = context.read(); + let percpu = PercpuBlock::current(); - let sessions = sessions(); - let session = sessions.get(&context.id)?; + // TODO: Some or all flags? + // Only stop if the tracer have asked for this breakpoint + if percpu.ptrace_flags.get().contains(match_flags) { + return None; + } - Arc::clone(session) - }; + let session = percpu.ptrace_session.borrow().as_ref()?.upgrade()?; let mut data = session.data.lock(); let breakpoint = data.breakpoint?; // only go to sleep if there's a breakpoint - // Only stop if the tracer have asked for this breakpoint - if breakpoint.flags & match_flags != match_flags { - return None; - } - // In case no tracer is waiting, make sure the next one gets the memo data.breakpoint .as_mut() diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 3ac5346d32..828500c6c0 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -111,10 +111,7 @@ pub fn exit(status: usize) -> ! { } pub fn getpid() -> Result { - let contexts = context::contexts(); - let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; - let context = context_lock.read(); - Ok(context.id) + Ok(context::context_id()) } pub fn getpgid(pid: ContextId) -> Result {