Optimize syscall fast path without tracing.

src/arch/aarch64/misc.rs

@@ -18,13 +18,7 @@ pub unsafe fn init(cpu_id: LogicalCpuId) {
     let frame = crate::memory::allocate_frames(1).expect("failed to allocate percpu memory");
     let virt = RmmA::phys_to_virt(frame.start_address()).data() as *mut PercpuBlock;
 
-    virt.write(PercpuBlock {
-        cpu_id,
-        switch_internals: crate::context::switch::ContextSwitchPercpu::default(),
-        current_addrsp: RefCell::new(None),
-        new_addrsp_tmp: Cell::new(None),
-        wants_tlb_shootdown: AtomicBool::new(false),
-    });
+    virt.write(PercpuBlock::init(cpu_id));
 
     crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64);
 }

src/arch/x86/gdt.rs

@@ -222,13 +222,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
     // Load the task register
     task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
 
-    pcr.percpu = crate::percpu::PercpuBlock {
-        cpu_id,
-        switch_internals: Default::default(),
-        current_addrsp: RefCell::new(None),
-        new_addrsp_tmp: Cell::new(None),
-        wants_tlb_shootdown: AtomicBool::new(false),
-    };
+    pcr.percpu = crate::percpu::PercpuBlock::init(cpu_id);
     crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu);
 }
 

src/arch/x86_64/gdt.rs

@@ -120,6 +120,7 @@ pub struct ProcessorControlRegion {
     // to correctly obtain GSBASE, uses SGDT to calculate the PCR offset.
     pub gdt: [GdtEntry; 8],
     pub percpu: PercpuBlock,
+    _rsvd: usize,
     pub tss: TaskStateSegment,
 
     // These two fields are read by the CPU, but not currently modified by the kernel. Instead, the
@@ -254,16 +255,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
     // Load the task register
     task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
 
-    pcr.percpu = PercpuBlock {
-        cpu_id,
-        switch_internals: Default::default(),
-        current_addrsp: RefCell::new(None),
-        new_addrsp_tmp: Cell::new(None),
-        wants_tlb_shootdown: AtomicBool::new(false),
-
-        #[cfg(feature = "profiling")]
-        profiling: None,
-    };
+    pcr.percpu = PercpuBlock::init(cpu_id);
 
     crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu);
 }

src/arch/x86_64/interrupt/syscall.rs

@@ -60,24 +60,14 @@ pub unsafe fn init() {
     msr::wrmsr(msr::IA32_EFER, efer | 1);
 }
 
-macro_rules! with_interrupt_stack {
-    (|$stack:ident| $code:block) => {{
-        let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None)
-            .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE)));
-
-        if allowed.unwrap_or(true) {
-            let $stack = &mut *$stack;
-            $code
-        }
-
-        ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None);
-    }};
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) {
-    with_interrupt_stack!(|stack| {
-        let scratch = &stack.scratch;
+    let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None)
+        .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE)));
+
+    if allowed.unwrap_or(true) {
+        let scratch = &(*stack).scratch;
+
         syscall::syscall(
             scratch.rax,
             scratch.rdi,
@@ -85,9 +75,11 @@ pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack)
             scratch.rdx,
             scratch.r10,
             scratch.r8,
-            stack,
+            &mut *stack,
         );
-    });
+    }
+
+    ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None);
 }
 
 #[naked]
@@ -205,32 +197,3 @@ extern "C" {
     // TODO: macro?
     pub fn enter_usermode();
 }
-
-interrupt_stack!(syscall, |stack| {
-    with_interrupt_stack!(|stack| {
-        {
-            let contexts = context::contexts();
-            let context = contexts.current();
-            if let Some(current) = context {
-                let current = current.read();
-                println!(
-                    "Warning: Context {} used deprecated `int 0x80` construct",
-                    current.name
-                );
-            } else {
-                println!("Warning: Unknown context used deprecated `int 0x80` construct");
-            }
-        }
-
-        let scratch = &stack.scratch;
-        syscall::syscall(
-            scratch.rax,
-            stack.preserved.rbx,
-            scratch.rcx,
-            scratch.rdx,
-            scratch.rsi,
-            scratch.rdi,
-            stack,
-        );
-    })
-});

src/arch/x86_shared/idt.rs

@@ -313,10 +313,13 @@ pub unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) {
     idt.set_reserved_mut(IpiKind::Pit as u8, true);
     let current_idt = &mut idt.entries;
 
-    // Set syscall function
-    current_idt[0x80].set_func(syscall::syscall);
-    current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT);
-    idt.set_reserved_mut(0x80, true);
+    #[cfg(target_arch = "x86")]
+    {
+        // Set syscall function
+        current_idt[0x80].set_func(syscall::syscall);
+        current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT);
+        idt.set_reserved_mut(0x80, true);
+    }
 
     #[cfg(feature = "profiling")]
     crate::profiling::maybe_setup_timer(idt, cpu_id);

src/context/arch/x86_64.rs

@@ -3,10 +3,12 @@ use core::{
     sync::atomic::AtomicBool,
 };
 
-use crate::{syscall::FloatRegisters, percpu::PercpuBlock};
+use crate::{percpu::PercpuBlock, ptrace, syscall::FloatRegisters};
 
 use core::mem::offset_of;
+use alloc::sync::Arc;
 use spin::Once;
+use syscall::PtraceFlags;
 use x86::msr;
 
 /// This must be used by the kernel to ensure that context switches are done atomically

src/context/switch.rs

@@ -1,13 +1,11 @@
 use core::{cell::Cell, mem, ops::Bound, sync::atomic::Ordering};
 
+use alloc::sync::Arc;
 use spinning_top::guard::ArcRwSpinlockWriteGuard;
+use syscall::PtraceFlags;
 
 use crate::{
-    context::{arch, contexts, Context},
-    cpu_set::LogicalCpuId,
-    interrupt,
-    percpu::PercpuBlock,
-    time,
+    context::{arch, contexts, Context}, cpu_set::LogicalCpuId, interrupt, percpu::PercpuBlock, ptrace, time
 };
 
 use super::{ContextId, Status};
@@ -210,6 +208,13 @@ pub fn switch() -> SwitchResult {
                 _next_guard: next_context_guard,
             }));
 
+        let (ptrace_session, ptrace_flags) = if let Some((session, bp)) = ptrace::sessions().get(&next_context.id).map(|s| (Arc::downgrade(s), s.data.lock().breakpoint)) {
+            (Some(session), bp.map_or(PtraceFlags::empty(), |f| f.flags))
+        } else { (None, PtraceFlags::empty()) };
+
+        *percpu.ptrace_session.borrow_mut() = ptrace_session;
+        percpu.ptrace_flags.set(ptrace_flags);
+
         unsafe {
             arch::switch_to(prev_context, next_context);
         }

src/percpu.rs

@@ -1,12 +1,14 @@
 use core::cell::{Cell, RefCell};
 use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
 
-use alloc::sync::Arc;
+use alloc::sync::{Arc, Weak};
 use rmm::Arch;
+use syscall::PtraceFlags;
 
 use crate::context::empty_cr3;
 use crate::context::memory::AddrSpaceWrapper;
 use crate::cpu_set::MAX_CPU_COUNT;
+use crate::ptrace::Session;
 use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId};
 
 /// The percpu block, that stored all percpu variables.
@@ -25,6 +27,9 @@ pub struct PercpuBlock {
     // first to avoid cache invalidation.
     #[cfg(feature = "profiling")]
     pub profiling: Option<&'static crate::profiling::RingBuffer>,
+
+    pub ptrace_flags: Cell<PtraceFlags>,
+    pub ptrace_session: RefCell<Option<Weak<Session>>>,
 }
 
 const NULL: AtomicPtr<PercpuBlock> = AtomicPtr::new(core::ptr::null_mut());
@@ -115,3 +120,19 @@ pub unsafe fn switch_arch_hook() {
         crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3());
     }
 }
+impl PercpuBlock {
+    pub fn init(cpu_id: LogicalCpuId) -> Self {
+        Self {
+            cpu_id,
+            switch_internals: Default::default(),
+            current_addrsp: RefCell::new(None),
+            new_addrsp_tmp: Cell::new(None),
+            wants_tlb_shootdown: AtomicBool::new(false),
+            ptrace_flags: Cell::new(Default::default()),
+            ptrace_session: RefCell::new(None),
+
+            #[cfg(feature = "profiling")]
+            profiling: None,
+        }
+    }
+}

src/ptrace.rs

@@ -3,11 +3,7 @@
 //! of the scheme.
 
 use crate::{
-    context::{self, ContextId},
-    event,
-    scheme::GlobalSchemes,
-    sync::WaitCondition,
-    syscall::{data::PtraceEvent, error::*, flag::*, ptrace_event},
+    context::{self, ContextId}, event, percpu::PercpuBlock, scheme::GlobalSchemes, sync::WaitCondition, syscall::{data::PtraceEvent, error::*, flag::*, ptrace_event}
 };
 
 use alloc::{collections::VecDeque, sync::Arc};
@@ -23,7 +19,7 @@ use spin::{Mutex, Once, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 #[derive(Debug)]
 pub struct SessionData {
-    breakpoint: Option<Breakpoint>,
+    pub(crate) breakpoint: Option<Breakpoint>,
     events: VecDeque<PtraceEvent>,
     file_id: usize,
 }
@@ -103,7 +99,7 @@ static SESSIONS: Once<RwLock<SessionMap>> = Once::new();
 fn init_sessions() -> RwLock<SessionMap> {
     RwLock::new(HashMap::new())
 }
-fn sessions() -> RwLockReadGuard<'static, SessionMap> {
+pub(crate) fn sessions() -> RwLockReadGuard<'static, SessionMap> {
     SESSIONS.call_once(init_sessions).read()
 }
 fn sessions_mut() -> RwLockWriteGuard<'static, SessionMap> {
@@ -202,9 +198,9 @@ pub fn send_event(event: PtraceEvent) -> Option<()> {
 //                           |_|
 
 #[derive(Debug, Clone, Copy)]
-struct Breakpoint {
+pub(crate) struct Breakpoint {
     reached: bool,
-    flags: PtraceFlags,
+    pub(crate) flags: PtraceFlags,
 }
 
 /// Wait for the tracee to stop, or return immediately if there's an unread
@@ -254,25 +250,19 @@ pub fn breakpoint_callback(
     event: Option<PtraceEvent>,
 ) -> Option<PtraceFlags> {
     loop {
-        let session = {
-            let contexts = context::contexts();
-            let context = contexts.current()?;
-            let context = context.read();
+        let percpu = PercpuBlock::current();
 
-            let sessions = sessions();
-            let session = sessions.get(&context.id)?;
+        // TODO: Some or all flags?
+        // Only stop if the tracer have asked for this breakpoint
+        if percpu.ptrace_flags.get().contains(match_flags) {
+            return None;
+        }
 
-            Arc::clone(session)
-        };
+        let session = percpu.ptrace_session.borrow().as_ref()?.upgrade()?;
 
         let mut data = session.data.lock();
         let breakpoint = data.breakpoint?; // only go to sleep if there's a breakpoint
 
-        // Only stop if the tracer have asked for this breakpoint
-        if breakpoint.flags & match_flags != match_flags {
-            return None;
-        }
-
         // In case no tracer is waiting, make sure the next one gets the memo
         data.breakpoint
             .as_mut()

src/syscall/process.rs

@@ -111,10 +111,7 @@ pub fn exit(status: usize) -> ! {
 }
 
 pub fn getpid() -> Result<ContextId> {
-    let contexts = context::contexts();
-    let context_lock = contexts.current().ok_or(Error::new(ESRCH))?;
-    let context = context_lock.read();
-    Ok(context.id)
+    Ok(context::context_id())
 }
 
 pub fn getpgid(pid: ContextId) -> Result<ContextId> {