Profiling

src/arch/x86_64/device/local_apic.rs

@@ -80,6 +80,7 @@ impl LocalApic {
             self.write(0xF0, 0x100);
         }
         self.setup_error_int();
+
         //self.setup_timer();
     }
 

src/arch/x86_64/gdt.rs

@@ -5,7 +5,7 @@ use core::mem;
 
 use crate::LogicalCpuId;
 use crate::paging::{RmmA, RmmArch};
-use crate::percpu::PercpuBlock;
+use crate::percpu::{PercpuBlock, RingBuffer};
 
 use x86::bits64::task::TaskStateSegment;
 use x86::Ring;
@@ -201,8 +201,19 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
     pcr.percpu = PercpuBlock {
         cpu_id,
         switch_internals: Default::default(),
+        profiling: None,
     };
 }
+pub unsafe fn init_allocator() {
+    let percpu = PercpuBlock::current();
+
+    if percpu.cpu_id.get() == 4 { return }
+
+    let profiling = RingBuffer::create();
+
+    crate::scheme::debug::BUFS[percpu.cpu_id.get() as usize].store(profiling as *const _ as *mut _, core::sync::atomic::Ordering::SeqCst);
+    (core::ptr::addr_of!(percpu.profiling) as *mut Option<&'static RingBuffer>).write(Some(profiling));
+}
 
 #[derive(Copy, Clone, Debug)]
 #[repr(packed)]

src/arch/x86_64/idt.rs

@@ -24,7 +24,7 @@ pub type IdtReservations = [AtomicU64; 4];
 
 #[repr(C)]
 pub struct Idt {
-    entries: IdtEntries,
+    pub(crate) entries: IdtEntries,
     reservations: IdtReservations,
 }
 impl Idt {
@@ -232,11 +232,12 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) {
         // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error.
         *current_reservations[0].get_mut() |= 0x0003_FFFF_0000_0000;
     } else {
+        current_idt[32].set_func(irq::aux_timer);
         // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU
         current_idt[49].set_func(irq::lapic_error);
 
         // reserve bit 49
-        *current_reservations[0].get_mut() |= 1 << 49;
+        *current_reservations[0].get_mut() |= 1 << 32 | 1 << 49;
     }
 
     // Set IPI handlers

src/arch/x86_64/interrupt/exception.rs

@@ -1,10 +1,11 @@
+use core::sync::atomic::Ordering;
+
 use x86::irq::PageFaultError;
 
 use crate::memory::GenericPfFlags;
 use crate::{
     interrupt::stack_trace,
     paging::VirtualAddress,
-    ptrace,
     syscall::flag::*,
 
     interrupt_stack,
@@ -23,7 +24,7 @@ interrupt_stack!(divide_by_zero, |stack| {
 });
 
 interrupt_stack!(debug, @paranoid, |stack| {
-    let mut handled = false;
+    /*let mut handled = false;
 
     // Disable singlestep before there is a breakpoint, since the breakpoint
     // handler might end up setting it again but unless it does we want the
@@ -38,19 +39,54 @@ interrupt_stack!(debug, @paranoid, |stack| {
         stack.set_singlestep(had_singlestep);
     }
 
-    if !handled {
+    if !handled {*/
         println!("Debug trap");
         stack.dump();
+        loop {}
+        /*
         ksignal(SIGTRAP);
-    }
+    }*/
 });
 
 interrupt_stack!(non_maskable, @paranoid, |stack| {
-    println!("Non-maskable interrupt");
-    stack.dump();
+    let Some(profiling) = crate::percpu::PercpuBlock::current().profiling else {
+        return;
+    };
+    if stack.iret.cs & 0b00 == 0b11 {
+        profiling.nmi_ucount.store(profiling.nmi_ucount.load(Ordering::Relaxed) + 1, Ordering::Relaxed);
+        return;
+    } else {
+        profiling.nmi_kcount.store(profiling.nmi_kcount.load(Ordering::Relaxed) + 1, Ordering::Relaxed);
+    };
+
+    let mut buf = [0_usize; 32];
+    buf[0] = stack.iret.rip & !(1<<63);
+    buf[1] = x86::time::rdtsc() as usize;
+
+    let mut bp = stack.preserved.rbp;
+
+    let mut len = 2;
+
+    for i in 2..32 {
+        if bp.saturating_add(16) < crate::KERNEL_HEAP_OFFSET || bp >= crate::KERNEL_HEAP_OFFSET + crate::PML4_SIZE {
+            break;
+        }
+        let ip = ((bp + 8) as *const usize).read();
+        bp = (bp as *const usize).read();
+
+        if ip < crate::kernel_executable_offsets::__text_start() || ip >= crate::kernel_executable_offsets::__text_end() {
+            break;
+        }
+        buf[i] = ip;
+
+        len = i + 1;
+    }
+
+    let _ = profiling.extend(&buf[..len]);
 });
 
 interrupt_stack!(breakpoint, |stack| {
+    /*
     // The processor lets RIP point to the instruction *after* int3, so
     // unhandled breakpoint interrupt don't go in an infinite loop. But we
     // throw SIGTRAP anyway, so that's not a problem.
@@ -64,11 +100,13 @@ interrupt_stack!(breakpoint, |stack| {
     // int3 instruction. After all, it's the sanest thing to do.
     stack.iret.rip -= 1;
 
-    if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_none() {
+    if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_none() {*/
         println!("Breakpoint trap");
         stack.dump();
+        loop {}
+        /*
         ksignal(SIGTRAP);
-    }
+    }*/
 });
 
 interrupt_stack!(overflow, |stack| {
@@ -127,8 +165,8 @@ interrupt_error!(stack_segment, |stack, _code| {
     ksignal(SIGSEGV);
 });
 
-interrupt_error!(protection, |stack, _code| {
-    println!("Protection fault");
+interrupt_error!(protection, |stack, code| {
+    println!("Protection fault code={:#0x}", code);
     stack.dump();
     stack_trace();
     ksignal(SIGSEGV);

src/arch/x86_64/interrupt/irq.rs

@@ -252,6 +252,10 @@ interrupt!(lapic_timer, || {
     println!("Local apic timer interrupt");
     lapic_eoi();
 });
+interrupt!(aux_timer, || {
+    lapic_eoi();
+    crate::ipi::ipi(IpiKind::Profile, IpiTarget::Other);
+});
 
 interrupt!(lapic_error, || {
     println!("Local apic internal error: ESR={:#0x}", local_apic::LOCAL_APIC.esr());

src/arch/x86_64/ipi.rs

@@ -5,6 +5,7 @@ pub enum IpiKind {
     Tlb = 0x41,
     Switch = 0x42,
     Pit = 0x43,
+    Profile = 0x44,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -24,6 +25,11 @@ pub fn ipi(_kind: IpiKind, _target: IpiTarget) {}
 pub fn ipi(kind: IpiKind, target: IpiTarget) {
     use crate::device::local_apic::LOCAL_APIC;
 
-    let icr = (target as u64) << 18 | 1 << 14 | (kind as u64);
-    unsafe { LOCAL_APIC.set_icr(icr) };
+    if matches!(kind, IpiKind::Profile) {
+        let icr = (target as u64) << 18 | 1 << 14 | 0b100 << 8;
+        unsafe { LOCAL_APIC.set_icr(icr) };
+    } else {
+        let icr = (target as u64) << 18 | 1 << 14 | (kind as u64);
+        unsafe { LOCAL_APIC.set_icr(icr) };
+    }
 }

src/arch/x86_64/start.rs

@@ -150,6 +150,8 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
         // Setup kernel heap
         allocator::init();
 
+        gdt::init_allocator();
+
         // Set up double buffer for grpahical debug now that heap is available
         #[cfg(feature = "graphical_debug")]
         graphical_debug::init_heap();
@@ -234,6 +236,8 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
         // Set up GDT with TLS
         gdt::init_paging(stack_end, cpu_id);
 
+        gdt::init_allocator();
+
         // Set up IDT for AP
         idt::init_paging_post_heap(false, cpu_id);
 

src/main.rs

@@ -47,6 +47,7 @@
 #![feature(int_roundings)]
 #![feature(let_chains)]
 #![feature(naked_functions)]
+#![feature(new_uninit)]
 #![feature(offset_of)]
 #![feature(sync_unsafe_cell)]
 #![feature(variant_count)]
@@ -59,6 +60,7 @@ extern crate alloc;
 #[macro_use]
 extern crate bitflags;
 
+use core::sync::atomic::AtomicUsize;
 use core::sync::atomic::{AtomicU32, Ordering};
 
 use crate::scheme::SchemeNamespace;
@@ -186,6 +188,7 @@ fn kmain(cpu_count: u32, bootstrap: Bootstrap) -> ! {
     info!("Env: {:?}", ::core::str::from_utf8(bootstrap.env));
 
     BOOTSTRAP.call_once(|| bootstrap);
+    crate::ACK.fetch_add(1, Ordering::SeqCst);
 
     match context::contexts_mut().spawn(userspace_init) {
         Ok(context_lock) => {
@@ -213,15 +216,41 @@ fn kmain(cpu_count: u32, bootstrap: Bootstrap) -> ! {
     }
 }
 
+pub static ACK: AtomicUsize = AtomicUsize::new(0);
+
 /// This is the main kernel entry point for secondary CPUs
 #[allow(unreachable_code, unused_variables)]
 fn kmain_ap(cpu_id: LogicalCpuId) -> ! {
+    if cpu_id.get() == 4 {
+        unsafe {
+            for i in 33..255 {
+                crate::idt::IDTS.write().as_mut().unwrap().get_mut(&cpu_id).unwrap().entries[i].set_func(crate::interrupt::ipi::wakeup);
+            }
+
+            let apic = &mut crate::device::local_apic::LOCAL_APIC;
+            apic.set_lvt_timer((0b01 << 17) | 32);
+            apic.set_div_conf(0b1011);
+            apic.set_init_count(0xffff);
+
+            while ACK.load(Ordering::Relaxed) < 4 {
+                core::hint::spin_loop();
+            }
+
+            interrupt::enable_and_nop();
+            loop {
+                interrupt::halt();
+            }
+        }
+    }
+
     if cfg!(feature = "multi_core") {
         context::init();
 
         let pid = syscall::getpid();
         info!("AP {}: {:?}", cpu_id, pid);
 
+        crate::ACK.fetch_add(1, Ordering::SeqCst);
+
         loop {
             unsafe {
                 interrupt::disable();

src/percpu.rs

@@ -1,3 +1,8 @@
+use core::cell::UnsafeCell;
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+use alloc::boxed::Box;
+
 use crate::LogicalCpuId;
 use crate::context::switch::ContextSwitchPercpu;
 
@@ -11,6 +16,76 @@ pub struct PercpuBlock {
 
     // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
     // first to avoid cache invalidation.
+
+    pub profiling: Option<&'static RingBuffer>,
+}
+
+const N: usize = 64 * 1024 * 1024;
+
+pub struct RingBuffer {
+    head: AtomicUsize,
+    tail: AtomicUsize,
+    buf: &'static [UnsafeCell<usize>; N],
+    pub(crate) nmi_kcount: AtomicUsize,
+    pub(crate) nmi_ucount: AtomicUsize,
+}
+
+impl RingBuffer {
+    unsafe fn advance_head(&self, n: usize) {
+        self.head.store(self.head.load(Ordering::Acquire).wrapping_add(n), Ordering::Release);
+    }
+    unsafe fn advance_tail(&self, n: usize) {
+        self.tail.store(self.tail.load(Ordering::Acquire).wrapping_add(n), Ordering::Release);
+    }
+    unsafe fn sender_owned(&self) -> [&[UnsafeCell<usize>]; 2] {
+        let head = self.head.load(Ordering::Acquire) % N;
+        let tail = self.tail.load(Ordering::Acquire) % N;
+
+        if head <= tail {
+            [&self.buf[tail..], &self.buf[..head]]
+        } else {
+            [&self.buf[tail..head], &[]]
+        }
+    }
+    unsafe fn receiver_owned(&self) -> [&[UnsafeCell<usize>]; 2] {
+        let head = self.head.load(Ordering::Acquire) % N;
+        let tail = self.tail.load(Ordering::Acquire) % N;
+
+        if head > tail {
+            [&self.buf[head..], &self.buf[..tail]]
+        } else {
+            [&self.buf[head..tail], &[]]
+        }
+    }
+    pub unsafe fn extend(&self, mut slice: &[usize]) -> usize {
+        let mut n = 0;
+        for mut sender_slice in self.sender_owned() {
+            while !slice.is_empty() && !sender_slice.is_empty() {
+                sender_slice[0].get().write(slice[0]);
+                slice = &slice[1..];
+                sender_slice = &sender_slice[1..];
+                n += 1;
+            }
+        }
+        self.advance_tail(n);
+        n
+    }
+    pub unsafe fn peek(&self) -> [&[usize]; 2] {
+        self.receiver_owned().map(|slice| core::slice::from_raw_parts(slice.as_ptr().cast(), slice.len()))
+    }
+    pub unsafe fn advance(&self, n: usize) {
+        self.advance_head(n)
+    }
+    pub fn create() -> &'static Self {
+
+        Box::leak(Box::new(Self {
+            head: AtomicUsize::new(0),
+            tail: AtomicUsize::new(0),
+            buf: Box::leak(unsafe { Box::new_zeroed().assume_init() }),
+            nmi_kcount: AtomicUsize::new(0),
+            nmi_ucount: AtomicUsize::new(0),
+        }))
+    }
 }
 
 // PercpuBlock::current() is implemented somewhere in the arch-specific modules

src/scheme/debug.rs

@@ -1,8 +1,9 @@
-use core::sync::atomic::{AtomicUsize, Ordering};
+use core::sync::atomic::{AtomicUsize, Ordering, AtomicPtr};
 use spin::RwLock;
 
 use crate::arch::debug::Writer;
 use crate::event;
+use crate::percpu::RingBuffer;
 use crate::scheme::*;
 use crate::sync::WaitQueue;
 use crate::syscall::flag::{EventFlags, EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK};
@@ -16,6 +17,7 @@ static INPUT: WaitQueue<u8> = WaitQueue::new();
 #[derive(Clone, Copy)]
 struct Handle {
     flags: usize,
+    num: usize,
 }
 
 // Using BTreeMap as hashbrown doesn't have a const constructor.
@@ -41,13 +43,17 @@ impl KernelScheme for DebugScheme {
             return Err(Error::new(EPERM));
         }
 
-        if ! path.is_empty() {
-            return Err(Error::new(ENOENT));
-        }
+        let num = match path {
+            "" => !0,
+            "profiling" => flags & 0xffff,
+
+            _ => return Err(Error::new(ENOENT)),
+        };
 
         let id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
         HANDLES.write().insert(id, Handle {
-            flags: flags & ! O_ACCMODE
+            flags: flags & ! O_ACCMODE,
+            num,
         });
 
         Ok(OpenResult::SchemeLocal(id))
@@ -102,8 +108,27 @@ impl KernelScheme for DebugScheme {
             *handles.get(&id).ok_or(Error::new(EBADF))?
         };
 
-        INPUT
-            .receive_into_user(buf, handle.flags & O_NONBLOCK != O_NONBLOCK, "DebugScheme::read")
+        if handle.num == !0 {
+            INPUT
+                .receive_into_user(buf, handle.flags & O_NONBLOCK != O_NONBLOCK, "DebugScheme::read")
+        } else {
+            unsafe {
+                let Some(src) = BUFS.get(handle.num).ok_or(Error::new(EBADFD))?.load(Ordering::Relaxed).as_ref() else {
+                    return Ok(0);
+                };
+                let byte_slices = src.peek().map(|words| core::slice::from_raw_parts(words.as_ptr().cast::<u8>(), words.len() * 8));
+
+                let copied_1 = buf.copy_common_bytes_from_slice(byte_slices[0])?;
+
+                let copied_2 = if let Some(remaining) = buf.advance(copied_1) {
+                    remaining.copy_common_bytes_from_slice(byte_slices[1])?
+                } else {
+                    0
+                };
+
+                Ok(copied_1 + copied_2)
+            }
+        }
     }
 
     fn kwrite(&self, id: usize, buf: UserSliceRo) -> Result<usize> {
@@ -140,3 +165,6 @@ impl KernelScheme for DebugScheme {
         Ok(byte_count)
     }
 }
+
+const NULL: AtomicPtr<RingBuffer> = AtomicPtr::new(core::ptr::null_mut());
+pub static BUFS: [AtomicPtr<RingBuffer>; 4] = [NULL; 4];