From f973bc4128cc2c68d124dfd25c89826fb6cd660a Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 7 Oct 2023 22:05:44 +0200 Subject: [PATCH] Profiling --- src/arch/x86_64/device/local_apic.rs | 1 + src/arch/x86_64/gdt.rs | 13 ++++- src/arch/x86_64/idt.rs | 5 +- src/arch/x86_64/interrupt/exception.rs | 58 ++++++++++++++++---- src/arch/x86_64/interrupt/irq.rs | 4 ++ src/arch/x86_64/ipi.rs | 10 +++- src/arch/x86_64/start.rs | 4 ++ src/main.rs | 29 ++++++++++ src/percpu.rs | 75 ++++++++++++++++++++++++++ src/scheme/debug.rs | 42 ++++++++++++--- 10 files changed, 219 insertions(+), 22 deletions(-) diff --git a/src/arch/x86_64/device/local_apic.rs b/src/arch/x86_64/device/local_apic.rs index 5b3b23ae4f..0c15b1744e 100644 --- a/src/arch/x86_64/device/local_apic.rs +++ b/src/arch/x86_64/device/local_apic.rs @@ -80,6 +80,7 @@ impl LocalApic { self.write(0xF0, 0x100); } self.setup_error_int(); + //self.setup_timer(); } diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 19bb132db8..04c8615a50 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -5,7 +5,7 @@ use core::mem; use crate::LogicalCpuId; use crate::paging::{RmmA, RmmArch}; -use crate::percpu::PercpuBlock; +use crate::percpu::{PercpuBlock, RingBuffer}; use x86::bits64::task::TaskStateSegment; use x86::Ring; @@ -201,8 +201,19 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { pcr.percpu = PercpuBlock { cpu_id, switch_internals: Default::default(), + profiling: None, }; } +pub unsafe fn init_allocator() { + let percpu = PercpuBlock::current(); + + if percpu.cpu_id.get() == 4 { return } + + let profiling = RingBuffer::create(); + + crate::scheme::debug::BUFS[percpu.cpu_id.get() as usize].store(profiling as *const _ as *mut _, core::sync::atomic::Ordering::SeqCst); + (core::ptr::addr_of!(percpu.profiling) as *mut Option<&'static RingBuffer>).write(Some(profiling)); +} #[derive(Copy, Clone, Debug)] #[repr(packed)] diff --git a/src/arch/x86_64/idt.rs b/src/arch/x86_64/idt.rs index a3b2e18665..ac4c1a54e7 100644 --- a/src/arch/x86_64/idt.rs +++ b/src/arch/x86_64/idt.rs @@ -24,7 +24,7 @@ pub type IdtReservations = [AtomicU64; 4]; #[repr(C)] pub struct Idt { - entries: IdtEntries, + pub(crate) entries: IdtEntries, reservations: IdtReservations, } impl Idt { @@ -232,11 +232,12 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) { // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error. *current_reservations[0].get_mut() |= 0x0003_FFFF_0000_0000; } else { + current_idt[32].set_func(irq::aux_timer); // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU current_idt[49].set_func(irq::lapic_error); // reserve bit 49 - *current_reservations[0].get_mut() |= 1 << 49; + *current_reservations[0].get_mut() |= 1 << 32 | 1 << 49; } // Set IPI handlers diff --git a/src/arch/x86_64/interrupt/exception.rs b/src/arch/x86_64/interrupt/exception.rs index ace4c5a77a..c5db3aed0e 100644 --- a/src/arch/x86_64/interrupt/exception.rs +++ b/src/arch/x86_64/interrupt/exception.rs @@ -1,10 +1,11 @@ +use core::sync::atomic::Ordering; + use x86::irq::PageFaultError; use crate::memory::GenericPfFlags; use crate::{ interrupt::stack_trace, paging::VirtualAddress, - ptrace, syscall::flag::*, interrupt_stack, @@ -23,7 +24,7 @@ interrupt_stack!(divide_by_zero, |stack| { }); interrupt_stack!(debug, @paranoid, |stack| { - let mut handled = false; + /*let mut handled = false; // Disable singlestep before there is a breakpoint, since the breakpoint // handler might end up setting it again but unless it does we want the @@ -38,19 +39,54 @@ interrupt_stack!(debug, @paranoid, |stack| { stack.set_singlestep(had_singlestep); } - if !handled { + if !handled {*/ println!("Debug trap"); stack.dump(); + loop {} + /* ksignal(SIGTRAP); - } + }*/ }); interrupt_stack!(non_maskable, @paranoid, |stack| { - println!("Non-maskable interrupt"); - stack.dump(); + let Some(profiling) = crate::percpu::PercpuBlock::current().profiling else { + return; + }; + if stack.iret.cs & 0b00 == 0b11 { + profiling.nmi_ucount.store(profiling.nmi_ucount.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + return; + } else { + profiling.nmi_kcount.store(profiling.nmi_kcount.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + }; + + let mut buf = [0_usize; 32]; + buf[0] = stack.iret.rip & !(1<<63); + buf[1] = x86::time::rdtsc() as usize; + + let mut bp = stack.preserved.rbp; + + let mut len = 2; + + for i in 2..32 { + if bp.saturating_add(16) < crate::KERNEL_HEAP_OFFSET || bp >= crate::KERNEL_HEAP_OFFSET + crate::PML4_SIZE { + break; + } + let ip = ((bp + 8) as *const usize).read(); + bp = (bp as *const usize).read(); + + if ip < crate::kernel_executable_offsets::__text_start() || ip >= crate::kernel_executable_offsets::__text_end() { + break; + } + buf[i] = ip; + + len = i + 1; + } + + let _ = profiling.extend(&buf[..len]); }); interrupt_stack!(breakpoint, |stack| { + /* // The processor lets RIP point to the instruction *after* int3, so // unhandled breakpoint interrupt don't go in an infinite loop. But we // throw SIGTRAP anyway, so that's not a problem. @@ -64,11 +100,13 @@ interrupt_stack!(breakpoint, |stack| { // int3 instruction. After all, it's the sanest thing to do. stack.iret.rip -= 1; - if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_none() { + if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_none() {*/ println!("Breakpoint trap"); stack.dump(); + loop {} + /* ksignal(SIGTRAP); - } + }*/ }); interrupt_stack!(overflow, |stack| { @@ -127,8 +165,8 @@ interrupt_error!(stack_segment, |stack, _code| { ksignal(SIGSEGV); }); -interrupt_error!(protection, |stack, _code| { - println!("Protection fault"); +interrupt_error!(protection, |stack, code| { + println!("Protection fault code={:#0x}", code); stack.dump(); stack_trace(); ksignal(SIGSEGV); diff --git a/src/arch/x86_64/interrupt/irq.rs b/src/arch/x86_64/interrupt/irq.rs index a61bb62c50..c40babf4a9 100644 --- a/src/arch/x86_64/interrupt/irq.rs +++ b/src/arch/x86_64/interrupt/irq.rs @@ -252,6 +252,10 @@ interrupt!(lapic_timer, || { println!("Local apic timer interrupt"); lapic_eoi(); }); +interrupt!(aux_timer, || { + lapic_eoi(); + crate::ipi::ipi(IpiKind::Profile, IpiTarget::Other); +}); interrupt!(lapic_error, || { println!("Local apic internal error: ESR={:#0x}", local_apic::LOCAL_APIC.esr()); diff --git a/src/arch/x86_64/ipi.rs b/src/arch/x86_64/ipi.rs index 28d21cd40b..4f3919b0f3 100644 --- a/src/arch/x86_64/ipi.rs +++ b/src/arch/x86_64/ipi.rs @@ -5,6 +5,7 @@ pub enum IpiKind { Tlb = 0x41, Switch = 0x42, Pit = 0x43, + Profile = 0x44, } #[derive(Clone, Copy, Debug)] @@ -24,6 +25,11 @@ pub fn ipi(_kind: IpiKind, _target: IpiTarget) {} pub fn ipi(kind: IpiKind, target: IpiTarget) { use crate::device::local_apic::LOCAL_APIC; - let icr = (target as u64) << 18 | 1 << 14 | (kind as u64); - unsafe { LOCAL_APIC.set_icr(icr) }; + if matches!(kind, IpiKind::Profile) { + let icr = (target as u64) << 18 | 1 << 14 | 0b100 << 8; + unsafe { LOCAL_APIC.set_icr(icr) }; + } else { + let icr = (target as u64) << 18 | 1 << 14 | (kind as u64); + unsafe { LOCAL_APIC.set_icr(icr) }; + } } diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index 977ea50f78..77ff2cdc85 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -150,6 +150,8 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Setup kernel heap allocator::init(); + gdt::init_allocator(); + // Set up double buffer for grpahical debug now that heap is available #[cfg(feature = "graphical_debug")] graphical_debug::init_heap(); @@ -234,6 +236,8 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { // Set up GDT with TLS gdt::init_paging(stack_end, cpu_id); + gdt::init_allocator(); + // Set up IDT for AP idt::init_paging_post_heap(false, cpu_id); diff --git a/src/main.rs b/src/main.rs index 2c28eb0c65..f523716e15 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,7 @@ #![feature(int_roundings)] #![feature(let_chains)] #![feature(naked_functions)] +#![feature(new_uninit)] #![feature(offset_of)] #![feature(sync_unsafe_cell)] #![feature(variant_count)] @@ -59,6 +60,7 @@ extern crate alloc; #[macro_use] extern crate bitflags; +use core::sync::atomic::AtomicUsize; use core::sync::atomic::{AtomicU32, Ordering}; use crate::scheme::SchemeNamespace; @@ -186,6 +188,7 @@ fn kmain(cpu_count: u32, bootstrap: Bootstrap) -> ! { info!("Env: {:?}", ::core::str::from_utf8(bootstrap.env)); BOOTSTRAP.call_once(|| bootstrap); + crate::ACK.fetch_add(1, Ordering::SeqCst); match context::contexts_mut().spawn(userspace_init) { Ok(context_lock) => { @@ -213,15 +216,41 @@ fn kmain(cpu_count: u32, bootstrap: Bootstrap) -> ! { } } +pub static ACK: AtomicUsize = AtomicUsize::new(0); + /// This is the main kernel entry point for secondary CPUs #[allow(unreachable_code, unused_variables)] fn kmain_ap(cpu_id: LogicalCpuId) -> ! { + if cpu_id.get() == 4 { + unsafe { + for i in 33..255 { + crate::idt::IDTS.write().as_mut().unwrap().get_mut(&cpu_id).unwrap().entries[i].set_func(crate::interrupt::ipi::wakeup); + } + + let apic = &mut crate::device::local_apic::LOCAL_APIC; + apic.set_lvt_timer((0b01 << 17) | 32); + apic.set_div_conf(0b1011); + apic.set_init_count(0xffff); + + while ACK.load(Ordering::Relaxed) < 4 { + core::hint::spin_loop(); + } + + interrupt::enable_and_nop(); + loop { + interrupt::halt(); + } + } + } + if cfg!(feature = "multi_core") { context::init(); let pid = syscall::getpid(); info!("AP {}: {:?}", cpu_id, pid); + crate::ACK.fetch_add(1, Ordering::SeqCst); + loop { unsafe { interrupt::disable(); diff --git a/src/percpu.rs b/src/percpu.rs index 905e34fd02..899296118d 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -1,3 +1,8 @@ +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicUsize, Ordering}; + +use alloc::boxed::Box; + use crate::LogicalCpuId; use crate::context::switch::ContextSwitchPercpu; @@ -11,6 +16,76 @@ pub struct PercpuBlock { // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. + + pub profiling: Option<&'static RingBuffer>, +} + +const N: usize = 64 * 1024 * 1024; + +pub struct RingBuffer { + head: AtomicUsize, + tail: AtomicUsize, + buf: &'static [UnsafeCell; N], + pub(crate) nmi_kcount: AtomicUsize, + pub(crate) nmi_ucount: AtomicUsize, +} + +impl RingBuffer { + unsafe fn advance_head(&self, n: usize) { + self.head.store(self.head.load(Ordering::Acquire).wrapping_add(n), Ordering::Release); + } + unsafe fn advance_tail(&self, n: usize) { + self.tail.store(self.tail.load(Ordering::Acquire).wrapping_add(n), Ordering::Release); + } + unsafe fn sender_owned(&self) -> [&[UnsafeCell]; 2] { + let head = self.head.load(Ordering::Acquire) % N; + let tail = self.tail.load(Ordering::Acquire) % N; + + if head <= tail { + [&self.buf[tail..], &self.buf[..head]] + } else { + [&self.buf[tail..head], &[]] + } + } + unsafe fn receiver_owned(&self) -> [&[UnsafeCell]; 2] { + let head = self.head.load(Ordering::Acquire) % N; + let tail = self.tail.load(Ordering::Acquire) % N; + + if head > tail { + [&self.buf[head..], &self.buf[..tail]] + } else { + [&self.buf[head..tail], &[]] + } + } + pub unsafe fn extend(&self, mut slice: &[usize]) -> usize { + let mut n = 0; + for mut sender_slice in self.sender_owned() { + while !slice.is_empty() && !sender_slice.is_empty() { + sender_slice[0].get().write(slice[0]); + slice = &slice[1..]; + sender_slice = &sender_slice[1..]; + n += 1; + } + } + self.advance_tail(n); + n + } + pub unsafe fn peek(&self) -> [&[usize]; 2] { + self.receiver_owned().map(|slice| core::slice::from_raw_parts(slice.as_ptr().cast(), slice.len())) + } + pub unsafe fn advance(&self, n: usize) { + self.advance_head(n) + } + pub fn create() -> &'static Self { + + Box::leak(Box::new(Self { + head: AtomicUsize::new(0), + tail: AtomicUsize::new(0), + buf: Box::leak(unsafe { Box::new_zeroed().assume_init() }), + nmi_kcount: AtomicUsize::new(0), + nmi_ucount: AtomicUsize::new(0), + })) + } } // PercpuBlock::current() is implemented somewhere in the arch-specific modules diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index 51107eaeb2..f2fca93d07 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -1,8 +1,9 @@ -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicUsize, Ordering, AtomicPtr}; use spin::RwLock; use crate::arch::debug::Writer; use crate::event; +use crate::percpu::RingBuffer; use crate::scheme::*; use crate::sync::WaitQueue; use crate::syscall::flag::{EventFlags, EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK}; @@ -16,6 +17,7 @@ static INPUT: WaitQueue = WaitQueue::new(); #[derive(Clone, Copy)] struct Handle { flags: usize, + num: usize, } // Using BTreeMap as hashbrown doesn't have a const constructor. @@ -41,13 +43,17 @@ impl KernelScheme for DebugScheme { return Err(Error::new(EPERM)); } - if ! path.is_empty() { - return Err(Error::new(ENOENT)); - } + let num = match path { + "" => !0, + "profiling" => flags & 0xffff, + + _ => return Err(Error::new(ENOENT)), + }; let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); HANDLES.write().insert(id, Handle { - flags: flags & ! O_ACCMODE + flags: flags & ! O_ACCMODE, + num, }); Ok(OpenResult::SchemeLocal(id)) @@ -102,8 +108,27 @@ impl KernelScheme for DebugScheme { *handles.get(&id).ok_or(Error::new(EBADF))? }; - INPUT - .receive_into_user(buf, handle.flags & O_NONBLOCK != O_NONBLOCK, "DebugScheme::read") + if handle.num == !0 { + INPUT + .receive_into_user(buf, handle.flags & O_NONBLOCK != O_NONBLOCK, "DebugScheme::read") + } else { + unsafe { + let Some(src) = BUFS.get(handle.num).ok_or(Error::new(EBADFD))?.load(Ordering::Relaxed).as_ref() else { + return Ok(0); + }; + let byte_slices = src.peek().map(|words| core::slice::from_raw_parts(words.as_ptr().cast::(), words.len() * 8)); + + let copied_1 = buf.copy_common_bytes_from_slice(byte_slices[0])?; + + let copied_2 = if let Some(remaining) = buf.advance(copied_1) { + remaining.copy_common_bytes_from_slice(byte_slices[1])? + } else { + 0 + }; + + Ok(copied_1 + copied_2) + } + } } fn kwrite(&self, id: usize, buf: UserSliceRo) -> Result { @@ -140,3 +165,6 @@ impl KernelScheme for DebugScheme { Ok(byte_count) } } + +const NULL: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +pub static BUFS: [AtomicPtr; 4] = [NULL; 4];