use core::cell::{Cell, RefCell}; use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use alloc::sync::Arc; use rmm::Arch; use crate::context::empty_cr3; use crate::context::memory::AddrSpaceWrapper; use crate::cpu_set::MAX_CPU_COUNT; use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId}; /// The percpu block, that stored all percpu variables. pub struct PercpuBlock { /// A unique immutable number that identifies the current CPU - used for scheduling pub cpu_id: LogicalCpuId, /// Context management pub switch_internals: ContextSwitchPercpu, pub current_addrsp: RefCell>>, pub new_addrsp_tmp: Cell>>, pub wants_tlb_shootdown: AtomicBool, // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. #[cfg(feature = "profiling")] pub profiling: Option<&'static crate::profiling::RingBuffer>, } const NULL: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); static ALL_PERCPU_BLOCKS: [AtomicPtr; MAX_CPU_COUNT as usize] = [NULL; MAX_CPU_COUNT as usize]; pub unsafe fn init_tlb_shootdown(id: LogicalCpuId, block: *mut PercpuBlock) { ALL_PERCPU_BLOCKS[id.get() as usize].store(block, Ordering::Release) } // PercpuBlock::current() is implemented somewhere in the arch-specific modules #[cfg(not(feature = "multi_core"))] pub fn shootdown_tlb_ipi(_target: Option) {} #[cfg(feature = "multi_core")] pub fn shootdown_tlb_ipi(target: Option) { if let Some(target) = target { let my_percpublock = PercpuBlock::current(); assert_ne!(target, my_percpublock.cpu_id); let Some(percpublock) = (unsafe { ALL_PERCPU_BLOCKS[target.get() as usize].load(Ordering::Acquire).as_ref() }) else { log::warn!("Trying to TLB shootdown a CPU that doesn't exist or isn't initialized."); return; }; while percpublock.wants_tlb_shootdown.swap(true, Ordering::Release) == true { // Load is faster than CAS or on x86, LOCK BTS while percpublock.wants_tlb_shootdown.load(Ordering::Relaxed) == true { my_percpublock.maybe_handle_tlb_shootdown(); core::hint::spin_loop(); } } crate::ipi::ipi_single(crate::ipi::IpiKind::Tlb, target); } else { for id in 0..crate::cpu_count() { // TODO: Optimize: use global counter and percpu ack counters, send IPI using // destination shorthand "all CPUs". shootdown_tlb_ipi(Some(LogicalCpuId::new(id))); } } } impl PercpuBlock { pub fn maybe_handle_tlb_shootdown(&self) { if self.wants_tlb_shootdown.swap(false, Ordering::Relaxed) == false { return; } // TODO: Finer-grained flush unsafe { x86::tlb::flush_all(); } if let Some(ref addrsp) = &*self.current_addrsp.borrow() { addrsp.tlb_ack.fetch_add(1, Ordering::Release); } } } pub unsafe fn switch_arch_hook() { let percpu = PercpuBlock::current(); let cur_addrsp = percpu.current_addrsp.borrow(); let next_addrsp = percpu.new_addrsp_tmp.take(); let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) { (Some(ref p), Some(ref n)) => Arc::ptr_eq(p, n), (Some(_), None) | (None, Some(_)) => false, (None, None) => true, }; if retain_pgtbl { // If we are not switching to a different address space, we can simply return early. } if let Some(ref prev_addrsp) = &*cur_addrsp { prev_addrsp.acquire_read().used_by.atomic_clear(percpu.cpu_id); } drop(cur_addrsp); // Tell future TLB shootdown handlers that old_addrsp_tmp is no longer the current address // space. *percpu.current_addrsp.borrow_mut() = next_addrsp; if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() { let next = next_addrsp.acquire_read(); next.used_by.atomic_set(percpu.cpu_id); next.table.utable.make_current(); } else { crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3()); } }