diff --git a/Cargo.toml b/Cargo.toml index cf920c2560..918cddaacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ cc = "1.0" toml = "0.8" [dependencies] -acpi_ext = { package = "acpi", git = "https://gitlab.redox-os.org/redox-os/acpi.git", branch = "redox-6.x" } arrayvec = { version = "0.7.4", default-features = false } bitfield = "0.13.2" bitflags = "2" diff --git a/build.rs b/build.rs index 751746ccdd..96c3ea5c78 100644 --- a/build.rs +++ b/build.rs @@ -77,7 +77,6 @@ fn main() { } "x86_64" => { println!("cargo::rerun-if-changed=src/asm/x86_64/trampoline.asm"); - println!("cargo::rerun-if-changed=src/asm/x86_64/s3_wakeup.asm"); let status = Command::new("nasm") .arg("-f") @@ -90,18 +89,6 @@ fn main() { if !status.success() { panic!("nasm failed with exit status {}", status); } - - let status = Command::new("nasm") - .arg("-f") - .arg("bin") - .arg("-o") - .arg(format!("{}/s3_wakeup", out_dir)) - .arg("src/asm/x86_64/s3_wakeup.asm") - .status() - .expect("failed to run nasm"); - if !status.success() { - panic!("nasm failed with exit status {}", status); - } } "riscv64" => { println!("cargo::rustc-cfg=dtb"); diff --git a/src/acpi/madt/arch/x86.rs b/src/acpi/madt/arch/x86.rs index f472c0886d..4dc2388398 100644 --- a/src/acpi/madt/arch/x86.rs +++ b/src/acpi/madt/arch/x86.rs @@ -18,7 +18,6 @@ use crate::{ use super::{Madt, MadtEntry}; -const AP_SPIN_LIMIT: u32 = 1_000_000; const TRAMPOLINE: usize = 0x8000; static TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trampoline")); @@ -43,17 +42,13 @@ pub(super) fn init(madt: Madt) { //TODO: do not have writable and executable! let mut mapper = KernelMapper::lock_rw(); - let result = match mapper.map_phys( - trampoline_page.start_address(), - trampoline_frame.base(), - PageFlags::new().execute(true).write(true), - ) { - Some(result) => result, - None => { - println!("KERNEL AP: failed to map trampoline page, AP bring-up disabled"); - return; - } - }; + let result = mapper + .map_phys( + trampoline_page.start_address(), + trampoline_frame.base(), + PageFlags::new().execute(true).write(true), + ) + .expect("failed to map trampoline"); (result, mapper.table().phys().data()) }; @@ -77,26 +72,16 @@ pub(super) fn init(madt: Madt) { if u32::from(ap_local_apic.id) == me.get() { debug!(" This is my local APIC"); } else if ap_local_apic.flags & 1 == 1 { - // Allocate a stack - let alloc = match allocate_p2frame(4) { - Some(frame) => frame, - None => { - println!("KERNEL AP: CPU {} no memory for stack, skipping", ap_local_apic.id); - continue; - } - }; - let stack_start = RmmA::phys_to_virt(alloc.base()).data(); - let stack_end = stack_start + (PAGE_SIZE << 4); + let cpu_id = LogicalCpuId::next(); - let next_cpu = crate::CPU_COUNT.load(Ordering::Relaxed); - if next_cpu >= crate::cpu_set::MAX_CPU_COUNT { - println!( - "KERNEL AP: CPU {} exceeds logical CPU limit, skipping", - ap_local_apic.id - ); - continue; - } - let cpu_id = LogicalCpuId::new(next_cpu); + // Allocate a stack + let stack_start = RmmA::phys_to_virt( + allocate_p2frame(4) + .expect("no more frames in acpi stack_start") + .base(), + ) + .data(); + let stack_end = stack_start + (PAGE_SIZE << 4); let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end); @@ -152,34 +137,13 @@ pub(super) fn init(madt: Madt) { local_apic.set_icr(icr); } - // Wait for trampoline ready with timeout - let mut trampoline_ready = false; - for _ in 0..AP_SPIN_LIMIT { - if unsafe { (*ap_ready.cast::()).load(Ordering::SeqCst) } != 0 { - trampoline_ready = true; - break; - } + // Wait for trampoline ready + while unsafe { (*ap_ready.cast::()).load(Ordering::SeqCst) } == 0 { hint::spin_loop(); } - if !trampoline_ready { - println!("KERNEL AP: CPU {} trampoline timeout, skipping", ap_local_apic.id); - continue; - } - - let mut kernel_ready = false; - for _ in 0..AP_SPIN_LIMIT { - if AP_READY.load(Ordering::SeqCst) { - kernel_ready = true; - break; - } + while !AP_READY.load(Ordering::SeqCst) { hint::spin_loop(); } - if !kernel_ready { - println!("KERNEL AP: CPU {} AP_READY timeout, skipping", ap_local_apic.id); - continue; - } - - crate::CPU_COUNT.fetch_add(1, Ordering::Relaxed); RmmA::invalidate_all(); } @@ -187,12 +151,10 @@ pub(super) fn init(madt: Madt) { } // Unmap trampoline - if let Some((_frame, _, flush)) = unsafe { + let (_frame, _, flush) = unsafe { KernelMapper::lock_rw() .unmap_phys(trampoline_page.start_address()) - } { - flush.flush(); - } else { - println!("KERNEL AP: failed to unmap trampoline page (non-fatal)"); - } + .expect("failed to unmap trampoline page") + }; + flush.flush(); } diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index bb676e0dbf..83900eb226 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -82,35 +82,10 @@ pub enum RxsdtEnum { Rsdt(Rsdt), Xsdt(Xsdt), } -#[derive(Clone, Copy, Debug)] -pub struct AcpiRootInfo { - pub revision: u8, - pub root_sdt_address: PhysicalAddress, -} - -pub static ACPI_ROOT_INFO: Once = Once::new(); - -#[derive(Clone, Copy, Debug)] -pub struct AcpiRootInfo { - pub revision: u8, - pub root_sdt_address: PhysicalAddress, -} - -pub static ACPI_ROOT_INFO: Once = Once::new(); - impl Rxsdt for RxsdtEnum { fn iter(&self) -> RxsdtIter { match self { Self::Rsdt(rsdt) => rsdt.iter(), - let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo { - revision: rsdp.revision(), - root_sdt_address: rsdp.sdt_address(), - }); - - if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() { - error!("ACPI_ROOT_INFO already initialized with a different RSDP root"); - } - Self::Xsdt(xsdt) => xsdt.iter(), } } @@ -126,15 +101,6 @@ pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp); if let Some(rsdp) = rsdp_opt { - let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo { - revision: rsdp.revision(), - root_sdt_address: rsdp.sdt_address(), - }); - - if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() { - error!("ACPI_ROOT_INFO already initialized with a different RSDP root"); - } - debug!("SDT address: {:#x}", rsdp.sdt_address().data()); let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw()); diff --git a/src/acpi/rsdp.rs b/src/acpi/rsdp.rs index 094c999757..d3319d2fca 100644 --- a/src/acpi/rsdp.rs +++ b/src/acpi/rsdp.rs @@ -59,8 +59,4 @@ impl Rsdp { self.rsdt_address as usize }) } - - pub fn revision(&self) -> u8 { - self.revision - } } diff --git a/src/allocator/mod.rs b/src/allocator/mod.rs index aaa719635e..4fdb0ba16e 100644 --- a/src/allocator/mod.rs +++ b/src/allocator/mod.rs @@ -7,40 +7,26 @@ mod linked_list; /// Size of kernel heap const KERNEL_HEAP_SIZE: usize = ::rmm::MEGABYTE; -#[cold] -fn halt_kernel_heap_init(message: &str) -> ! { - print!("{message}"); - println!("Kernel heap initialization cannot continue. Halting."); - loop { - core::hint::spin_loop(); - } -} - unsafe fn map_heap(mapper: &mut KernelMapper, offset: usize, size: usize) { let mut flush_all = PageFlushAll::new(); let heap_start_page = Page::containing_address(VirtualAddress::new(offset)); let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1)); for page in Page::range_inclusive(heap_start_page, heap_end_page) { - let phys = match mapper.allocator_mut().allocate_one() { - Some(phys) => phys, - None => halt_kernel_heap_init( - "FATAL: failed to allocate physical frame for kernel heap\n", - ), - }; + let phys = mapper + .allocator_mut() + .allocate_one() + .expect("failed to allocate kernel heap"); let flush = unsafe { - match mapper.map_phys( - page.start_address(), - phys, - PageFlags::new() - .write(true) - .global(cfg!(not(feature = "pti"))), - ) { - Some(flush) => flush, - None => halt_kernel_heap_init( - "FATAL: failed to map kernel heap virtual page\n", - ), - } + mapper + .map_phys( + page.start_address(), + phys, + PageFlags::new() + .write(true) + .global(cfg!(not(feature = "pti"))), + ) + .expect("failed to map kernel heap") }; flush_all.consume(flush); } diff --git a/src/arch/x86_shared/gdt.rs b/src/arch/x86_shared/gdt.rs index b1879b40dd..cad344f3c2 100644 --- a/src/arch/x86_shared/gdt.rs +++ b/src/arch/x86_shared/gdt.rs @@ -192,24 +192,6 @@ impl ProcessorControlRegion { } } -#[cold] -fn halt_pcr_init() -> ! { - println!("FATAL: failed to allocate physical memory for Processor Control Region"); - println!("Processor startup cannot continue. Halting."); - loop { - core::hint::spin_loop(); - } -} - -#[cold] -fn halt_pcr_init() -> ! { - println!("FATAL: failed to allocate physical memory for Processor Control Region"); - println!("Processor startup cannot continue. Halting."); - loop { - core::hint::spin_loop(); - } -} - pub unsafe fn pcr() -> *mut ProcessorControlRegion { unsafe { // Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that @@ -393,10 +375,7 @@ pub fn allocate_and_init_pcr( .next_power_of_two() .trailing_zeros(); - let pcr_frame = match crate::memory::allocate_p2frame(alloc_order) { - Some(frame) => frame, - None => halt_pcr_init(), - }; + let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR"); let pcr_ptr = RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion; unsafe { core::ptr::write(pcr_ptr, ProcessorControlRegion::new_partial_init(cpu_id)) }; diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 2bfe327e62..a4d9eb6a9d 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -78,15 +78,6 @@ static INIT_BSP_IDT: SyncUnsafeCell = SyncUnsafeCell::new(Idt::new()); pub(crate) static IDTS: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); -#[cold] -fn halt_idt_init() -> ! { - println!("FATAL: failed to allocate physical pages for backup interrupt stack"); - println!("Interrupt setup cannot continue. Halting."); - loop { - core::hint::spin_loop(); - } -} - #[inline] pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool { if cpu_id == LogicalCpuId::BSP { @@ -170,10 +161,8 @@ pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt { .or_insert_with(|| Box::leak(Box::new(Idt::new()))); use crate::memory::{RmmA, RmmArch}; - let frames = match crate::memory::allocate_p2frame(4) { - Some(frames) => frames, - None => halt_idt_init(), - }; + let frames = crate::memory::allocate_p2frame(4) + .expect("failed to allocate pages for backup interrupt stack"); // Physical pages are mapped linearly. So is the linearly mapped virtual memory. let base_address = RmmA::phys_to_virt(frames.base()); diff --git a/src/arch/x86_shared/mod.rs b/src/arch/x86_shared/mod.rs index 004d65ceaa..19f5036c1f 100644 --- a/src/arch/x86_shared/mod.rs +++ b/src/arch/x86_shared/mod.rs @@ -28,8 +28,6 @@ pub mod pti; /// Initialization and start function pub mod start; -pub mod sleep; - /// Stop function pub mod stop; diff --git a/src/arch/x86_shared/sleep.rs b/src/arch/x86_shared/sleep.rs deleted file mode 100644 index 9f98c0d892..0000000000 --- a/src/arch/x86_shared/sleep.rs +++ /dev/null @@ -1,712 +0,0 @@ -use alloc::{sync::Arc, vec::Vec}; -use core::{ - ptr::NonNull, - str::FromStr, - sync::atomic::{AtomicU32, Ordering}, -}; - -use acpi_ext::{ - aml::{namespace::AmlName, object::Object, Interpreter}, - registers::FixedRegisters, - sdt::{facs::Facs, fadt::Fadt, SdtHeader}, - AcpiTables, Handle, Handler, PhysicalMapping, -}; -use spin::Mutex; -use syscall::error::{Error, EINVAL, EIO}; -use x86::{segmentation::SegmentSelector, task, Ring}; - -use crate::{ - acpi::ACPI_ROOT_INFO, - arch::interrupt, - memory::{ - round_down_pages, round_up_pages, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, - RmmArch, VirtualAddress, PAGE_SIZE, - }, - syscall::io::{Io, Pio}, -}; - -const ACPI_SLP_TYP_SHIFT: u16 = 10; -const ACPI_SLP_TYP_MASK: u16 = 0x1C00; -const ACPI_SLP_EN: u16 = 1 << 13; -const WAKE_TRAMPOLINE_PHYS: usize = 0x8000; -const SLEEP_RETURN_OK: usize = 0; - -#[cfg(target_arch = "x86_64")] -static WAKE_TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/s3_wakeup")); - -#[repr(C, packed)] -#[derive(Clone, Copy, Debug, Default)] -struct DescriptorTableRegister { - limit: u16, - base: u64, -} - -#[repr(C, align(64))] -#[derive(Clone, Copy, Debug)] -struct FpuState { - bytes: [u8; 4096], -} - -impl Default for FpuState { - fn default() -> Self { - Self { bytes: [0; 4096] } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum SleepState { - S3, - S5, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum SleepError { - UnsupportedArch, - MissingAcpi, - MissingFadt, - MissingFacs, - MissingSleepObject, - InvalidSleepObject, - UnsupportedPmControl, - UnsupportedAmlOperation, - SleepDidNotEnter, -} - -impl SleepError { - fn code(self) -> usize { - match self { - Self::UnsupportedArch => EINVAL as usize, - Self::MissingAcpi - | Self::MissingFadt - | Self::MissingFacs - | Self::MissingSleepObject - | Self::UnsupportedAmlOperation => EIO as usize, - Self::InvalidSleepObject | Self::UnsupportedPmControl | Self::SleepDidNotEnter => { - EINVAL as usize - } - } - } - - fn from_code(code: usize) -> Self { - match code as i32 { - x if x == EINVAL => Self::InvalidSleepObject, - _ => Self::MissingAcpi, - } - } -} - -#[derive(Clone, Copy, Debug, Default)] -struct SavedCpuContext { - entry_rsp: usize, - runtime_rsp: usize, - facs_address: usize, - cr0: usize, - cr2: usize, - cr3: usize, - cr4: usize, - rflags: usize, - gdtr: DescriptorTableRegister, - idtr: DescriptorTableRegister, - efer: u64, - fs_base: u64, - gs_base: u64, - kernel_gs_base: u64, - fpu: FpuState, -} - -static SAVED_CONTEXT: Mutex> = Mutex::new(None); -static AML_MUTEX_IDS: AtomicU32 = AtomicU32::new(1); - -#[derive(Clone, Copy, Debug)] -struct SleepTypeData { - a: u16, - b: u16, -} - -#[derive(Clone, Copy)] -struct KernelAcpiHandler; - -impl KernelAcpiHandler { - fn map_range(physical_address: usize, size: usize) -> (*mut u8, usize) { - let map_base = round_down_pages(physical_address); - let map_offset = physical_address - map_base; - let mapped_length = round_up_pages(size + map_offset); - - // SAFETY: The ACPI interpreter only requests firmware-described physical regions. - unsafe { - let mut mapper = KernelMapper::lock_rw(); - for page_index in 0..mapped_length / PAGE_SIZE { - let (_, flush) = mapper - .map_linearly( - PhysicalAddress::new(map_base + page_index * PAGE_SIZE), - PageFlags::new(), - ) - .expect("failed to linearly map ACPI physical region"); - flush.flush(); - } - } - - let virtual_base = RmmA::phys_to_virt(PhysicalAddress::new(map_base)).data(); - ((virtual_base + map_offset) as *mut u8, mapped_length) - } -} - -impl Handler for KernelAcpiHandler { - unsafe fn map_physical_region(&self, physical_address: usize, size: usize) -> PhysicalMapping { - let (virtual_start, mapped_length) = Self::map_range(physical_address, size); - PhysicalMapping { - physical_start: physical_address, - virtual_start: NonNull::new(virtual_start.cast::()) - .expect("expected mapped ACPI virtual address to be non-null"), - region_length: size, - mapped_length, - handler: *self, - } - } - - fn unmap_physical_region(_region: &PhysicalMapping) {} - - fn read_u8(&self, address: usize) -> u8 { - // SAFETY: AML system-memory accesses are byte-addressable firmware regions. - unsafe { core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u8) } - } - - fn read_u16(&self, address: usize) -> u16 { - // SAFETY: AML system-memory accesses are word-addressable firmware regions. - unsafe { - core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u16) - } - } - - fn read_u32(&self, address: usize) -> u32 { - // SAFETY: AML system-memory accesses are dword-addressable firmware regions. - unsafe { - core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u32) - } - } - - fn read_u64(&self, address: usize) -> u64 { - // SAFETY: AML system-memory accesses are qword-addressable firmware regions. - unsafe { - core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u64) - } - } - - fn write_u8(&self, address: usize, value: u8) { - // SAFETY: AML system-memory accesses are byte-addressable firmware regions. - unsafe { - core::ptr::write_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u8, value) - } - } - - fn write_u16(&self, address: usize, value: u16) { - // SAFETY: AML system-memory accesses are word-addressable firmware regions. - unsafe { - core::ptr::write_volatile( - RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u16, - value, - ) - } - } - - fn write_u32(&self, address: usize, value: u32) { - // SAFETY: AML system-memory accesses are dword-addressable firmware regions. - unsafe { - core::ptr::write_volatile( - RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u32, - value, - ) - } - } - - fn write_u64(&self, address: usize, value: u64) { - // SAFETY: AML system-memory accesses are qword-addressable firmware regions. - unsafe { - core::ptr::write_volatile( - RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u64, - value, - ) - } - } - - fn read_io_u8(&self, port: u16) -> u8 { - Pio::::new(port).read() - } - - fn read_io_u16(&self, port: u16) -> u16 { - Pio::::new(port).read() - } - - fn read_io_u32(&self, port: u16) -> u32 { - Pio::::new(port).read() - } - - fn write_io_u8(&self, port: u16, value: u8) { - Pio::::new(port).write(value) - } - - fn write_io_u16(&self, port: u16, value: u16) { - Pio::::new(port).write(value) - } - - fn write_io_u32(&self, port: u16, value: u32) { - Pio::::new(port).write(value) - } - - fn read_pci_u8(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u8 { - 0 - } - - fn read_pci_u16(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u16 { - 0 - } - - fn read_pci_u32(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u32 { - 0 - } - - fn write_pci_u8(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u8) {} - - fn write_pci_u16(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u16) {} - - fn write_pci_u32(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u32) {} - - fn nanos_since_boot(&self) -> u64 { - 0 - } - - fn stall(&self, microseconds: u64) { - for _ in 0..(microseconds.saturating_mul(64)) { - core::hint::spin_loop(); - } - } - - fn sleep(&self, milliseconds: u64) { - for _ in 0..(milliseconds.saturating_mul(64_000)) { - core::hint::spin_loop(); - } - } - - fn create_mutex(&self) -> Handle { - Handle(AML_MUTEX_IDS.fetch_add(1, Ordering::Relaxed)) - } - - fn acquire(&self, _mutex: Handle, _timeout: u16) -> Result<(), acpi_ext::aml::AmlError> { - Ok(()) - } - - fn release(&self, _mutex: Handle) {} -} - -fn sleep_state_name(state: SleepState) -> &'static str { - match state { - SleepState::S3 => "\\_S3", - SleepState::S5 => "\\_S5", - } -} - -fn encode_sleep_type(value: u16) -> u16 { - if value <= 0x7 { - value << ACPI_SLP_TYP_SHIFT - } else { - value & ACPI_SLP_TYP_MASK - } -} - -fn load_interpreter() -> Result<( - Arc>, - PhysicalMapping, - Interpreter, -), SleepError> { - let root = *ACPI_ROOT_INFO.get().ok_or(SleepError::MissingAcpi)?; - let handler = KernelAcpiHandler; - - // SAFETY: ACPI root info is captured from the firmware-provided, already validated root table. - let tables = unsafe { - AcpiTables::from_rsdt(handler, root.revision, root.root_sdt_address.data()) - .map_err(|_| SleepError::MissingAcpi)? - }; - let fadt = tables.find_table::().ok_or(SleepError::MissingFadt)?; - let registers = Arc::new( - FixedRegisters::new(&fadt, handler).map_err(|_| SleepError::UnsupportedPmControl)?, - ); - let facs_address = fadt.facs_address().map_err(|_| SleepError::MissingFacs)?; - - // SAFETY: The FADT-supplied FACS address is used exactly as described by the ACPI spec. - let facs = unsafe { handler.map_physical_region::(facs_address, core::mem::size_of::()) }; - // SAFETY: The AML interpreter only needs an owned mapping of the same firmware FACS table. - let interpreter_facs = unsafe { - handler.map_physical_region::(facs_address, core::mem::size_of::()) - }; - let dsdt = tables.dsdt().map_err(|_| SleepError::MissingFadt)?; - let interpreter = Interpreter::new(handler, dsdt.revision, Arc::clone(®isters), Some(interpreter_facs)); - - // SAFETY: Each AML table mapping is owned by the interpreter during table loading. - unsafe { - let mapping = handler.map_physical_region::(dsdt.phys_address, dsdt.length as usize); - let stream = core::slice::from_raw_parts( - mapping.virtual_start.as_ptr().byte_add(core::mem::size_of::()) as *const u8, - dsdt.length as usize - core::mem::size_of::(), - ); - interpreter - .load_table(stream) - .map_err(|_| SleepError::UnsupportedAmlOperation)?; - - for ssdt in tables.ssdts() { - let mapping = handler.map_physical_region::(ssdt.phys_address, ssdt.length as usize); - let stream = core::slice::from_raw_parts( - mapping.virtual_start.as_ptr().byte_add(core::mem::size_of::()) as *const u8, - ssdt.length as usize - core::mem::size_of::(), - ); - interpreter - .load_table(stream) - .map_err(|_| SleepError::UnsupportedAmlOperation)?; - } - } - - Ok((registers, facs, interpreter)) -} - -fn sleep_type_data_from_interpreter( - interpreter: &Interpreter, - state: SleepState, -) -> Result { - let name = AmlName::from_str(sleep_state_name(state)).map_err(|_| SleepError::MissingSleepObject)?; - let object = interpreter - .evaluate(name, Vec::new()) - .map_err(|_| SleepError::MissingSleepObject)?; - - let Object::Package(package) = &*object else { - return Err(SleepError::InvalidSleepObject); - }; - - let Some(typa_object) = package.first() else { - return Err(SleepError::InvalidSleepObject); - }; - let Some(typb_object) = package.get(1) else { - return Err(SleepError::InvalidSleepObject); - }; - - let Object::Integer(typa) = &**typa_object else { - return Err(SleepError::InvalidSleepObject); - }; - let Object::Integer(typb) = &**typb_object else { - return Err(SleepError::InvalidSleepObject); - }; - - Ok(SleepTypeData { - a: encode_sleep_type(*typa as u16), - b: encode_sleep_type(*typb as u16), - }) -} - -fn sleep_type_data(state: SleepState) -> Result { - let (_registers, _facs, interpreter) = load_interpreter()?; - sleep_type_data_from_interpreter(&interpreter, state) -} - -fn install_wake_trampoline(stack_rsp: usize, cr3: usize) { - let trampoline_page = Page::containing_address(VirtualAddress::new(WAKE_TRAMPOLINE_PHYS)); - let trampoline_frame = PhysicalAddress::new(WAKE_TRAMPOLINE_PHYS); - - // SAFETY: The 0x8000 low-memory trampoline page is reserved by the kernel for bootstrap stubs. - let (result, _) = unsafe { - let mut mapper = KernelMapper::lock_rw(); - let result = mapper - .map_phys( - trampoline_page.start_address(), - trampoline_frame, - PageFlags::new().execute(true).write(true), - ) - .expect("failed to map S3 wake trampoline page"); - (result, mapper.table().phys().data()) - }; - result.flush(); - - for (index, value) in WAKE_TRAMPOLINE_DATA.iter().enumerate() { - // SAFETY: The trampoline page is mapped writable at the same virtual address as the physical page. - unsafe { - core::ptr::write_volatile((WAKE_TRAMPOLINE_PHYS as *mut u8).add(index), *value); - } - } - - // SAFETY: The wake trampoline layout reserves three qword fields immediately after the jump. - unsafe { - let stack_slot = (WAKE_TRAMPOLINE_PHYS + 8) as *mut u64; - let page_table_slot = stack_slot.add(1); - let code_slot = stack_slot.add(2); - stack_slot.write(stack_rsp as u64); - page_table_slot.write(cr3 as u64); - #[expect(clippy::fn_to_numeric_cast)] - code_slot.write(resume_from_s3_trampoline as usize as u64); - } - - // SAFETY: The trampoline mapping is no longer needed once the physical page has been populated. - let (_frame, _, flush) = unsafe { - KernelMapper::lock_rw() - .unmap_phys(trampoline_page.start_address()) - .expect("failed to unmap S3 wake trampoline page") - }; - flush.flush(); -} - -fn save_descriptor_tables(context: &mut SavedCpuContext) { - // SAFETY: SGDT/SIDT only read the current CPU descriptor-table registers into the provided storage. - unsafe { - core::arch::asm!("sgdt [{}]", in(reg) &mut context.gdtr, options(nostack, preserves_flags)); - core::arch::asm!("sidt [{}]", in(reg) &mut context.idtr, options(nostack, preserves_flags)); - } -} - -fn save_fpu_state(context: &mut SavedCpuContext) { - // SAFETY: The kernel owns the current CPU at suspend entry and the FXSAVE buffer is 64-byte aligned. - unsafe { - core::arch::asm!( - "fxsave64 [{}]", - in(reg) context.fpu.bytes.as_mut_ptr(), - ); - } -} - -fn restore_fpu_state(context: &SavedCpuContext) { - // SAFETY: The saved FXSAVE image belongs to the same CPU context and matches the restore instruction. - unsafe { - core::arch::asm!( - "fxrstor64 [{}]", - in(reg) context.fpu.bytes.as_ptr(), - ); - } -} - -fn save_cpu_context(entry_rsp: usize) -> SavedCpuContext { - let mut context = SavedCpuContext { - entry_rsp, - ..SavedCpuContext::default() - }; - - // SAFETY: Reading control registers and MSRs is required to reconstruct the CPU execution state on wake. - unsafe { - core::arch::asm!( - "mov {}, cr0", - out(reg) context.cr0, - options(nostack, preserves_flags) - ); - core::arch::asm!( - "mov {}, cr2", - out(reg) context.cr2, - options(nostack, preserves_flags) - ); - core::arch::asm!( - "mov {}, cr3", - out(reg) context.cr3, - options(nostack, preserves_flags) - ); - core::arch::asm!( - "mov {}, cr4", - out(reg) context.cr4, - options(nostack, preserves_flags) - ); - core::arch::asm!( - "pushfq", - "pop {}", - out(reg) context.rflags, - options(preserves_flags) - ); - core::arch::asm!("mov {}, rsp", out(reg) context.runtime_rsp, options(nostack, preserves_flags)); - - context.efer = x86::msr::rdmsr(x86::msr::IA32_EFER); - context.fs_base = x86::msr::rdmsr(x86::msr::IA32_FS_BASE); - context.gs_base = x86::msr::rdmsr(x86::msr::IA32_GS_BASE); - context.kernel_gs_base = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE); - } - - save_descriptor_tables(&mut context); - save_fpu_state(&mut context); - context -} - -fn set_firmware_waking_vector(facs: &mut PhysicalMapping, vector: usize) { - facs.firmware_waking_vector = vector as u32; - facs.x_firmware_waking_vector = vector as u64; -} - -fn write_pm1_control_block( - registers: &FixedRegisters, - sleep_type: SleepTypeData, -) -> Result<(), SleepError> { - let current_a = registers - .pm1_control_registers - .pm1a - .read() - .map_err(|_| SleepError::UnsupportedPmControl)? as u16; - let armed_a = (current_a & !(ACPI_SLP_TYP_MASK | ACPI_SLP_EN)) | sleep_type.a; - - registers - .pm1_control_registers - .pm1a - .write(u64::from(armed_a)) - .map_err(|_| SleepError::UnsupportedPmControl)?; - - if let Some(pm1b) = ®isters.pm1_control_registers.pm1b { - let current_b = pm1b.read().map_err(|_| SleepError::UnsupportedPmControl)? as u16; - let armed_b = (current_b & !(ACPI_SLP_TYP_MASK | ACPI_SLP_EN)) | sleep_type.b; - pm1b.write(u64::from(armed_b)) - .map_err(|_| SleepError::UnsupportedPmControl)?; - pm1b.write(u64::from(armed_b | ACPI_SLP_EN)) - .map_err(|_| SleepError::UnsupportedPmControl)?; - } - - // SAFETY: WBINVD is required here to flush dirty cache lines before firmware powers down the CPU package. - unsafe { - core::arch::asm!("wbinvd", options(nostack, preserves_flags)); - } - - registers - .pm1_control_registers - .pm1a - .write(u64::from(armed_a | ACPI_SLP_EN)) - .map_err(|_| SleepError::UnsupportedPmControl)?; - - Ok(()) -} - -#[unsafe(naked)] -unsafe extern "sysv64" fn enter_sleep_raw(state: usize) -> usize { - core::arch::naked_asm!( - "mov rsi, rsp", - "jmp {inner}", - inner = sym enter_sleep_raw_inner, - ); -} - -extern "C" fn enter_sleep_raw_inner(state: usize, entry_rsp: usize) -> usize { - let state = match state { - 3 => SleepState::S3, - 5 => SleepState::S5, - _ => return SleepError::InvalidSleepObject.code(), - }; - - let (registers, mut facs, interpreter) = match load_interpreter() { - Ok(tuple) => tuple, - Err(error) => return error.code(), - }; - let sleep_type = match sleep_type_data_from_interpreter(&interpreter, state) { - Ok(data) => data, - Err(error) => return error.code(), - }; - - let mut context = save_cpu_context(entry_rsp); - context.facs_address = facs.physical_start; - install_wake_trampoline(context.runtime_rsp, context.cr3); - set_firmware_waking_vector(&mut facs, WAKE_TRAMPOLINE_PHYS); - - { - let mut saved = SAVED_CONTEXT.lock(); - *saved = Some(context); - } - - // SAFETY: Suspend entry must not be interrupted while the wake vector and PM1 control block are being armed. - unsafe { - interrupt::disable(); - } - - if let Err(error) = write_pm1_control_block(registers.as_ref(), sleep_type) { - return error.code(); - } - - // SAFETY: The final CLI+HLT sequence is the architectural handoff point after asserting SLP_EN. - unsafe { - core::arch::asm!("cli; hlt", options(nostack)); - } - - SleepError::SleepDidNotEnter.code() -} - -extern "C" fn resume_from_s3_trampoline() -> ! { - let mut saved = SAVED_CONTEXT.lock(); - let context = saved.take().expect("S3 wake trampoline resumed without saved CPU context"); - drop(saved); - - // SAFETY: The saved FACS physical address was captured from the validated FADT during suspend entry. - if context.facs_address != 0 { - let mut facs = unsafe { - KernelAcpiHandler.map_physical_region::( - context.facs_address, - core::mem::size_of::(), - ) - }; - set_firmware_waking_vector(&mut facs, 0); - } - - // SAFETY: The wake trampoline already switched to the saved kernel CR3 and long mode, so the remaining restores are architectural register state only. - unsafe { - x86::msr::wrmsr(x86::msr::IA32_EFER, context.efer); - core::arch::asm!("mov cr3, {}", in(reg) context.cr3, options(nostack)); - core::arch::asm!("mov cr4, {}", in(reg) context.cr4, options(nostack)); - core::arch::asm!("mov cr2, {}", in(reg) context.cr2, options(nostack)); - core::arch::asm!("mov cr0, {}", in(reg) context.cr0, options(nostack)); - core::arch::asm!("lgdt [{}]", in(reg) &context.gdtr, options(nostack)); - core::arch::asm!("lidt [{}]", in(reg) &context.idtr, options(nostack)); - - task::load_tr(SegmentSelector::new(crate::arch::gdt::GDT_TSS as u16, Ring::Ring0)); - - x86::msr::wrmsr(x86::msr::IA32_FS_BASE, context.fs_base); - x86::msr::wrmsr(x86::msr::IA32_GS_BASE, context.gs_base); - x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, context.kernel_gs_base); - } - - restore_fpu_state(&context); - - // SAFETY: Returning with the original entry stack and RFLAGS completes the suspend call as a successful function return. - unsafe { - core::arch::asm!( - "mov rsp, {entry_rsp}", - "push {rflags}", - "popfq", - "xor eax, eax", - "ret", - entry_rsp = in(reg) context.entry_rsp, - rflags = in(reg) context.rflags, - options(noreturn) - ); - } -} - -pub fn enter_sleep_state(state: SleepState) -> core::result::Result<(), SleepError> { - #[cfg(not(target_arch = "x86_64"))] - { - let _ = state; - return Err(SleepError::UnsupportedArch); - } - - #[cfg(target_arch = "x86_64")] - { - let raw = unsafe { - enter_sleep_raw(match state { - SleepState::S3 => 3, - SleepState::S5 => 5, - }) - }; - if raw == SLEEP_RETURN_OK { - Ok(()) - } else { - Err(SleepError::from_code(raw)) - } - } -} - -pub fn available_sleep_states() -> &'static [u8] { - if sleep_type_data(SleepState::S3).is_ok() { - b"S3\nS5\n" - } else { - b"S5\n" - } -} - -pub fn trigger_sleep_request(request: &str) -> Result<(), Error> { - match request.trim() { - "S3" => enter_sleep_state(SleepState::S3).map_err(|_| Error::new(EIO)), - "S5" => enter_sleep_state(SleepState::S5).map_err(|_| Error::new(EIO)), - _ => Err(Error::new(EINVAL)), - } -} diff --git a/src/asm/x86_64/s3_wakeup.asm b/src/asm/x86_64/s3_wakeup.asm deleted file mode 100644 index 7beeccf603..0000000000 --- a/src/asm/x86_64/s3_wakeup.asm +++ /dev/null @@ -1,110 +0,0 @@ -; ACPI S3 wake trampoline -; compiled with nasm by build.rs, copied to physical 0x8000 before S3 entry - -ORG 0x8000 -SECTION .text -USE16 - -trampoline: - jmp short startup_wake - times 8 - ($ - trampoline) nop - .stack: dq 0 - .page_table: dq 0 - .code: dq 0 - -startup_wake: - cli - - xor ax, ax - mov ds, ax - mov es, ax - mov ss, ax - mov sp, 0 - - mov edi, [trampoline.page_table] - mov cr3, edi - - mov eax, cr0 - and al, 11110011b - or al, 00100010b - mov cr0, eax - - mov eax, cr4 - or eax, 1 << 9 | 1 << 7 | 1 << 5 | 1 << 4 - mov cr4, eax - - fninit - - lgdt [gdtr] - - mov ecx, 0xC0000080 - rdmsr - or eax, 1 << 11 | 1 << 8 - wrmsr - - mov ebx, cr0 - or ebx, 1 << 31 | 1 << 16 | 1 - mov cr0, ebx - - jmp gdt.kernel_code:long_mode_wake - -USE64 -long_mode_wake: - mov rax, gdt.kernel_data - mov ds, rax - mov es, rax - mov fs, rax - mov gs, rax - mov ss, rax - - mov rsp, [trampoline.stack] - mov rax, [trampoline.code] - jmp rax - -struc GDTEntry - .limitl resw 1 - .basel resw 1 - .basem resb 1 - .attribute resb 1 - .flags__limith resb 1 - .baseh resb 1 -endstruc - -attrib: - .present equ 1 << 7 - .user equ 1 << 4 - .code equ 1 << 3 - .writable equ 1 << 1 - -flags: - .long_mode equ 1 << 5 - -gdtr: - dw gdt.end + 1 - dq gdt - -gdt: -.null equ $ - gdt - dq 0 - -.kernel_code equ $ - gdt -istruc GDTEntry - at GDTEntry.limitl, dw 0 - at GDTEntry.basel, dw 0 - at GDTEntry.basem, db 0 - at GDTEntry.attribute, db attrib.present | attrib.user | attrib.code - at GDTEntry.flags__limith, db flags.long_mode - at GDTEntry.baseh, db 0 -iend - -.kernel_data equ $ - gdt -istruc GDTEntry - at GDTEntry.limitl, dw 0 - at GDTEntry.basel, dw 0 - at GDTEntry.basem, db 0 - at GDTEntry.attribute, db attrib.present | attrib.user | attrib.writable - at GDTEntry.flags__limith, db 0 - at GDTEntry.baseh, db 0 -iend - -.end equ $ - gdt diff --git a/src/context/context.rs b/src/context/context.rs index 8dce642377..4ee0e56302 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -435,7 +435,9 @@ impl Context { ) -> Option { let is_upper = i.get() & syscall::UPPER_FDTBL_TAG != 0; let result = self.files.write(lock_token.token()).remove_file(i); - + if is_upper && result.is_some() { + crate::info!("Context::remove_file ctx={} upper_idx={}", self.name, i.get()); + } result } @@ -967,7 +969,9 @@ impl FdTbl { let removed_file_opt = fdtbl.get_mut(real_index).and_then(|opt| opt.take()); if removed_file_opt.is_some() { self.active_count -= 1; - + if index & syscall::UPPER_FDTBL_TAG != 0 { + crate::info!("FdTbl::remove_file upper idx={} active={}", real_index, self.active_count); + } } removed_file_opt diff --git a/src/context/file.rs b/src/context/file.rs index 150f483a47..2d3790f147 100644 --- a/src/context/file.rs +++ b/src/context/file.rs @@ -4,7 +4,7 @@ use crate::{ event, scheme::{self, SchemeId}, sync::{CleanLockToken, RwLock, L6}, - syscall::error::{Error, Result, ESTALE}, + syscall::error::Result, }; use alloc::sync::Arc; use syscall::{schemev2::NewFdFlags, RwFlags, O_APPEND, O_NONBLOCK}; @@ -18,7 +18,6 @@ pub struct FileDescription { pub offset: u64, /// The scheme that this file refers to pub scheme: SchemeId, - pub scheme_generation: Option, /// The number the scheme uses to refer to this file pub number: usize, /// The flags passed to open or fcntl(SETFL) @@ -33,52 +32,6 @@ bitflags! { } } impl FileDescription { - pub fn with_generation( - scheme: SchemeId, - scheme_generation: Option, - number: usize, - offset: u64, - flags: u32, - internal_flags: InternalFlags, - ) -> Self { - Self { - offset, - scheme, - scheme_generation, - number, - flags, - internal_flags, - } - } - - pub fn new( - scheme: SchemeId, - number: usize, - offset: u64, - flags: u32, - internal_flags: InternalFlags, - token: &mut CleanLockToken, - ) -> Self { - Self::with_generation( - scheme, - Some(scheme::current_scheme_generation(token.token(), scheme)), - number, - offset, - flags, - internal_flags, - ) - } - - pub fn get_scheme(&self, token: &mut CleanLockToken) -> Result { - if let Some(expected_generation) = self.scheme_generation - && expected_generation != scheme::current_scheme_generation(token.token(), self.scheme) - { - return Err(Error::new(ESTALE)); - } - - scheme::get_scheme(token.token(), self.scheme) - } - pub fn rw_flags(&self, rw: RwFlags) -> u32 { let mut ret = self.flags & !(O_NONBLOCK | O_APPEND) as u32; if rw.contains(RwFlags::APPEND) { @@ -123,7 +76,7 @@ impl FileDescription { pub fn try_close(self, token: &mut CleanLockToken) -> Result<()> { event::unregister_file(self.scheme, self.number, token); - let scheme = self.get_scheme(token)?; + let scheme = scheme::get_scheme(token.token(), self.scheme)?; scheme.close(self.number, token) } @@ -132,12 +85,12 @@ impl FileDescription { impl FileDescriptor { pub fn close(self, token: &mut CleanLockToken) -> Result<()> { { - let (desc, number, internal_flags) = { + let (scheme_id, number, internal_flags) = { let desc = self.description.read(token.token()); - (*desc, desc.number, desc.internal_flags) + (desc.scheme, desc.number, desc.internal_flags) }; if internal_flags.contains(InternalFlags::NOTIFY_ON_NEXT_DETACH) { - let scheme = desc.get_scheme(token)?; + let scheme = scheme::get_scheme(token.token(), scheme_id)?; scheme.detach(number, token)?; } } diff --git a/src/context/memory.rs b/src/context/memory.rs index a84f445e0d..39037db91a 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -64,13 +64,14 @@ impl UnmapResult { return Ok(()); }; - let (scheme, number) = { - let desc = *description.read(token.token()); - (desc.get_scheme(token)?, desc.number) + let (scheme_id, number) = { + let desc = description.write(token.token()); + (desc.scheme, desc.number) }; - let funmap_result = scheme - .kfunmap(number, base_offset, self.size, self.flags, token); + let scheme_opt = scheme::get_scheme(token.token(), scheme_id); + let funmap_result = scheme_opt + .and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags, token)); if let Ok(fd) = Arc::try_unwrap(description) { fd.into_inner().try_close(token)?; @@ -2705,13 +2706,20 @@ fn correct_inner<'l>( // XXX: This is cheating, but guaranteed we won't deadlock because we've dropped addr_space_guard let mut token = unsafe { CleanLockToken::new() }; - let desc = *file_ref.description.read(token.token()); - let scheme = desc.get_scheme(&mut token).map_err(|_| PfError::Segv)?; - let scheme_number = desc.number; - let user_inner = match scheme { - KernelSchemes::User(user) => user.inner, - _ => return Err(PfError::Segv), + let (scheme_id, scheme_number) = { + let desc = &file_ref.description.read(token.token()); + (desc.scheme, desc.number) }; + let user_inner = scheme::get_scheme(token.token(), scheme_id) + .ok() + .and_then(|s| { + if let KernelSchemes::User(user) = s { + Some(user.inner) + } else { + None + } + }) + .ok_or(PfError::Segv)?; let offset = file_ref.base_offset as u64 + (pages_from_grant_start * PAGE_SIZE) as u64; user_inner diff --git a/src/scheme/acpi.rs b/src/scheme/acpi.rs index 6e773a6b2b..9997b7e0e6 100644 --- a/src/scheme/acpi.rs +++ b/src/scheme/acpi.rs @@ -10,8 +10,6 @@ use crate::{ acpi::{RxsdtEnum, RXSDT_ENUM}, arch::x86_64::s3_resume, context::file::InternalFlags, - arch::sleep, - arch::sleep, scheme::{SchemeExt, StrOrBytes}, sync::CleanLockToken, }; @@ -41,9 +39,7 @@ bitflags! { static RXSDT_DATA: Once> = Once::new(); /// Phase I: kstop reason codes. Read via the CheckShutdown - SleepControl, /// AcpiVerb (kcall 2). The reason tells acpid what AML - SleepControl, /// sequence to run. /// /// | Value | Reason | acpid's response | @@ -193,10 +189,8 @@ impl AcpiScheme { } impl KernelScheme for AcpiScheme { - HandleKind::SleepControl => sleep::available_sleep_states().len() as u64, fn scheme_root(&self, _token: &mut CleanLockToken) -> Result { Ok((HandleBits::CAN_READ_RXSDT | HandleBits::CAN_REGISTER_KSTOP).bits()) - HandleKind::SleepControl => sleep::available_sleep_states().len() as u64, } fn kopenat( &self, @@ -256,11 +250,9 @@ impl KernelScheme for AcpiScheme { AcpiVerb::CheckShutdown => { if handle != HandleBits::KSTOP_HANDLE { return Err(Error::new(EINVAL)); - HandleKind::SleepControl => sleep::available_sleep_states(), } // Phase I.5: return the u8 reason, not the // pre-Phase-I.5 bool. acpid's CheckShutdown - HandleKind::SleepControl => sleep::available_sleep_states(), // verb handler is updated to switch on the // reason value. Ok(*KSTOP_FLAG.lock(token.token()) as usize) @@ -332,20 +324,10 @@ impl KernelScheme for AcpiScheme { // the FADT (ACPI 1.0 or very old hardware). } Ok(0) - HandleKind::SleepControl => Stat { - st_mode: MODE_FILE, - st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX), - ..Default::default() - }, } AcpiVerb::EnterS3 => { // Phase II.X.W: acpid has done the AML prep // (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written - HandleKind::SleepControl => Stat { - st_mode: MODE_FILE, - st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX), - ..Default::default() - }, // the trampoline address to FACS via // `SetS3WakingVector`. The acpid now requests // the kernel to enter S3. The kernel's diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index 4a23b3cf4f..c70ac5792b 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -22,10 +22,9 @@ struct Handle { static HANDLES: RwLock> = RwLock::new(HandleMap::new()); -/// Add to the input queue, translating CR to NL (ICRNL) for serial console compatibility. +/// Add to the input queue pub fn debug_input(data: u8, token: &mut CleanLockToken) { - let translated = if data == b'\r' { b'\n' } else { data }; - INPUT.send(translated, token); + INPUT.send(data, token); } // Notify readers of input updates @@ -107,16 +106,12 @@ impl KernelScheme for DebugScheme { fn fevent( &self, id: usize, - flags: EventFlags, + _flags: EventFlags, token: &mut CleanLockToken, ) -> Result { let _handle = *HANDLES.read(token.token()).get(id)?; - let mut ready = EventFlags::empty(); - if flags.contains(EventFlags::EVENT_READ) { - ready |= EventFlags::EVENT_READ; - } - Ok(ready) + Ok(EventFlags::empty()) } fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 68a13a7397..0923c47ef4 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -12,7 +12,7 @@ use alloc::{ }; use core::{ str, - sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + sync::atomic::{AtomicUsize, Ordering}, }; use hashbrown::hash_map::{self, DefaultHashBuilder, HashMap}; use spin::Once; @@ -167,7 +167,6 @@ enum Handle { /// Schemes list static HANDLES: Once>> = Once::new(); -static SCHEME_GENERATIONS: Once>> = Once::new(); static SCHEME_LIST_NEXT_ID: AtomicUsize = AtomicUsize::new(MAX_GLOBAL_SCHEMES); static SCHEME_LIST_ID: AtomicUsize = AtomicUsize::new(0); @@ -203,10 +202,6 @@ fn init_schemes() -> RwLock> { RwLock::new(handles) } -fn init_scheme_generations() -> RwLock> { - RwLock::new(HashMap::new()) -} - /// Get a handle to a scheme. pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result { match handles().read(token).get(&scheme_id) { @@ -215,33 +210,10 @@ pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result, scheme_id: SchemeId) -> u64 { - scheme_generations() - .read(token) - .get(&scheme_id) - .map(|generation| generation.load(Ordering::Acquire)) - .unwrap_or(0) -} - fn handles<'a>() -> &'a RwLock> { HANDLES.call_once(init_schemes) } -fn scheme_generations<'a>() -> &'a RwLock> { - SCHEME_GENERATIONS.call_once(init_scheme_generations) -} - -fn increment_scheme_generation(scheme_id: SchemeId, token: &mut CleanLockToken) { - match scheme_generations().write(token.token()).entry(scheme_id) { - hash_map::Entry::Occupied(entry) => { - entry.get().fetch_add(1, Ordering::AcqRel); - } - hash_map::Entry::Vacant(entry) => { - entry.insert(AtomicU64::new(1)); - } - } -} - /// Scheme list type pub struct SchemeList; @@ -286,14 +258,9 @@ impl SchemeList { /// Remove a scheme fn remove(&self, id: usize, token: &mut CleanLockToken) { - let scheme_id = SchemeId(id); - let scheme = handles().write(token.token()).remove(&scheme_id); + let scheme = handles().write(token.token()).remove(&SchemeId(id)); assert!(scheme.is_some()); - if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme.as_ref() { - user.inner.fail_pending_calls(token); - } - increment_scheme_generation(scheme_id, token); if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme && let Some(user) = Arc::into_inner(user.inner) { @@ -318,32 +285,32 @@ impl KernelScheme for SchemeList { token: &mut CleanLockToken, ) -> Result { let scheme_id = SchemeId(scheme_id); - let maybe_inner = { - let handles = handles().read(token.token()); - match handles.get(&scheme_id).ok_or(Error::new(EBADF))? { - Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => Some(inner.clone()), - Handle::SchemeCreationCapability => None, - _ => return Err(Error::new(EBADF)), + match handles() + .read(token.token()) + .get(&scheme_id) + .ok_or(Error::new(EBADF))? + { + Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => { + let inner = inner.clone(); + assert!(scheme_id == inner.scheme_id); + let scheme = scheme_id; + let params = unsafe { user_buf.read_exact::()? }; + + return Ok(OpenResult::External(Arc::new(RwLock::new( + FileDescription { + scheme, + number: params.number, + offset: params.offset, + flags: params.flags as u32, + internal_flags: InternalFlags::from_extra0(params.internal_flags) + .ok_or(Error::new(EINVAL))?, + }, + )))); } + Handle::SchemeCreationCapability => (), + _ => return Err(Error::new(EBADF)), }; - if let Some(inner) = maybe_inner { - assert!(scheme_id == inner.scheme_id); - let params = unsafe { user_buf.read_exact::()? }; - - return Ok(OpenResult::External(Arc::new(RwLock::new( - FileDescription::new( - scheme_id, - params.number, - params.offset, - params.flags as u32, - InternalFlags::from_extra0(params.internal_flags) - .ok_or(Error::new(EINVAL))?, - token, - ), - )))); - } - const EXPECTED: &[u8] = b"create-scheme"; let mut buf = [0u8; EXPECTED.len()]; @@ -808,7 +775,6 @@ pub enum OpenResult { SchemeLocal(usize, InternalFlags), External(Arc), } - pub groups: alloc::vec::Vec, pub struct CallerCtx { pub pid: usize, pub uid: u32, @@ -817,7 +783,6 @@ pub struct CallerCtx { } impl CallerCtx { pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self { - groups: self.groups, if self.uid == 0 && self.gid == 0 { Self { pid: self.pid, diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index ff337254d6..189cf538ab 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -783,7 +783,13 @@ impl KernelScheme for ProcScheme { } (pc, uc, ul, pl) }; - + let ctx_name = context.read(token.token()).name; + info!( + "AwaitingFiletableChange applied: ctx='{}' posix_fds={} upper_fds={}", + ctx_name, posix_count, upper_count + ); + info!("upper FDs present:{}", upper_list); + info!("posix FDs present:{}", posix_list); context.write(token.token()).files = new_ft; } _ => (), diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 306461cc7e..5d078b6df3 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -80,7 +80,6 @@ const ONE: NonZeroUsize = match NonZeroUsize::new(1) { Some(one) => one, None => unreachable!(), }; -const MAX_SPURIOUS_WAKEUPS: usize = 100; enum ParsedCqe { TriggerFevent { @@ -210,8 +209,6 @@ impl UserInner { caller_responsible: &mut PageSpan, token: &mut CleanLockToken, ) -> Result { - let mut remaining_spurious_wakeups = MAX_SPURIOUS_WAKEUPS; - { // Disable preemption to avoid context switches between setting the // process state and sending the scheme request. The process is made @@ -264,10 +261,7 @@ impl UserInner { }; let states = self.states.lock(token.token()); - let (mut states, mut state_token) = states.into_split(); - let mut timed_out_descriptions = None; - let mut remove_state = false; - let mut timed_out = false; + let (mut states, mut token) = states.into_split(); match states.get_mut(sqe.tag as usize) { // invalid state None => return Err(Error::new(EBADFD)), @@ -280,35 +274,24 @@ impl UserInner { fds, } => { let maybe_eintr = - eintr_if_sigkill(&mut callee_responsible, &mut state_token.token()); - - if maybe_eintr.is_ok() { - remaining_spurious_wakeups = - remaining_spurious_wakeups.saturating_sub(1); - } - - if maybe_eintr.is_ok() && remaining_spurious_wakeups == 0 { - timed_out_descriptions = Some(Self::collect_descriptions_to_close(fds)); - remove_state = true; - } else { - *o = State::Waiting { - canceling: true, - callee_responsible, - context, - fds, - }; - } + eintr_if_sigkill(&mut callee_responsible, &mut token.token()); + *o = State::Waiting { + canceling: true, + callee_responsible, + context, + fds, + }; maybe_eintr?; - if remove_state { - states.remove(sqe.tag as usize); - timed_out = true; - } else { - context::current() - .write(state_token.token()) - .block("UserInner::call (woken up after cancelation request)"); - } + context::current() + .write(token.token()) + .block("UserInner::call (woken up after cancelation request)"); + + // We do not want to drop the lock before blocking + // as if we get preempted in between we might miss a + // wakeup. + drop(states); } // spurious wakeup State::Waiting { @@ -317,76 +300,60 @@ impl UserInner { context, mut callee_responsible, } => { + let maybe_eintr = eintr_if_sigkill(&mut callee_responsible, &mut token); let current_context = context::current(); - let maybe_eintr = - eintr_if_sigkill(&mut callee_responsible, &mut state_token); - if maybe_eintr.is_ok() { - remaining_spurious_wakeups = - remaining_spurious_wakeups.saturating_sub(1); - } - - if maybe_eintr.is_ok() && remaining_spurious_wakeups == 0 { - timed_out_descriptions = Some(Self::collect_descriptions_to_close(fds)); - remove_state = true; - } else { - *o = State::Waiting { - // Currently we treat all spurious wakeups to have the same behavior - // as signals (i.e., we send a cancellation request). It is not something - // that should happen, but it certainly can happen, for example if a context - // is awoken through its thread handle without setting any sig bits, or if the - // caller clears its own sig bits. If it actually is a signal, then it is the - // intended behavior. - canceling: true, - fds, - context, - callee_responsible, - }; - } + *o = State::Waiting { + // Currently we treat all spurious wakeups to have the same behavior + // as signals (i.e., we send a cancellation request). It is not something + // that should happen, but it certainly can happen, for example if a context + // is awoken through its thread handle without setting any sig bits, or if the + // caller clears its own sig bits. If it actually is a signal, then it is the + // intended behavior. + canceling: true, + fds, + context, + callee_responsible, + }; maybe_eintr?; - if remove_state { - states.remove(sqe.tag as usize); - timed_out = true; - } else { - // We do not want to preempt between sending the - // cancellation and blocking again where we might - // miss a wakeup. - let mut preempt = - PreemptGuardL1::new(¤t_context, &mut state_token); - let token = preempt.token(); + // We do not want to preempt between sending the + // cancellation and blocking again where we might + // miss a wakeup. + let mut preempt = PreemptGuardL1::new(¤t_context, &mut token); + let token = preempt.token(); - self.todo.send_locked( - Sqe { - opcode: Opcode::Cancel as u8, - sqe_flags: SqeFlags::ONEWAY, - tag: sqe.tag, - ..Default::default() - }, - token.token(), - ); - event::trigger_locked( - self.root_id, - self.scheme_id.get(), - EVENT_READ, - token.token(), - ); + self.todo.send_locked( + Sqe { + opcode: Opcode::Cancel as u8, + sqe_flags: SqeFlags::ONEWAY, + tag: sqe.tag, + ..Default::default() + }, + token.token(), + ); + event::trigger_locked( + self.root_id, + self.scheme_id.get(), + EVENT_READ, + token.token(), + ); - // 1. If cancellation was requested and arrived - // before the scheme processed the request, an - // acknowledgement will be sent back after the - // cancellation is processed and we will be woken up - // again. State will be State::Responded then. - // - // 2. If cancellation was requested but the scheme - // already processed the request, we will receive - // the actual response next and woken up again. - // State will be State::Responded then. - context::current() - .write(token.token()) - .block("UserInner::call (spurious wakeup)"); - } + // 1. If cancellation was requested and arrived + // before the scheme processed the request, an + // acknowledgement will be sent back after the + // cancellation is processed and we will be woken up + // again. State will be State::Responded then. + // + // 2. If cancellation was requested but the scheme + // already processed the request, we will receive + // the actual response next and woken up again. + // State will be State::Responded then. + context::current() + .write(token.token()) + .block("UserInner::call (spurious wakeup)"); + drop(states); } // invalid state @@ -401,70 +368,10 @@ impl UserInner { } }, } - - if let Some(descriptions) = timed_out_descriptions { - drop(states); - for desc in descriptions { - let _ = desc.try_close(token); - } - } - - if timed_out { - return Err(Error::new(ETIMEDOUT)); - } } } } - fn collect_descriptions_to_close( - fds: Vec>, - ) -> Vec { - fds.into_iter() - .filter_map(|fd| Arc::try_unwrap(fd).ok()) - .map(RwLock::into_inner) - .collect() - } - - pub fn fail_pending_calls(&self, token: &mut CleanLockToken) { - let descriptions_to_close = { - let mut states_lock = self.states.lock(token.token()); - let (states, mut lock_token) = states_lock.token_split(); - let mut descriptions_to_close = Vec::new(); - let mut states_to_remove = Vec::new(); - - for (id, state) in states.iter_mut() { - match mem::replace(state, State::Placeholder) { - State::Waiting { context, fds, .. } => { - descriptions_to_close.extend(Self::collect_descriptions_to_close(fds)); - - match context.upgrade() { - Some(context) => { - *state = State::Responded(Response::Regular( - Err(Error::new(ENODEV)), - 0, - false, - )); - context.write(lock_token.token()).unblock(); - } - None => states_to_remove.push(id), - } - } - old_state => *state = old_state, - } - } - - for id in states_to_remove { - states.remove(id); - } - - descriptions_to_close - }; - - for desc in descriptions_to_close { - let _ = desc.try_close(token); - } - } - /// Map a readable structure to the scheme's userspace and return the /// pointer #[must_use = "copying back to head/tail buffers can fail"] @@ -1376,7 +1283,6 @@ impl UserInner { } pub fn into_drop(self, token: &mut CleanLockToken) { - self.fail_pending_calls(token); self.todo.condition.into_drop(token); } } diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 353fed4b80..4596adf7fd 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -140,7 +140,7 @@ extern "C" fn userspace_init() { info!("userspace_init: entered bootstrap context"); let mut token = unsafe { CleanLockToken::new() }; let bootstrap = BOOTSTRAP.get().expect("BOOTSTRAP was not set"); - + info!("userspace_init: calling usermode_bootstrap"); unsafe { crate::syscall::process::usermode_bootstrap(bootstrap, &mut token) } } @@ -154,15 +154,6 @@ static BOOTSTRAP: spin::Once = spin::Once::new(); pub(crate) static AP_READY: AtomicBool = AtomicBool::new(false); static BSP_READY: AtomicBool = AtomicBool::new(false); -#[cold] -fn halt_boot(message: &str) -> ! { - print!("{message}"); - println!("Kernel boot cannot continue. Halting."); - loop { - hint::spin_loop(); - } -} - /// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { let mut token = unsafe { CleanLockToken::new() }; @@ -194,11 +185,13 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { context.euid = 0; context.egid = 0; } - Err(_err) => halt_boot("FATAL: failed to spawn first userspace process userspace_init\n"), + Err(err) => { + panic!("failed to spawn userspace_init: {:?}", err); + } } numa::dump_info(); - + info!("kmain: dump_info done, calling run_userspace"); run_userspace(&mut token) } @@ -248,7 +241,9 @@ fn run_userspace(token: &mut CleanLockToken) -> ! { interrupt::enable_and_nop(); } SwitchResult::AllContextsIdle => { - + if switch_count == 0 { + info!("run_userspace: AllContextsIdle on first switch — no runnable contexts"); + } switch_count += 1; // Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired. interrupt::enable_and_halt(); diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index 4acbff26f0..e8a85611cf 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -12,7 +12,7 @@ use crate::{ memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions}, }, memory::{Page, VirtualAddress, PAGE_SIZE}, - scheme::{FileHandle, KernelScheme, OpenResult, StrOrBytes}, + scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes}, sync::{CleanLockToken, RwLock}, syscall::{data::Stat, error::*, flag::*}, }; @@ -45,7 +45,7 @@ pub fn file_op_generic_ext( (file, desc) }; - let scheme = desc.get_scheme(token)?; + let scheme = scheme::get_scheme(token.token(), desc.scheme)?; op(&*scheme, file.description, desc, token) } @@ -165,18 +165,14 @@ pub fn openat_into( token: &mut CleanLockToken, ) -> Result { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; - let desc = { + let (scheme_id, number) = { let current_lock = context::current(); let mut current = current_lock.read(token.token()); - let (context, mut context_token) = current.token_split(); - let pipe = context - .get_file(fh, &mut context_token) - .ok_or(Error::new(EBADF))?; - *pipe.description.read(context_token.token()) + let (context, mut token) = current.token_split(); + let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?; + let desc = pipe.description.read(token.token()); + (desc.scheme, desc.number) }; - let scheme = desc.get_scheme(token)?; - let number = desc.number; - let scheme_id = desc.scheme; let caller_ctx = context::current().read(token.token()).caller_ctx(); let new_description = { let scheme = scheme::get_scheme(token.token(), scheme_id)?; @@ -564,6 +560,8 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) // Communicate fcntl with scheme if cmd != F_GETFD && cmd != F_SETFD { + let scheme = scheme::get_scheme(token.token(), scheme_id)?; + scheme.fcntl(number, cmd, arg, token)?; }; @@ -656,11 +654,13 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken let path = RedoxPath::from_absolute(&path_buf).ok_or(Error::new(EINVAL))?; let (_, reference) = path.as_parts().ok_or(Error::new(EINVAL))?; - let (number, scheme) = { - let desc = *file.description.read(token.token()); - (desc.number, desc.get_scheme(token)?) + let (number, scheme_id) = { + let desc = file.description.read(token.token()); + (desc.number, desc.scheme) }; + let scheme = scheme::get_scheme(token.token(), scheme_id)?; + // TODO: Check EXDEV. /* if scheme_id != description.scheme { @@ -866,7 +866,10 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) -> Ok(bytes_read) } pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { - + if fd.get() <= 5 || fd.get() > 0x4000000000000000 { + let name = context::current().read(token.token()).name; + info!("sys_write ENTER fd={} buf_len={} ctx='{}'", fd.get(), buf.len(), name); + } let result = (|| { let (bytes_written, desc_arc, desc) = file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| { diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 5942007e23..5f5209ba21 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -148,7 +148,9 @@ const KERNEL_METADATA_PAGE_COUNT: usize = syscall::KERNEL_METADATA_SIZE / PAGE_S }; pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockToken) { + info!("usermode_bootstrap: ENTER, page_count={}", bootstrap.page_count); assert_ne!(bootstrap.page_count, 0); + info!("usermode_bootstrap: page_count ok, getting addr_space"); { let addr_space = Arc::clone( @@ -157,7 +159,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok .addr_space() .expect("expected bootstrap context to have an address space"), ); - + info!("usermode_bootstrap: got addr_space, calling mmap for {} pages", bootstrap.page_count); let base = Page::containing_address(VirtualAddress::new(PAGE_SIZE)); let flags = MapFlags::MAP_FIXED_NOREPLACE @@ -171,7 +173,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok let _base_page = { let mut lock_token = token.token(); let mut addr_space_lock = addr_space.acquire_write(lock_token.downgrade()); - + info!("usermode_bootstrap: acquired write lock, calling mmap"); addr_space_lock .mmap( &addr_space, @@ -192,7 +194,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok ) .expect("Failed to allocate bootstrap pages") }; - + info!("usermode_bootstrap: mmap done"); // Insert kernel schemes root capabilities. let mut kernel_schemes_infos = @@ -215,7 +217,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok matches!(scheme, GlobalSchemes::Proc), token, ); - + info!("inserted fd {:#x} for scheme_id {} cap_fd {}", inserted, scheme.scheme_id().get(), cap_fd); inserted }; } @@ -298,7 +300,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok .expect("failed to copy memory to bootstrap"); let bootstrap_entry = u64::from_le_bytes(bootstrap_slice[0x1a..0x22].try_into().unwrap()); - + info!("Bootstrap entry point: {:#x}, pages: {}, stack at {:#x}", bootstrap_entry, bootstrap.page_count, PAGE_SIZE + bootstrap.page_count * PAGE_SIZE + 8 * PAGE_SIZE); assert_ne!(bootstrap_entry, 0); // Map a minimal user stack for the bootstrap process. Without this, @@ -361,7 +363,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok regs.set_instr_pointer(bootstrap_entry.try_into().unwrap()); regs.set_stack_pointer(stack_top_vaddr); } - + info!("usermode_bootstrap: EXIT, entry={:#x}, sp={:#x}", bootstrap_entry, stack_top_vaddr); } unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] { @@ -374,26 +376,23 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] } fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize { - let description = Arc::new(RwLock::new(FileDescription::new( - scheme, - number, - 0, - (O_CREAT | O_RDWR) as u32, - InternalFlags::empty(), - token, - ))); - let current_lock = context::current(); let mut current = current_lock.read(token.token()); - let (context, mut context_token) = current.token_split(); + let (context, mut token) = current.token_split(); context .add_file_min( FileDescriptor { - description, + description: Arc::new(RwLock::new(FileDescription { + scheme, + number, + offset: 0, + flags: (O_CREAT | O_RDWR) as u32, + internal_flags: InternalFlags::empty(), + })), cloexec, }, syscall::flag::UPPER_FDTBL_TAG + scheme.get(), - &mut context_token, + &mut token, ) .expect("failed to insert fd to current context") .get()