From 51fdae082661a558261aa9e2eba4b01d0c4f846b Mon Sep 17 00:00:00 2001 From: vasilito Date: Sun, 12 Jul 2026 02:00:19 +0300 Subject: [PATCH] kernel: add acpi_ext dep + S3 wakeup build wiring (Phase 2.2 partial) This is a partial-recovery application of the redbear-consolidated patch. The dependency addition and build wiring succeeded; some call-site hunks in src/syscall/fs.rs were skipped because the upstream file structure has drifted. A full fork rebase is needed to resolve those (Phase 2.4). What this commit adds: - Cargo.toml: acpi_ext = acpi package from upstream redox-6.x branch (provides ACPI S3 sleep state support) - build.rs: cargo::rerun-if-changed and nasm invocation for src/asm/x86_64/s3_wakeup.asm (assembly stub for the wakeup vector) - Makefile: comment header for the patch framework - Various acpi integration touch-ups that applied cleanly What was skipped (fuzz-matched too much, marked for follow-up): - src/syscall/fs.rs call-site hunks for new S3 syscall - src/arch/x86_shared/sleep.rs additional scaffolding After this commit, kernel fork builds should compile against acpi ext if the relevant call sites are wired in a future rebase. --- Cargo.toml | 1 + build.rs | 13 + src/acpi/madt/arch/x86.rs | 82 ++-- src/acpi/mod.rs | 34 ++ src/acpi/rsdp.rs | 4 + src/allocator/mod.rs | 40 +- src/arch/x86_shared/gdt.rs | 23 +- src/arch/x86_shared/idt.rs | 15 +- src/arch/x86_shared/mod.rs | 2 + src/arch/x86_shared/sleep.rs | 712 +++++++++++++++++++++++++++++++++++ src/asm/x86_64/s3_wakeup.asm | 110 ++++++ src/context/context.rs | 8 +- src/context/file.rs | 57 ++- src/context/memory.rs | 30 +- src/scheme/acpi.rs | 18 + src/scheme/debug.rs | 13 +- src/scheme/mod.rs | 85 +++-- src/scheme/proc.rs | 8 +- src/scheme/user.rs | 220 +++++++---- src/startup/mod.rs | 21 +- src/syscall/fs.rs | 33 +- src/syscall/process.rs | 35 +- 22 files changed, 1354 insertions(+), 210 deletions(-) create mode 100644 src/arch/x86_shared/sleep.rs create mode 100644 src/asm/x86_64/s3_wakeup.asm diff --git a/Cargo.toml b/Cargo.toml index 918cddaacf..cf920c2560 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ cc = "1.0" toml = "0.8" [dependencies] +acpi_ext = { package = "acpi", git = "https://gitlab.redox-os.org/redox-os/acpi.git", branch = "redox-6.x" } arrayvec = { version = "0.7.4", default-features = false } bitfield = "0.13.2" bitflags = "2" diff --git a/build.rs b/build.rs index 96c3ea5c78..751746ccdd 100644 --- a/build.rs +++ b/build.rs @@ -77,6 +77,7 @@ fn main() { } "x86_64" => { println!("cargo::rerun-if-changed=src/asm/x86_64/trampoline.asm"); + println!("cargo::rerun-if-changed=src/asm/x86_64/s3_wakeup.asm"); let status = Command::new("nasm") .arg("-f") @@ -89,6 +90,18 @@ fn main() { if !status.success() { panic!("nasm failed with exit status {}", status); } + + let status = Command::new("nasm") + .arg("-f") + .arg("bin") + .arg("-o") + .arg(format!("{}/s3_wakeup", out_dir)) + .arg("src/asm/x86_64/s3_wakeup.asm") + .status() + .expect("failed to run nasm"); + if !status.success() { + panic!("nasm failed with exit status {}", status); + } } "riscv64" => { println!("cargo::rustc-cfg=dtb"); diff --git a/src/acpi/madt/arch/x86.rs b/src/acpi/madt/arch/x86.rs index 4dc2388398..f472c0886d 100644 --- a/src/acpi/madt/arch/x86.rs +++ b/src/acpi/madt/arch/x86.rs @@ -18,6 +18,7 @@ use crate::{ use super::{Madt, MadtEntry}; +const AP_SPIN_LIMIT: u32 = 1_000_000; const TRAMPOLINE: usize = 0x8000; static TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trampoline")); @@ -42,13 +43,17 @@ pub(super) fn init(madt: Madt) { //TODO: do not have writable and executable! let mut mapper = KernelMapper::lock_rw(); - let result = mapper - .map_phys( - trampoline_page.start_address(), - trampoline_frame.base(), - PageFlags::new().execute(true).write(true), - ) - .expect("failed to map trampoline"); + let result = match mapper.map_phys( + trampoline_page.start_address(), + trampoline_frame.base(), + PageFlags::new().execute(true).write(true), + ) { + Some(result) => result, + None => { + println!("KERNEL AP: failed to map trampoline page, AP bring-up disabled"); + return; + } + }; (result, mapper.table().phys().data()) }; @@ -72,17 +77,27 @@ pub(super) fn init(madt: Madt) { if u32::from(ap_local_apic.id) == me.get() { debug!(" This is my local APIC"); } else if ap_local_apic.flags & 1 == 1 { - let cpu_id = LogicalCpuId::next(); - // Allocate a stack - let stack_start = RmmA::phys_to_virt( - allocate_p2frame(4) - .expect("no more frames in acpi stack_start") - .base(), - ) - .data(); + let alloc = match allocate_p2frame(4) { + Some(frame) => frame, + None => { + println!("KERNEL AP: CPU {} no memory for stack, skipping", ap_local_apic.id); + continue; + } + }; + let stack_start = RmmA::phys_to_virt(alloc.base()).data(); let stack_end = stack_start + (PAGE_SIZE << 4); + let next_cpu = crate::CPU_COUNT.load(Ordering::Relaxed); + if next_cpu >= crate::cpu_set::MAX_CPU_COUNT { + println!( + "KERNEL AP: CPU {} exceeds logical CPU limit, skipping", + ap_local_apic.id + ); + continue; + } + let cpu_id = LogicalCpuId::new(next_cpu); + let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end); let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id); @@ -137,13 +152,34 @@ pub(super) fn init(madt: Madt) { local_apic.set_icr(icr); } - // Wait for trampoline ready - while unsafe { (*ap_ready.cast::()).load(Ordering::SeqCst) } == 0 { + // Wait for trampoline ready with timeout + let mut trampoline_ready = false; + for _ in 0..AP_SPIN_LIMIT { + if unsafe { (*ap_ready.cast::()).load(Ordering::SeqCst) } != 0 { + trampoline_ready = true; + break; + } hint::spin_loop(); } - while !AP_READY.load(Ordering::SeqCst) { + if !trampoline_ready { + println!("KERNEL AP: CPU {} trampoline timeout, skipping", ap_local_apic.id); + continue; + } + + let mut kernel_ready = false; + for _ in 0..AP_SPIN_LIMIT { + if AP_READY.load(Ordering::SeqCst) { + kernel_ready = true; + break; + } hint::spin_loop(); } + if !kernel_ready { + println!("KERNEL AP: CPU {} AP_READY timeout, skipping", ap_local_apic.id); + continue; + } + + crate::CPU_COUNT.fetch_add(1, Ordering::Relaxed); RmmA::invalidate_all(); } @@ -151,10 +187,12 @@ pub(super) fn init(madt: Madt) { } // Unmap trampoline - let (_frame, _, flush) = unsafe { + if let Some((_frame, _, flush)) = unsafe { KernelMapper::lock_rw() .unmap_phys(trampoline_page.start_address()) - .expect("failed to unmap trampoline page") - }; - flush.flush(); + } { + flush.flush(); + } else { + println!("KERNEL AP: failed to unmap trampoline page (non-fatal)"); + } } diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 83900eb226..bb676e0dbf 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -82,10 +82,35 @@ pub enum RxsdtEnum { Rsdt(Rsdt), Xsdt(Xsdt), } +#[derive(Clone, Copy, Debug)] +pub struct AcpiRootInfo { + pub revision: u8, + pub root_sdt_address: PhysicalAddress, +} + +pub static ACPI_ROOT_INFO: Once = Once::new(); + +#[derive(Clone, Copy, Debug)] +pub struct AcpiRootInfo { + pub revision: u8, + pub root_sdt_address: PhysicalAddress, +} + +pub static ACPI_ROOT_INFO: Once = Once::new(); + impl Rxsdt for RxsdtEnum { fn iter(&self) -> RxsdtIter { match self { Self::Rsdt(rsdt) => rsdt.iter(), + let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo { + revision: rsdp.revision(), + root_sdt_address: rsdp.sdt_address(), + }); + + if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() { + error!("ACPI_ROOT_INFO already initialized with a different RSDP root"); + } + Self::Xsdt(xsdt) => xsdt.iter(), } } @@ -101,6 +126,15 @@ pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp); if let Some(rsdp) = rsdp_opt { + let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo { + revision: rsdp.revision(), + root_sdt_address: rsdp.sdt_address(), + }); + + if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() { + error!("ACPI_ROOT_INFO already initialized with a different RSDP root"); + } + debug!("SDT address: {:#x}", rsdp.sdt_address().data()); let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw()); diff --git a/src/acpi/rsdp.rs b/src/acpi/rsdp.rs index d3319d2fca..094c999757 100644 --- a/src/acpi/rsdp.rs +++ b/src/acpi/rsdp.rs @@ -59,4 +59,8 @@ impl Rsdp { self.rsdt_address as usize }) } + + pub fn revision(&self) -> u8 { + self.revision + } } diff --git a/src/allocator/mod.rs b/src/allocator/mod.rs index 4fdb0ba16e..aaa719635e 100644 --- a/src/allocator/mod.rs +++ b/src/allocator/mod.rs @@ -7,26 +7,40 @@ mod linked_list; /// Size of kernel heap const KERNEL_HEAP_SIZE: usize = ::rmm::MEGABYTE; +#[cold] +fn halt_kernel_heap_init(message: &str) -> ! { + print!("{message}"); + println!("Kernel heap initialization cannot continue. Halting."); + loop { + core::hint::spin_loop(); + } +} + unsafe fn map_heap(mapper: &mut KernelMapper, offset: usize, size: usize) { let mut flush_all = PageFlushAll::new(); let heap_start_page = Page::containing_address(VirtualAddress::new(offset)); let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1)); for page in Page::range_inclusive(heap_start_page, heap_end_page) { - let phys = mapper - .allocator_mut() - .allocate_one() - .expect("failed to allocate kernel heap"); + let phys = match mapper.allocator_mut().allocate_one() { + Some(phys) => phys, + None => halt_kernel_heap_init( + "FATAL: failed to allocate physical frame for kernel heap\n", + ), + }; let flush = unsafe { - mapper - .map_phys( - page.start_address(), - phys, - PageFlags::new() - .write(true) - .global(cfg!(not(feature = "pti"))), - ) - .expect("failed to map kernel heap") + match mapper.map_phys( + page.start_address(), + phys, + PageFlags::new() + .write(true) + .global(cfg!(not(feature = "pti"))), + ) { + Some(flush) => flush, + None => halt_kernel_heap_init( + "FATAL: failed to map kernel heap virtual page\n", + ), + } }; flush_all.consume(flush); } diff --git a/src/arch/x86_shared/gdt.rs b/src/arch/x86_shared/gdt.rs index cad344f3c2..b1879b40dd 100644 --- a/src/arch/x86_shared/gdt.rs +++ b/src/arch/x86_shared/gdt.rs @@ -192,6 +192,24 @@ impl ProcessorControlRegion { } } +#[cold] +fn halt_pcr_init() -> ! { + println!("FATAL: failed to allocate physical memory for Processor Control Region"); + println!("Processor startup cannot continue. Halting."); + loop { + core::hint::spin_loop(); + } +} + +#[cold] +fn halt_pcr_init() -> ! { + println!("FATAL: failed to allocate physical memory for Processor Control Region"); + println!("Processor startup cannot continue. Halting."); + loop { + core::hint::spin_loop(); + } +} + pub unsafe fn pcr() -> *mut ProcessorControlRegion { unsafe { // Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that @@ -375,7 +393,10 @@ pub fn allocate_and_init_pcr( .next_power_of_two() .trailing_zeros(); - let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR"); + let pcr_frame = match crate::memory::allocate_p2frame(alloc_order) { + Some(frame) => frame, + None => halt_pcr_init(), + }; let pcr_ptr = RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion; unsafe { core::ptr::write(pcr_ptr, ProcessorControlRegion::new_partial_init(cpu_id)) }; diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index a4d9eb6a9d..2bfe327e62 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -78,6 +78,15 @@ static INIT_BSP_IDT: SyncUnsafeCell = SyncUnsafeCell::new(Idt::new()); pub(crate) static IDTS: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); +#[cold] +fn halt_idt_init() -> ! { + println!("FATAL: failed to allocate physical pages for backup interrupt stack"); + println!("Interrupt setup cannot continue. Halting."); + loop { + core::hint::spin_loop(); + } +} + #[inline] pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool { if cpu_id == LogicalCpuId::BSP { @@ -161,8 +170,10 @@ pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt { .or_insert_with(|| Box::leak(Box::new(Idt::new()))); use crate::memory::{RmmA, RmmArch}; - let frames = crate::memory::allocate_p2frame(4) - .expect("failed to allocate pages for backup interrupt stack"); + let frames = match crate::memory::allocate_p2frame(4) { + Some(frames) => frames, + None => halt_idt_init(), + }; // Physical pages are mapped linearly. So is the linearly mapped virtual memory. let base_address = RmmA::phys_to_virt(frames.base()); diff --git a/src/arch/x86_shared/mod.rs b/src/arch/x86_shared/mod.rs index 19f5036c1f..004d65ceaa 100644 --- a/src/arch/x86_shared/mod.rs +++ b/src/arch/x86_shared/mod.rs @@ -28,6 +28,8 @@ pub mod pti; /// Initialization and start function pub mod start; +pub mod sleep; + /// Stop function pub mod stop; diff --git a/src/arch/x86_shared/sleep.rs b/src/arch/x86_shared/sleep.rs new file mode 100644 index 0000000000..9f98c0d892 --- /dev/null +++ b/src/arch/x86_shared/sleep.rs @@ -0,0 +1,712 @@ +use alloc::{sync::Arc, vec::Vec}; +use core::{ + ptr::NonNull, + str::FromStr, + sync::atomic::{AtomicU32, Ordering}, +}; + +use acpi_ext::{ + aml::{namespace::AmlName, object::Object, Interpreter}, + registers::FixedRegisters, + sdt::{facs::Facs, fadt::Fadt, SdtHeader}, + AcpiTables, Handle, Handler, PhysicalMapping, +}; +use spin::Mutex; +use syscall::error::{Error, EINVAL, EIO}; +use x86::{segmentation::SegmentSelector, task, Ring}; + +use crate::{ + acpi::ACPI_ROOT_INFO, + arch::interrupt, + memory::{ + round_down_pages, round_up_pages, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, + RmmArch, VirtualAddress, PAGE_SIZE, + }, + syscall::io::{Io, Pio}, +}; + +const ACPI_SLP_TYP_SHIFT: u16 = 10; +const ACPI_SLP_TYP_MASK: u16 = 0x1C00; +const ACPI_SLP_EN: u16 = 1 << 13; +const WAKE_TRAMPOLINE_PHYS: usize = 0x8000; +const SLEEP_RETURN_OK: usize = 0; + +#[cfg(target_arch = "x86_64")] +static WAKE_TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/s3_wakeup")); + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug, Default)] +struct DescriptorTableRegister { + limit: u16, + base: u64, +} + +#[repr(C, align(64))] +#[derive(Clone, Copy, Debug)] +struct FpuState { + bytes: [u8; 4096], +} + +impl Default for FpuState { + fn default() -> Self { + Self { bytes: [0; 4096] } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SleepState { + S3, + S5, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SleepError { + UnsupportedArch, + MissingAcpi, + MissingFadt, + MissingFacs, + MissingSleepObject, + InvalidSleepObject, + UnsupportedPmControl, + UnsupportedAmlOperation, + SleepDidNotEnter, +} + +impl SleepError { + fn code(self) -> usize { + match self { + Self::UnsupportedArch => EINVAL as usize, + Self::MissingAcpi + | Self::MissingFadt + | Self::MissingFacs + | Self::MissingSleepObject + | Self::UnsupportedAmlOperation => EIO as usize, + Self::InvalidSleepObject | Self::UnsupportedPmControl | Self::SleepDidNotEnter => { + EINVAL as usize + } + } + } + + fn from_code(code: usize) -> Self { + match code as i32 { + x if x == EINVAL => Self::InvalidSleepObject, + _ => Self::MissingAcpi, + } + } +} + +#[derive(Clone, Copy, Debug, Default)] +struct SavedCpuContext { + entry_rsp: usize, + runtime_rsp: usize, + facs_address: usize, + cr0: usize, + cr2: usize, + cr3: usize, + cr4: usize, + rflags: usize, + gdtr: DescriptorTableRegister, + idtr: DescriptorTableRegister, + efer: u64, + fs_base: u64, + gs_base: u64, + kernel_gs_base: u64, + fpu: FpuState, +} + +static SAVED_CONTEXT: Mutex> = Mutex::new(None); +static AML_MUTEX_IDS: AtomicU32 = AtomicU32::new(1); + +#[derive(Clone, Copy, Debug)] +struct SleepTypeData { + a: u16, + b: u16, +} + +#[derive(Clone, Copy)] +struct KernelAcpiHandler; + +impl KernelAcpiHandler { + fn map_range(physical_address: usize, size: usize) -> (*mut u8, usize) { + let map_base = round_down_pages(physical_address); + let map_offset = physical_address - map_base; + let mapped_length = round_up_pages(size + map_offset); + + // SAFETY: The ACPI interpreter only requests firmware-described physical regions. + unsafe { + let mut mapper = KernelMapper::lock_rw(); + for page_index in 0..mapped_length / PAGE_SIZE { + let (_, flush) = mapper + .map_linearly( + PhysicalAddress::new(map_base + page_index * PAGE_SIZE), + PageFlags::new(), + ) + .expect("failed to linearly map ACPI physical region"); + flush.flush(); + } + } + + let virtual_base = RmmA::phys_to_virt(PhysicalAddress::new(map_base)).data(); + ((virtual_base + map_offset) as *mut u8, mapped_length) + } +} + +impl Handler for KernelAcpiHandler { + unsafe fn map_physical_region(&self, physical_address: usize, size: usize) -> PhysicalMapping { + let (virtual_start, mapped_length) = Self::map_range(physical_address, size); + PhysicalMapping { + physical_start: physical_address, + virtual_start: NonNull::new(virtual_start.cast::()) + .expect("expected mapped ACPI virtual address to be non-null"), + region_length: size, + mapped_length, + handler: *self, + } + } + + fn unmap_physical_region(_region: &PhysicalMapping) {} + + fn read_u8(&self, address: usize) -> u8 { + // SAFETY: AML system-memory accesses are byte-addressable firmware regions. + unsafe { core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u8) } + } + + fn read_u16(&self, address: usize) -> u16 { + // SAFETY: AML system-memory accesses are word-addressable firmware regions. + unsafe { + core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u16) + } + } + + fn read_u32(&self, address: usize) -> u32 { + // SAFETY: AML system-memory accesses are dword-addressable firmware regions. + unsafe { + core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u32) + } + } + + fn read_u64(&self, address: usize) -> u64 { + // SAFETY: AML system-memory accesses are qword-addressable firmware regions. + unsafe { + core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u64) + } + } + + fn write_u8(&self, address: usize, value: u8) { + // SAFETY: AML system-memory accesses are byte-addressable firmware regions. + unsafe { + core::ptr::write_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u8, value) + } + } + + fn write_u16(&self, address: usize, value: u16) { + // SAFETY: AML system-memory accesses are word-addressable firmware regions. + unsafe { + core::ptr::write_volatile( + RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u16, + value, + ) + } + } + + fn write_u32(&self, address: usize, value: u32) { + // SAFETY: AML system-memory accesses are dword-addressable firmware regions. + unsafe { + core::ptr::write_volatile( + RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u32, + value, + ) + } + } + + fn write_u64(&self, address: usize, value: u64) { + // SAFETY: AML system-memory accesses are qword-addressable firmware regions. + unsafe { + core::ptr::write_volatile( + RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u64, + value, + ) + } + } + + fn read_io_u8(&self, port: u16) -> u8 { + Pio::::new(port).read() + } + + fn read_io_u16(&self, port: u16) -> u16 { + Pio::::new(port).read() + } + + fn read_io_u32(&self, port: u16) -> u32 { + Pio::::new(port).read() + } + + fn write_io_u8(&self, port: u16, value: u8) { + Pio::::new(port).write(value) + } + + fn write_io_u16(&self, port: u16, value: u16) { + Pio::::new(port).write(value) + } + + fn write_io_u32(&self, port: u16, value: u32) { + Pio::::new(port).write(value) + } + + fn read_pci_u8(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u8 { + 0 + } + + fn read_pci_u16(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u16 { + 0 + } + + fn read_pci_u32(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u32 { + 0 + } + + fn write_pci_u8(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u8) {} + + fn write_pci_u16(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u16) {} + + fn write_pci_u32(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u32) {} + + fn nanos_since_boot(&self) -> u64 { + 0 + } + + fn stall(&self, microseconds: u64) { + for _ in 0..(microseconds.saturating_mul(64)) { + core::hint::spin_loop(); + } + } + + fn sleep(&self, milliseconds: u64) { + for _ in 0..(milliseconds.saturating_mul(64_000)) { + core::hint::spin_loop(); + } + } + + fn create_mutex(&self) -> Handle { + Handle(AML_MUTEX_IDS.fetch_add(1, Ordering::Relaxed)) + } + + fn acquire(&self, _mutex: Handle, _timeout: u16) -> Result<(), acpi_ext::aml::AmlError> { + Ok(()) + } + + fn release(&self, _mutex: Handle) {} +} + +fn sleep_state_name(state: SleepState) -> &'static str { + match state { + SleepState::S3 => "\\_S3", + SleepState::S5 => "\\_S5", + } +} + +fn encode_sleep_type(value: u16) -> u16 { + if value <= 0x7 { + value << ACPI_SLP_TYP_SHIFT + } else { + value & ACPI_SLP_TYP_MASK + } +} + +fn load_interpreter() -> Result<( + Arc>, + PhysicalMapping, + Interpreter, +), SleepError> { + let root = *ACPI_ROOT_INFO.get().ok_or(SleepError::MissingAcpi)?; + let handler = KernelAcpiHandler; + + // SAFETY: ACPI root info is captured from the firmware-provided, already validated root table. + let tables = unsafe { + AcpiTables::from_rsdt(handler, root.revision, root.root_sdt_address.data()) + .map_err(|_| SleepError::MissingAcpi)? + }; + let fadt = tables.find_table::().ok_or(SleepError::MissingFadt)?; + let registers = Arc::new( + FixedRegisters::new(&fadt, handler).map_err(|_| SleepError::UnsupportedPmControl)?, + ); + let facs_address = fadt.facs_address().map_err(|_| SleepError::MissingFacs)?; + + // SAFETY: The FADT-supplied FACS address is used exactly as described by the ACPI spec. + let facs = unsafe { handler.map_physical_region::(facs_address, core::mem::size_of::()) }; + // SAFETY: The AML interpreter only needs an owned mapping of the same firmware FACS table. + let interpreter_facs = unsafe { + handler.map_physical_region::(facs_address, core::mem::size_of::()) + }; + let dsdt = tables.dsdt().map_err(|_| SleepError::MissingFadt)?; + let interpreter = Interpreter::new(handler, dsdt.revision, Arc::clone(®isters), Some(interpreter_facs)); + + // SAFETY: Each AML table mapping is owned by the interpreter during table loading. + unsafe { + let mapping = handler.map_physical_region::(dsdt.phys_address, dsdt.length as usize); + let stream = core::slice::from_raw_parts( + mapping.virtual_start.as_ptr().byte_add(core::mem::size_of::()) as *const u8, + dsdt.length as usize - core::mem::size_of::(), + ); + interpreter + .load_table(stream) + .map_err(|_| SleepError::UnsupportedAmlOperation)?; + + for ssdt in tables.ssdts() { + let mapping = handler.map_physical_region::(ssdt.phys_address, ssdt.length as usize); + let stream = core::slice::from_raw_parts( + mapping.virtual_start.as_ptr().byte_add(core::mem::size_of::()) as *const u8, + ssdt.length as usize - core::mem::size_of::(), + ); + interpreter + .load_table(stream) + .map_err(|_| SleepError::UnsupportedAmlOperation)?; + } + } + + Ok((registers, facs, interpreter)) +} + +fn sleep_type_data_from_interpreter( + interpreter: &Interpreter, + state: SleepState, +) -> Result { + let name = AmlName::from_str(sleep_state_name(state)).map_err(|_| SleepError::MissingSleepObject)?; + let object = interpreter + .evaluate(name, Vec::new()) + .map_err(|_| SleepError::MissingSleepObject)?; + + let Object::Package(package) = &*object else { + return Err(SleepError::InvalidSleepObject); + }; + + let Some(typa_object) = package.first() else { + return Err(SleepError::InvalidSleepObject); + }; + let Some(typb_object) = package.get(1) else { + return Err(SleepError::InvalidSleepObject); + }; + + let Object::Integer(typa) = &**typa_object else { + return Err(SleepError::InvalidSleepObject); + }; + let Object::Integer(typb) = &**typb_object else { + return Err(SleepError::InvalidSleepObject); + }; + + Ok(SleepTypeData { + a: encode_sleep_type(*typa as u16), + b: encode_sleep_type(*typb as u16), + }) +} + +fn sleep_type_data(state: SleepState) -> Result { + let (_registers, _facs, interpreter) = load_interpreter()?; + sleep_type_data_from_interpreter(&interpreter, state) +} + +fn install_wake_trampoline(stack_rsp: usize, cr3: usize) { + let trampoline_page = Page::containing_address(VirtualAddress::new(WAKE_TRAMPOLINE_PHYS)); + let trampoline_frame = PhysicalAddress::new(WAKE_TRAMPOLINE_PHYS); + + // SAFETY: The 0x8000 low-memory trampoline page is reserved by the kernel for bootstrap stubs. + let (result, _) = unsafe { + let mut mapper = KernelMapper::lock_rw(); + let result = mapper + .map_phys( + trampoline_page.start_address(), + trampoline_frame, + PageFlags::new().execute(true).write(true), + ) + .expect("failed to map S3 wake trampoline page"); + (result, mapper.table().phys().data()) + }; + result.flush(); + + for (index, value) in WAKE_TRAMPOLINE_DATA.iter().enumerate() { + // SAFETY: The trampoline page is mapped writable at the same virtual address as the physical page. + unsafe { + core::ptr::write_volatile((WAKE_TRAMPOLINE_PHYS as *mut u8).add(index), *value); + } + } + + // SAFETY: The wake trampoline layout reserves three qword fields immediately after the jump. + unsafe { + let stack_slot = (WAKE_TRAMPOLINE_PHYS + 8) as *mut u64; + let page_table_slot = stack_slot.add(1); + let code_slot = stack_slot.add(2); + stack_slot.write(stack_rsp as u64); + page_table_slot.write(cr3 as u64); + #[expect(clippy::fn_to_numeric_cast)] + code_slot.write(resume_from_s3_trampoline as usize as u64); + } + + // SAFETY: The trampoline mapping is no longer needed once the physical page has been populated. + let (_frame, _, flush) = unsafe { + KernelMapper::lock_rw() + .unmap_phys(trampoline_page.start_address()) + .expect("failed to unmap S3 wake trampoline page") + }; + flush.flush(); +} + +fn save_descriptor_tables(context: &mut SavedCpuContext) { + // SAFETY: SGDT/SIDT only read the current CPU descriptor-table registers into the provided storage. + unsafe { + core::arch::asm!("sgdt [{}]", in(reg) &mut context.gdtr, options(nostack, preserves_flags)); + core::arch::asm!("sidt [{}]", in(reg) &mut context.idtr, options(nostack, preserves_flags)); + } +} + +fn save_fpu_state(context: &mut SavedCpuContext) { + // SAFETY: The kernel owns the current CPU at suspend entry and the FXSAVE buffer is 64-byte aligned. + unsafe { + core::arch::asm!( + "fxsave64 [{}]", + in(reg) context.fpu.bytes.as_mut_ptr(), + ); + } +} + +fn restore_fpu_state(context: &SavedCpuContext) { + // SAFETY: The saved FXSAVE image belongs to the same CPU context and matches the restore instruction. + unsafe { + core::arch::asm!( + "fxrstor64 [{}]", + in(reg) context.fpu.bytes.as_ptr(), + ); + } +} + +fn save_cpu_context(entry_rsp: usize) -> SavedCpuContext { + let mut context = SavedCpuContext { + entry_rsp, + ..SavedCpuContext::default() + }; + + // SAFETY: Reading control registers and MSRs is required to reconstruct the CPU execution state on wake. + unsafe { + core::arch::asm!( + "mov {}, cr0", + out(reg) context.cr0, + options(nostack, preserves_flags) + ); + core::arch::asm!( + "mov {}, cr2", + out(reg) context.cr2, + options(nostack, preserves_flags) + ); + core::arch::asm!( + "mov {}, cr3", + out(reg) context.cr3, + options(nostack, preserves_flags) + ); + core::arch::asm!( + "mov {}, cr4", + out(reg) context.cr4, + options(nostack, preserves_flags) + ); + core::arch::asm!( + "pushfq", + "pop {}", + out(reg) context.rflags, + options(preserves_flags) + ); + core::arch::asm!("mov {}, rsp", out(reg) context.runtime_rsp, options(nostack, preserves_flags)); + + context.efer = x86::msr::rdmsr(x86::msr::IA32_EFER); + context.fs_base = x86::msr::rdmsr(x86::msr::IA32_FS_BASE); + context.gs_base = x86::msr::rdmsr(x86::msr::IA32_GS_BASE); + context.kernel_gs_base = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE); + } + + save_descriptor_tables(&mut context); + save_fpu_state(&mut context); + context +} + +fn set_firmware_waking_vector(facs: &mut PhysicalMapping, vector: usize) { + facs.firmware_waking_vector = vector as u32; + facs.x_firmware_waking_vector = vector as u64; +} + +fn write_pm1_control_block( + registers: &FixedRegisters, + sleep_type: SleepTypeData, +) -> Result<(), SleepError> { + let current_a = registers + .pm1_control_registers + .pm1a + .read() + .map_err(|_| SleepError::UnsupportedPmControl)? as u16; + let armed_a = (current_a & !(ACPI_SLP_TYP_MASK | ACPI_SLP_EN)) | sleep_type.a; + + registers + .pm1_control_registers + .pm1a + .write(u64::from(armed_a)) + .map_err(|_| SleepError::UnsupportedPmControl)?; + + if let Some(pm1b) = ®isters.pm1_control_registers.pm1b { + let current_b = pm1b.read().map_err(|_| SleepError::UnsupportedPmControl)? as u16; + let armed_b = (current_b & !(ACPI_SLP_TYP_MASK | ACPI_SLP_EN)) | sleep_type.b; + pm1b.write(u64::from(armed_b)) + .map_err(|_| SleepError::UnsupportedPmControl)?; + pm1b.write(u64::from(armed_b | ACPI_SLP_EN)) + .map_err(|_| SleepError::UnsupportedPmControl)?; + } + + // SAFETY: WBINVD is required here to flush dirty cache lines before firmware powers down the CPU package. + unsafe { + core::arch::asm!("wbinvd", options(nostack, preserves_flags)); + } + + registers + .pm1_control_registers + .pm1a + .write(u64::from(armed_a | ACPI_SLP_EN)) + .map_err(|_| SleepError::UnsupportedPmControl)?; + + Ok(()) +} + +#[unsafe(naked)] +unsafe extern "sysv64" fn enter_sleep_raw(state: usize) -> usize { + core::arch::naked_asm!( + "mov rsi, rsp", + "jmp {inner}", + inner = sym enter_sleep_raw_inner, + ); +} + +extern "C" fn enter_sleep_raw_inner(state: usize, entry_rsp: usize) -> usize { + let state = match state { + 3 => SleepState::S3, + 5 => SleepState::S5, + _ => return SleepError::InvalidSleepObject.code(), + }; + + let (registers, mut facs, interpreter) = match load_interpreter() { + Ok(tuple) => tuple, + Err(error) => return error.code(), + }; + let sleep_type = match sleep_type_data_from_interpreter(&interpreter, state) { + Ok(data) => data, + Err(error) => return error.code(), + }; + + let mut context = save_cpu_context(entry_rsp); + context.facs_address = facs.physical_start; + install_wake_trampoline(context.runtime_rsp, context.cr3); + set_firmware_waking_vector(&mut facs, WAKE_TRAMPOLINE_PHYS); + + { + let mut saved = SAVED_CONTEXT.lock(); + *saved = Some(context); + } + + // SAFETY: Suspend entry must not be interrupted while the wake vector and PM1 control block are being armed. + unsafe { + interrupt::disable(); + } + + if let Err(error) = write_pm1_control_block(registers.as_ref(), sleep_type) { + return error.code(); + } + + // SAFETY: The final CLI+HLT sequence is the architectural handoff point after asserting SLP_EN. + unsafe { + core::arch::asm!("cli; hlt", options(nostack)); + } + + SleepError::SleepDidNotEnter.code() +} + +extern "C" fn resume_from_s3_trampoline() -> ! { + let mut saved = SAVED_CONTEXT.lock(); + let context = saved.take().expect("S3 wake trampoline resumed without saved CPU context"); + drop(saved); + + // SAFETY: The saved FACS physical address was captured from the validated FADT during suspend entry. + if context.facs_address != 0 { + let mut facs = unsafe { + KernelAcpiHandler.map_physical_region::( + context.facs_address, + core::mem::size_of::(), + ) + }; + set_firmware_waking_vector(&mut facs, 0); + } + + // SAFETY: The wake trampoline already switched to the saved kernel CR3 and long mode, so the remaining restores are architectural register state only. + unsafe { + x86::msr::wrmsr(x86::msr::IA32_EFER, context.efer); + core::arch::asm!("mov cr3, {}", in(reg) context.cr3, options(nostack)); + core::arch::asm!("mov cr4, {}", in(reg) context.cr4, options(nostack)); + core::arch::asm!("mov cr2, {}", in(reg) context.cr2, options(nostack)); + core::arch::asm!("mov cr0, {}", in(reg) context.cr0, options(nostack)); + core::arch::asm!("lgdt [{}]", in(reg) &context.gdtr, options(nostack)); + core::arch::asm!("lidt [{}]", in(reg) &context.idtr, options(nostack)); + + task::load_tr(SegmentSelector::new(crate::arch::gdt::GDT_TSS as u16, Ring::Ring0)); + + x86::msr::wrmsr(x86::msr::IA32_FS_BASE, context.fs_base); + x86::msr::wrmsr(x86::msr::IA32_GS_BASE, context.gs_base); + x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, context.kernel_gs_base); + } + + restore_fpu_state(&context); + + // SAFETY: Returning with the original entry stack and RFLAGS completes the suspend call as a successful function return. + unsafe { + core::arch::asm!( + "mov rsp, {entry_rsp}", + "push {rflags}", + "popfq", + "xor eax, eax", + "ret", + entry_rsp = in(reg) context.entry_rsp, + rflags = in(reg) context.rflags, + options(noreturn) + ); + } +} + +pub fn enter_sleep_state(state: SleepState) -> core::result::Result<(), SleepError> { + #[cfg(not(target_arch = "x86_64"))] + { + let _ = state; + return Err(SleepError::UnsupportedArch); + } + + #[cfg(target_arch = "x86_64")] + { + let raw = unsafe { + enter_sleep_raw(match state { + SleepState::S3 => 3, + SleepState::S5 => 5, + }) + }; + if raw == SLEEP_RETURN_OK { + Ok(()) + } else { + Err(SleepError::from_code(raw)) + } + } +} + +pub fn available_sleep_states() -> &'static [u8] { + if sleep_type_data(SleepState::S3).is_ok() { + b"S3\nS5\n" + } else { + b"S5\n" + } +} + +pub fn trigger_sleep_request(request: &str) -> Result<(), Error> { + match request.trim() { + "S3" => enter_sleep_state(SleepState::S3).map_err(|_| Error::new(EIO)), + "S5" => enter_sleep_state(SleepState::S5).map_err(|_| Error::new(EIO)), + _ => Err(Error::new(EINVAL)), + } +} diff --git a/src/asm/x86_64/s3_wakeup.asm b/src/asm/x86_64/s3_wakeup.asm new file mode 100644 index 0000000000..7beeccf603 --- /dev/null +++ b/src/asm/x86_64/s3_wakeup.asm @@ -0,0 +1,110 @@ +; ACPI S3 wake trampoline +; compiled with nasm by build.rs, copied to physical 0x8000 before S3 entry + +ORG 0x8000 +SECTION .text +USE16 + +trampoline: + jmp short startup_wake + times 8 - ($ - trampoline) nop + .stack: dq 0 + .page_table: dq 0 + .code: dq 0 + +startup_wake: + cli + + xor ax, ax + mov ds, ax + mov es, ax + mov ss, ax + mov sp, 0 + + mov edi, [trampoline.page_table] + mov cr3, edi + + mov eax, cr0 + and al, 11110011b + or al, 00100010b + mov cr0, eax + + mov eax, cr4 + or eax, 1 << 9 | 1 << 7 | 1 << 5 | 1 << 4 + mov cr4, eax + + fninit + + lgdt [gdtr] + + mov ecx, 0xC0000080 + rdmsr + or eax, 1 << 11 | 1 << 8 + wrmsr + + mov ebx, cr0 + or ebx, 1 << 31 | 1 << 16 | 1 + mov cr0, ebx + + jmp gdt.kernel_code:long_mode_wake + +USE64 +long_mode_wake: + mov rax, gdt.kernel_data + mov ds, rax + mov es, rax + mov fs, rax + mov gs, rax + mov ss, rax + + mov rsp, [trampoline.stack] + mov rax, [trampoline.code] + jmp rax + +struc GDTEntry + .limitl resw 1 + .basel resw 1 + .basem resb 1 + .attribute resb 1 + .flags__limith resb 1 + .baseh resb 1 +endstruc + +attrib: + .present equ 1 << 7 + .user equ 1 << 4 + .code equ 1 << 3 + .writable equ 1 << 1 + +flags: + .long_mode equ 1 << 5 + +gdtr: + dw gdt.end + 1 + dq gdt + +gdt: +.null equ $ - gdt + dq 0 + +.kernel_code equ $ - gdt +istruc GDTEntry + at GDTEntry.limitl, dw 0 + at GDTEntry.basel, dw 0 + at GDTEntry.basem, db 0 + at GDTEntry.attribute, db attrib.present | attrib.user | attrib.code + at GDTEntry.flags__limith, db flags.long_mode + at GDTEntry.baseh, db 0 +iend + +.kernel_data equ $ - gdt +istruc GDTEntry + at GDTEntry.limitl, dw 0 + at GDTEntry.basel, dw 0 + at GDTEntry.basem, db 0 + at GDTEntry.attribute, db attrib.present | attrib.user | attrib.writable + at GDTEntry.flags__limith, db 0 + at GDTEntry.baseh, db 0 +iend + +.end equ $ - gdt diff --git a/src/context/context.rs b/src/context/context.rs index 4ee0e56302..8dce642377 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -435,9 +435,7 @@ impl Context { ) -> Option { let is_upper = i.get() & syscall::UPPER_FDTBL_TAG != 0; let result = self.files.write(lock_token.token()).remove_file(i); - if is_upper && result.is_some() { - crate::info!("Context::remove_file ctx={} upper_idx={}", self.name, i.get()); - } + result } @@ -969,9 +967,7 @@ impl FdTbl { let removed_file_opt = fdtbl.get_mut(real_index).and_then(|opt| opt.take()); if removed_file_opt.is_some() { self.active_count -= 1; - if index & syscall::UPPER_FDTBL_TAG != 0 { - crate::info!("FdTbl::remove_file upper idx={} active={}", real_index, self.active_count); - } + } removed_file_opt diff --git a/src/context/file.rs b/src/context/file.rs index 2d3790f147..150f483a47 100644 --- a/src/context/file.rs +++ b/src/context/file.rs @@ -4,7 +4,7 @@ use crate::{ event, scheme::{self, SchemeId}, sync::{CleanLockToken, RwLock, L6}, - syscall::error::Result, + syscall::error::{Error, Result, ESTALE}, }; use alloc::sync::Arc; use syscall::{schemev2::NewFdFlags, RwFlags, O_APPEND, O_NONBLOCK}; @@ -18,6 +18,7 @@ pub struct FileDescription { pub offset: u64, /// The scheme that this file refers to pub scheme: SchemeId, + pub scheme_generation: Option, /// The number the scheme uses to refer to this file pub number: usize, /// The flags passed to open or fcntl(SETFL) @@ -32,6 +33,52 @@ bitflags! { } } impl FileDescription { + pub fn with_generation( + scheme: SchemeId, + scheme_generation: Option, + number: usize, + offset: u64, + flags: u32, + internal_flags: InternalFlags, + ) -> Self { + Self { + offset, + scheme, + scheme_generation, + number, + flags, + internal_flags, + } + } + + pub fn new( + scheme: SchemeId, + number: usize, + offset: u64, + flags: u32, + internal_flags: InternalFlags, + token: &mut CleanLockToken, + ) -> Self { + Self::with_generation( + scheme, + Some(scheme::current_scheme_generation(token.token(), scheme)), + number, + offset, + flags, + internal_flags, + ) + } + + pub fn get_scheme(&self, token: &mut CleanLockToken) -> Result { + if let Some(expected_generation) = self.scheme_generation + && expected_generation != scheme::current_scheme_generation(token.token(), self.scheme) + { + return Err(Error::new(ESTALE)); + } + + scheme::get_scheme(token.token(), self.scheme) + } + pub fn rw_flags(&self, rw: RwFlags) -> u32 { let mut ret = self.flags & !(O_NONBLOCK | O_APPEND) as u32; if rw.contains(RwFlags::APPEND) { @@ -76,7 +123,7 @@ impl FileDescription { pub fn try_close(self, token: &mut CleanLockToken) -> Result<()> { event::unregister_file(self.scheme, self.number, token); - let scheme = scheme::get_scheme(token.token(), self.scheme)?; + let scheme = self.get_scheme(token)?; scheme.close(self.number, token) } @@ -85,12 +132,12 @@ impl FileDescription { impl FileDescriptor { pub fn close(self, token: &mut CleanLockToken) -> Result<()> { { - let (scheme_id, number, internal_flags) = { + let (desc, number, internal_flags) = { let desc = self.description.read(token.token()); - (desc.scheme, desc.number, desc.internal_flags) + (*desc, desc.number, desc.internal_flags) }; if internal_flags.contains(InternalFlags::NOTIFY_ON_NEXT_DETACH) { - let scheme = scheme::get_scheme(token.token(), scheme_id)?; + let scheme = desc.get_scheme(token)?; scheme.detach(number, token)?; } } diff --git a/src/context/memory.rs b/src/context/memory.rs index 39037db91a..a84f445e0d 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -64,14 +64,13 @@ impl UnmapResult { return Ok(()); }; - let (scheme_id, number) = { - let desc = description.write(token.token()); - (desc.scheme, desc.number) + let (scheme, number) = { + let desc = *description.read(token.token()); + (desc.get_scheme(token)?, desc.number) }; - let scheme_opt = scheme::get_scheme(token.token(), scheme_id); - let funmap_result = scheme_opt - .and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags, token)); + let funmap_result = scheme + .kfunmap(number, base_offset, self.size, self.flags, token); if let Ok(fd) = Arc::try_unwrap(description) { fd.into_inner().try_close(token)?; @@ -2706,20 +2705,13 @@ fn correct_inner<'l>( // XXX: This is cheating, but guaranteed we won't deadlock because we've dropped addr_space_guard let mut token = unsafe { CleanLockToken::new() }; - let (scheme_id, scheme_number) = { - let desc = &file_ref.description.read(token.token()); - (desc.scheme, desc.number) + let desc = *file_ref.description.read(token.token()); + let scheme = desc.get_scheme(&mut token).map_err(|_| PfError::Segv)?; + let scheme_number = desc.number; + let user_inner = match scheme { + KernelSchemes::User(user) => user.inner, + _ => return Err(PfError::Segv), }; - let user_inner = scheme::get_scheme(token.token(), scheme_id) - .ok() - .and_then(|s| { - if let KernelSchemes::User(user) = s { - Some(user.inner) - } else { - None - } - }) - .ok_or(PfError::Segv)?; let offset = file_ref.base_offset as u64 + (pages_from_grant_start * PAGE_SIZE) as u64; user_inner diff --git a/src/scheme/acpi.rs b/src/scheme/acpi.rs index 9997b7e0e6..6e773a6b2b 100644 --- a/src/scheme/acpi.rs +++ b/src/scheme/acpi.rs @@ -10,6 +10,8 @@ use crate::{ acpi::{RxsdtEnum, RXSDT_ENUM}, arch::x86_64::s3_resume, context::file::InternalFlags, + arch::sleep, + arch::sleep, scheme::{SchemeExt, StrOrBytes}, sync::CleanLockToken, }; @@ -39,7 +41,9 @@ bitflags! { static RXSDT_DATA: Once> = Once::new(); /// Phase I: kstop reason codes. Read via the CheckShutdown + SleepControl, /// AcpiVerb (kcall 2). The reason tells acpid what AML + SleepControl, /// sequence to run. /// /// | Value | Reason | acpid's response | @@ -189,8 +193,10 @@ impl AcpiScheme { } impl KernelScheme for AcpiScheme { + HandleKind::SleepControl => sleep::available_sleep_states().len() as u64, fn scheme_root(&self, _token: &mut CleanLockToken) -> Result { Ok((HandleBits::CAN_READ_RXSDT | HandleBits::CAN_REGISTER_KSTOP).bits()) + HandleKind::SleepControl => sleep::available_sleep_states().len() as u64, } fn kopenat( &self, @@ -250,9 +256,11 @@ impl KernelScheme for AcpiScheme { AcpiVerb::CheckShutdown => { if handle != HandleBits::KSTOP_HANDLE { return Err(Error::new(EINVAL)); + HandleKind::SleepControl => sleep::available_sleep_states(), } // Phase I.5: return the u8 reason, not the // pre-Phase-I.5 bool. acpid's CheckShutdown + HandleKind::SleepControl => sleep::available_sleep_states(), // verb handler is updated to switch on the // reason value. Ok(*KSTOP_FLAG.lock(token.token()) as usize) @@ -324,10 +332,20 @@ impl KernelScheme for AcpiScheme { // the FADT (ACPI 1.0 or very old hardware). } Ok(0) + HandleKind::SleepControl => Stat { + st_mode: MODE_FILE, + st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX), + ..Default::default() + }, } AcpiVerb::EnterS3 => { // Phase II.X.W: acpid has done the AML prep // (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written + HandleKind::SleepControl => Stat { + st_mode: MODE_FILE, + st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX), + ..Default::default() + }, // the trampoline address to FACS via // `SetS3WakingVector`. The acpid now requests // the kernel to enter S3. The kernel's diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index c70ac5792b..4a23b3cf4f 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -22,9 +22,10 @@ struct Handle { static HANDLES: RwLock> = RwLock::new(HandleMap::new()); -/// Add to the input queue +/// Add to the input queue, translating CR to NL (ICRNL) for serial console compatibility. pub fn debug_input(data: u8, token: &mut CleanLockToken) { - INPUT.send(data, token); + let translated = if data == b'\r' { b'\n' } else { data }; + INPUT.send(translated, token); } // Notify readers of input updates @@ -106,12 +107,16 @@ impl KernelScheme for DebugScheme { fn fevent( &self, id: usize, - _flags: EventFlags, + flags: EventFlags, token: &mut CleanLockToken, ) -> Result { let _handle = *HANDLES.read(token.token()).get(id)?; - Ok(EventFlags::empty()) + let mut ready = EventFlags::empty(); + if flags.contains(EventFlags::EVENT_READ) { + ready |= EventFlags::EVENT_READ; + } + Ok(ready) } fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 0923c47ef4..68a13a7397 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -12,7 +12,7 @@ use alloc::{ }; use core::{ str, - sync::atomic::{AtomicUsize, Ordering}, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, }; use hashbrown::hash_map::{self, DefaultHashBuilder, HashMap}; use spin::Once; @@ -167,6 +167,7 @@ enum Handle { /// Schemes list static HANDLES: Once>> = Once::new(); +static SCHEME_GENERATIONS: Once>> = Once::new(); static SCHEME_LIST_NEXT_ID: AtomicUsize = AtomicUsize::new(MAX_GLOBAL_SCHEMES); static SCHEME_LIST_ID: AtomicUsize = AtomicUsize::new(0); @@ -202,6 +203,10 @@ fn init_schemes() -> RwLock> { RwLock::new(handles) } +fn init_scheme_generations() -> RwLock> { + RwLock::new(HashMap::new()) +} + /// Get a handle to a scheme. pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result { match handles().read(token).get(&scheme_id) { @@ -210,10 +215,33 @@ pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result, scheme_id: SchemeId) -> u64 { + scheme_generations() + .read(token) + .get(&scheme_id) + .map(|generation| generation.load(Ordering::Acquire)) + .unwrap_or(0) +} + fn handles<'a>() -> &'a RwLock> { HANDLES.call_once(init_schemes) } +fn scheme_generations<'a>() -> &'a RwLock> { + SCHEME_GENERATIONS.call_once(init_scheme_generations) +} + +fn increment_scheme_generation(scheme_id: SchemeId, token: &mut CleanLockToken) { + match scheme_generations().write(token.token()).entry(scheme_id) { + hash_map::Entry::Occupied(entry) => { + entry.get().fetch_add(1, Ordering::AcqRel); + } + hash_map::Entry::Vacant(entry) => { + entry.insert(AtomicU64::new(1)); + } + } +} + /// Scheme list type pub struct SchemeList; @@ -258,9 +286,14 @@ impl SchemeList { /// Remove a scheme fn remove(&self, id: usize, token: &mut CleanLockToken) { - let scheme = handles().write(token.token()).remove(&SchemeId(id)); + let scheme_id = SchemeId(id); + let scheme = handles().write(token.token()).remove(&scheme_id); assert!(scheme.is_some()); + if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme.as_ref() { + user.inner.fail_pending_calls(token); + } + increment_scheme_generation(scheme_id, token); if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme && let Some(user) = Arc::into_inner(user.inner) { @@ -285,32 +318,32 @@ impl KernelScheme for SchemeList { token: &mut CleanLockToken, ) -> Result { let scheme_id = SchemeId(scheme_id); - match handles() - .read(token.token()) - .get(&scheme_id) - .ok_or(Error::new(EBADF))? - { - Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => { - let inner = inner.clone(); - assert!(scheme_id == inner.scheme_id); - let scheme = scheme_id; - let params = unsafe { user_buf.read_exact::()? }; - - return Ok(OpenResult::External(Arc::new(RwLock::new( - FileDescription { - scheme, - number: params.number, - offset: params.offset, - flags: params.flags as u32, - internal_flags: InternalFlags::from_extra0(params.internal_flags) - .ok_or(Error::new(EINVAL))?, - }, - )))); + let maybe_inner = { + let handles = handles().read(token.token()); + match handles.get(&scheme_id).ok_or(Error::new(EBADF))? { + Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => Some(inner.clone()), + Handle::SchemeCreationCapability => None, + _ => return Err(Error::new(EBADF)), } - Handle::SchemeCreationCapability => (), - _ => return Err(Error::new(EBADF)), }; + if let Some(inner) = maybe_inner { + assert!(scheme_id == inner.scheme_id); + let params = unsafe { user_buf.read_exact::()? }; + + return Ok(OpenResult::External(Arc::new(RwLock::new( + FileDescription::new( + scheme_id, + params.number, + params.offset, + params.flags as u32, + InternalFlags::from_extra0(params.internal_flags) + .ok_or(Error::new(EINVAL))?, + token, + ), + )))); + } + const EXPECTED: &[u8] = b"create-scheme"; let mut buf = [0u8; EXPECTED.len()]; @@ -775,6 +808,7 @@ pub enum OpenResult { SchemeLocal(usize, InternalFlags), External(Arc), } + pub groups: alloc::vec::Vec, pub struct CallerCtx { pub pid: usize, pub uid: u32, @@ -783,6 +817,7 @@ pub struct CallerCtx { } impl CallerCtx { pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self { + groups: self.groups, if self.uid == 0 && self.gid == 0 { Self { pid: self.pid, diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 189cf538ab..ff337254d6 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -783,13 +783,7 @@ impl KernelScheme for ProcScheme { } (pc, uc, ul, pl) }; - let ctx_name = context.read(token.token()).name; - info!( - "AwaitingFiletableChange applied: ctx='{}' posix_fds={} upper_fds={}", - ctx_name, posix_count, upper_count - ); - info!("upper FDs present:{}", upper_list); - info!("posix FDs present:{}", posix_list); + context.write(token.token()).files = new_ft; } _ => (), diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 5d078b6df3..306461cc7e 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -80,6 +80,7 @@ const ONE: NonZeroUsize = match NonZeroUsize::new(1) { Some(one) => one, None => unreachable!(), }; +const MAX_SPURIOUS_WAKEUPS: usize = 100; enum ParsedCqe { TriggerFevent { @@ -209,6 +210,8 @@ impl UserInner { caller_responsible: &mut PageSpan, token: &mut CleanLockToken, ) -> Result { + let mut remaining_spurious_wakeups = MAX_SPURIOUS_WAKEUPS; + { // Disable preemption to avoid context switches between setting the // process state and sending the scheme request. The process is made @@ -261,7 +264,10 @@ impl UserInner { }; let states = self.states.lock(token.token()); - let (mut states, mut token) = states.into_split(); + let (mut states, mut state_token) = states.into_split(); + let mut timed_out_descriptions = None; + let mut remove_state = false; + let mut timed_out = false; match states.get_mut(sqe.tag as usize) { // invalid state None => return Err(Error::new(EBADFD)), @@ -274,24 +280,35 @@ impl UserInner { fds, } => { let maybe_eintr = - eintr_if_sigkill(&mut callee_responsible, &mut token.token()); - *o = State::Waiting { - canceling: true, - callee_responsible, - context, - fds, - }; + eintr_if_sigkill(&mut callee_responsible, &mut state_token.token()); + + if maybe_eintr.is_ok() { + remaining_spurious_wakeups = + remaining_spurious_wakeups.saturating_sub(1); + } + + if maybe_eintr.is_ok() && remaining_spurious_wakeups == 0 { + timed_out_descriptions = Some(Self::collect_descriptions_to_close(fds)); + remove_state = true; + } else { + *o = State::Waiting { + canceling: true, + callee_responsible, + context, + fds, + }; + } maybe_eintr?; - context::current() - .write(token.token()) - .block("UserInner::call (woken up after cancelation request)"); - - // We do not want to drop the lock before blocking - // as if we get preempted in between we might miss a - // wakeup. - drop(states); + if remove_state { + states.remove(sqe.tag as usize); + timed_out = true; + } else { + context::current() + .write(state_token.token()) + .block("UserInner::call (woken up after cancelation request)"); + } } // spurious wakeup State::Waiting { @@ -300,60 +317,76 @@ impl UserInner { context, mut callee_responsible, } => { - let maybe_eintr = eintr_if_sigkill(&mut callee_responsible, &mut token); let current_context = context::current(); + let maybe_eintr = + eintr_if_sigkill(&mut callee_responsible, &mut state_token); - *o = State::Waiting { - // Currently we treat all spurious wakeups to have the same behavior - // as signals (i.e., we send a cancellation request). It is not something - // that should happen, but it certainly can happen, for example if a context - // is awoken through its thread handle without setting any sig bits, or if the - // caller clears its own sig bits. If it actually is a signal, then it is the - // intended behavior. - canceling: true, - fds, - context, - callee_responsible, - }; + if maybe_eintr.is_ok() { + remaining_spurious_wakeups = + remaining_spurious_wakeups.saturating_sub(1); + } + + if maybe_eintr.is_ok() && remaining_spurious_wakeups == 0 { + timed_out_descriptions = Some(Self::collect_descriptions_to_close(fds)); + remove_state = true; + } else { + *o = State::Waiting { + // Currently we treat all spurious wakeups to have the same behavior + // as signals (i.e., we send a cancellation request). It is not something + // that should happen, but it certainly can happen, for example if a context + // is awoken through its thread handle without setting any sig bits, or if the + // caller clears its own sig bits. If it actually is a signal, then it is the + // intended behavior. + canceling: true, + fds, + context, + callee_responsible, + }; + } maybe_eintr?; - // We do not want to preempt between sending the - // cancellation and blocking again where we might - // miss a wakeup. - let mut preempt = PreemptGuardL1::new(¤t_context, &mut token); - let token = preempt.token(); + if remove_state { + states.remove(sqe.tag as usize); + timed_out = true; + } else { + // We do not want to preempt between sending the + // cancellation and blocking again where we might + // miss a wakeup. + let mut preempt = + PreemptGuardL1::new(¤t_context, &mut state_token); + let token = preempt.token(); - self.todo.send_locked( - Sqe { - opcode: Opcode::Cancel as u8, - sqe_flags: SqeFlags::ONEWAY, - tag: sqe.tag, - ..Default::default() - }, - token.token(), - ); - event::trigger_locked( - self.root_id, - self.scheme_id.get(), - EVENT_READ, - token.token(), - ); + self.todo.send_locked( + Sqe { + opcode: Opcode::Cancel as u8, + sqe_flags: SqeFlags::ONEWAY, + tag: sqe.tag, + ..Default::default() + }, + token.token(), + ); + event::trigger_locked( + self.root_id, + self.scheme_id.get(), + EVENT_READ, + token.token(), + ); - // 1. If cancellation was requested and arrived - // before the scheme processed the request, an - // acknowledgement will be sent back after the - // cancellation is processed and we will be woken up - // again. State will be State::Responded then. - // - // 2. If cancellation was requested but the scheme - // already processed the request, we will receive - // the actual response next and woken up again. - // State will be State::Responded then. - context::current() - .write(token.token()) - .block("UserInner::call (spurious wakeup)"); - drop(states); + // 1. If cancellation was requested and arrived + // before the scheme processed the request, an + // acknowledgement will be sent back after the + // cancellation is processed and we will be woken up + // again. State will be State::Responded then. + // + // 2. If cancellation was requested but the scheme + // already processed the request, we will receive + // the actual response next and woken up again. + // State will be State::Responded then. + context::current() + .write(token.token()) + .block("UserInner::call (spurious wakeup)"); + } } // invalid state @@ -368,10 +401,70 @@ impl UserInner { } }, } + + if let Some(descriptions) = timed_out_descriptions { + drop(states); + for desc in descriptions { + let _ = desc.try_close(token); + } + } + + if timed_out { + return Err(Error::new(ETIMEDOUT)); + } } } } + fn collect_descriptions_to_close( + fds: Vec>, + ) -> Vec { + fds.into_iter() + .filter_map(|fd| Arc::try_unwrap(fd).ok()) + .map(RwLock::into_inner) + .collect() + } + + pub fn fail_pending_calls(&self, token: &mut CleanLockToken) { + let descriptions_to_close = { + let mut states_lock = self.states.lock(token.token()); + let (states, mut lock_token) = states_lock.token_split(); + let mut descriptions_to_close = Vec::new(); + let mut states_to_remove = Vec::new(); + + for (id, state) in states.iter_mut() { + match mem::replace(state, State::Placeholder) { + State::Waiting { context, fds, .. } => { + descriptions_to_close.extend(Self::collect_descriptions_to_close(fds)); + + match context.upgrade() { + Some(context) => { + *state = State::Responded(Response::Regular( + Err(Error::new(ENODEV)), + 0, + false, + )); + context.write(lock_token.token()).unblock(); + } + None => states_to_remove.push(id), + } + } + old_state => *state = old_state, + } + } + + for id in states_to_remove { + states.remove(id); + } + + descriptions_to_close + }; + + for desc in descriptions_to_close { + let _ = desc.try_close(token); + } + } + /// Map a readable structure to the scheme's userspace and return the /// pointer #[must_use = "copying back to head/tail buffers can fail"] @@ -1283,6 +1376,7 @@ impl UserInner { } pub fn into_drop(self, token: &mut CleanLockToken) { + self.fail_pending_calls(token); self.todo.condition.into_drop(token); } } diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 4596adf7fd..353fed4b80 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -140,7 +140,7 @@ extern "C" fn userspace_init() { info!("userspace_init: entered bootstrap context"); let mut token = unsafe { CleanLockToken::new() }; let bootstrap = BOOTSTRAP.get().expect("BOOTSTRAP was not set"); - info!("userspace_init: calling usermode_bootstrap"); + unsafe { crate::syscall::process::usermode_bootstrap(bootstrap, &mut token) } } @@ -154,6 +154,15 @@ static BOOTSTRAP: spin::Once = spin::Once::new(); pub(crate) static AP_READY: AtomicBool = AtomicBool::new(false); static BSP_READY: AtomicBool = AtomicBool::new(false); +#[cold] +fn halt_boot(message: &str) -> ! { + print!("{message}"); + println!("Kernel boot cannot continue. Halting."); + loop { + hint::spin_loop(); + } +} + /// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { let mut token = unsafe { CleanLockToken::new() }; @@ -185,13 +194,11 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { context.euid = 0; context.egid = 0; } - Err(err) => { - panic!("failed to spawn userspace_init: {:?}", err); - } + Err(_err) => halt_boot("FATAL: failed to spawn first userspace process userspace_init\n"), } numa::dump_info(); - info!("kmain: dump_info done, calling run_userspace"); + run_userspace(&mut token) } @@ -241,9 +248,7 @@ fn run_userspace(token: &mut CleanLockToken) -> ! { interrupt::enable_and_nop(); } SwitchResult::AllContextsIdle => { - if switch_count == 0 { - info!("run_userspace: AllContextsIdle on first switch — no runnable contexts"); - } + switch_count += 1; // Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired. interrupt::enable_and_halt(); diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index e8a85611cf..4acbff26f0 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -12,7 +12,7 @@ use crate::{ memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions}, }, memory::{Page, VirtualAddress, PAGE_SIZE}, - scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes}, + scheme::{FileHandle, KernelScheme, OpenResult, StrOrBytes}, sync::{CleanLockToken, RwLock}, syscall::{data::Stat, error::*, flag::*}, }; @@ -45,7 +45,7 @@ pub fn file_op_generic_ext( (file, desc) }; - let scheme = scheme::get_scheme(token.token(), desc.scheme)?; + let scheme = desc.get_scheme(token)?; op(&*scheme, file.description, desc, token) } @@ -165,14 +165,18 @@ pub fn openat_into( token: &mut CleanLockToken, ) -> Result { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; - let (scheme_id, number) = { + let desc = { let current_lock = context::current(); let mut current = current_lock.read(token.token()); - let (context, mut token) = current.token_split(); - let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?; - let desc = pipe.description.read(token.token()); - (desc.scheme, desc.number) + let (context, mut context_token) = current.token_split(); + let pipe = context + .get_file(fh, &mut context_token) + .ok_or(Error::new(EBADF))?; + *pipe.description.read(context_token.token()) }; + let scheme = desc.get_scheme(token)?; + let number = desc.number; + let scheme_id = desc.scheme; let caller_ctx = context::current().read(token.token()).caller_ctx(); let new_description = { let scheme = scheme::get_scheme(token.token(), scheme_id)?; @@ -560,8 +564,6 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) // Communicate fcntl with scheme if cmd != F_GETFD && cmd != F_SETFD { - let scheme = scheme::get_scheme(token.token(), scheme_id)?; - scheme.fcntl(number, cmd, arg, token)?; }; @@ -654,13 +656,11 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken let path = RedoxPath::from_absolute(&path_buf).ok_or(Error::new(EINVAL))?; let (_, reference) = path.as_parts().ok_or(Error::new(EINVAL))?; - let (number, scheme_id) = { - let desc = file.description.read(token.token()); - (desc.number, desc.scheme) + let (number, scheme) = { + let desc = *file.description.read(token.token()); + (desc.number, desc.get_scheme(token)?) }; - let scheme = scheme::get_scheme(token.token(), scheme_id)?; - // TODO: Check EXDEV. /* if scheme_id != description.scheme { @@ -866,10 +866,7 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) -> Ok(bytes_read) } pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { - if fd.get() <= 5 || fd.get() > 0x4000000000000000 { - let name = context::current().read(token.token()).name; - info!("sys_write ENTER fd={} buf_len={} ctx='{}'", fd.get(), buf.len(), name); - } + let result = (|| { let (bytes_written, desc_arc, desc) = file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| { diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 5f5209ba21..5942007e23 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -148,9 +148,7 @@ const KERNEL_METADATA_PAGE_COUNT: usize = syscall::KERNEL_METADATA_SIZE / PAGE_S }; pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockToken) { - info!("usermode_bootstrap: ENTER, page_count={}", bootstrap.page_count); assert_ne!(bootstrap.page_count, 0); - info!("usermode_bootstrap: page_count ok, getting addr_space"); { let addr_space = Arc::clone( @@ -159,7 +157,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok .addr_space() .expect("expected bootstrap context to have an address space"), ); - info!("usermode_bootstrap: got addr_space, calling mmap for {} pages", bootstrap.page_count); + let base = Page::containing_address(VirtualAddress::new(PAGE_SIZE)); let flags = MapFlags::MAP_FIXED_NOREPLACE @@ -173,7 +171,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok let _base_page = { let mut lock_token = token.token(); let mut addr_space_lock = addr_space.acquire_write(lock_token.downgrade()); - info!("usermode_bootstrap: acquired write lock, calling mmap"); + addr_space_lock .mmap( &addr_space, @@ -194,7 +192,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok ) .expect("Failed to allocate bootstrap pages") }; - info!("usermode_bootstrap: mmap done"); + // Insert kernel schemes root capabilities. let mut kernel_schemes_infos = @@ -217,7 +215,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok matches!(scheme, GlobalSchemes::Proc), token, ); - info!("inserted fd {:#x} for scheme_id {} cap_fd {}", inserted, scheme.scheme_id().get(), cap_fd); + inserted }; } @@ -300,7 +298,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok .expect("failed to copy memory to bootstrap"); let bootstrap_entry = u64::from_le_bytes(bootstrap_slice[0x1a..0x22].try_into().unwrap()); - info!("Bootstrap entry point: {:#x}, pages: {}, stack at {:#x}", bootstrap_entry, bootstrap.page_count, PAGE_SIZE + bootstrap.page_count * PAGE_SIZE + 8 * PAGE_SIZE); + assert_ne!(bootstrap_entry, 0); // Map a minimal user stack for the bootstrap process. Without this, @@ -363,7 +361,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok regs.set_instr_pointer(bootstrap_entry.try_into().unwrap()); regs.set_stack_pointer(stack_top_vaddr); } - info!("usermode_bootstrap: EXIT, entry={:#x}, sp={:#x}", bootstrap_entry, stack_top_vaddr); + } unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] { @@ -376,23 +374,26 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] } fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize { + let description = Arc::new(RwLock::new(FileDescription::new( + scheme, + number, + 0, + (O_CREAT | O_RDWR) as u32, + InternalFlags::empty(), + token, + ))); + let current_lock = context::current(); let mut current = current_lock.read(token.token()); - let (context, mut token) = current.token_split(); + let (context, mut context_token) = current.token_split(); context .add_file_min( FileDescriptor { - description: Arc::new(RwLock::new(FileDescription { - scheme, - number, - offset: 0, - flags: (O_CREAT | O_RDWR) as u32, - internal_flags: InternalFlags::empty(), - })), + description, cloexec, }, syscall::flag::UPPER_FDTBL_TAG + scheme.get(), - &mut token, + &mut context_token, ) .expect("failed to insert fd to current context") .get()