diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000..33d2be1164 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +rustflags = [ + # Kernel should preserve floating-point registers + "-Ctarget-feature=-sse,-sse2", +] diff --git a/.verify-fork-functions.exclude b/.verify-fork-functions.exclude new file mode 100644 index 0000000000..c626dfb23f --- /dev/null +++ b/.verify-fork-functions.exclude @@ -0,0 +1,32 @@ +# Kernel functions intentionally removed/replaced by Red Bear commits. +# These exist in upstream but were replaced with RB-specific implementations +# or removed as part of RB refactoring. Documented per Phase 17 verification. +# +# Format: : + +# profiling was removed/replaced by RB diagnostics refactoring +src/profiling.rs:pub lookup_dbg_id +src/profiling.rs:unsafe walk_kstack +src/profiling.rs:unsafe walk_ustack + +# context functions replaced by RB's _upper variants or refactored +src/context/context.rs:bulk_insert_files +src/context/context.rs:pub bulk_insert_files +src/context/context.rs:pub resize +src/context/memory.rs:insert_hole +src/context/memory.rs:remove_hole +src/context/memory.rs:resize_hole + +# statistics functions removed during RB diagnostics cleanup +src/context/timeout.rs:pub get_timeout_stat +src/event.rs:pub get_event_stat +src/scheme/pipe.rs:pub get_pipe_stat +src/syscall/futex.rs:pub get_futex_stat + +# event.rs — trait impls replaced by RB's event refactoring +src/event.rs:eq +src/event.rs:hash +src/event.rs:pub read_with_timeout + +# syscall instruction end marker — RB refactored syscall dispatch +src/arch/x86_64/interrupt/syscall.rs:pub __syscall_instruction_end diff --git a/Cargo.lock b/Cargo.lock index b40bfac6d1..2071b76f8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,9 +16,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "bit_field" @@ -40,15 +40,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "cc" -version = "1.2.63" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "shlex", @@ -104,11 +104,11 @@ dependencies = [ [[package]] name = "kernel" -version = "0.5.12" +version = "0.5.12+rb0.3.1" dependencies = [ "arrayvec", "bitfield", - "bitflags 2.12.1", + "bitflags 2.13.0", "cc", "fdt", "hashbrown 0.14.5", @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "object" @@ -177,9 +177,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -201,25 +201,23 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7" [[package]] name = "redox_syscall" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" +version = "0.9.0+rb0.3.1" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", ] [[package]] name = "rmm" version = "0.1.0" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", ] [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "sbi-rt" @@ -294,9 +292,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "spin" @@ -318,9 +316,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -402,20 +400,24 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", "syn", ] + +[[patch.unused]] +name = "libredox" +version = "0.1.18+rb0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 825b56a74a..918cddaacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,11 @@ [workspace] resolver = "3" +members = [".", "rmm"] [package] name = "kernel" -version = "0.5.12" +version = "0.5.12+rb0.3.1" +authors = ["Jeremy Soller ", "vasilito "] build = "build.rs" edition = "2024" @@ -19,7 +21,7 @@ fdt = { git = "https://github.com/repnop/fdt.git", rev = "2fb1409edd1877c714a0aa hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] } linked_list_allocator = "0.9.0" redox-path = "0.2.0" -redox_syscall = { version = "0.9.0", default-features = false } +redox_syscall = { path = "../syscall", default-features = false } rmm = { path = "rmm", default-features = false } slab = { version = "0.4", default-features = false } smallvec = { version = "1.15.1", default-features = false } @@ -85,7 +87,7 @@ default = [ "x86_kvm_pv", #"busy_panic", #"drop_panic", - #"syscall_debug", + #"syscall_debug" ] # Activates some limited code-overwriting optimizations, based on CPU features. @@ -94,7 +96,6 @@ self_modifying = [] acpi = [] lpss_debug = [] multi_core = ["acpi"] -numa = ["multi_core"] profiling = [] #TODO: remove when threading issues are fixed pti = [] @@ -119,3 +120,18 @@ panic = "abort" panic = "abort" #lto = true debug = "full" + +# Red Bear OS Phase J: see local/sources/base/Cargo.toml for +# the rationale. Both the kernel and the base workspace need +# the libredox override so that the libredox::error::Error +# type is the same compile-time type as syscall::Error. With +# the local libredox fork at local/sources/libredox/ using +# the local syscall fork at local/sources/syscall/, the +# libredox::error::Error (re-exported from the local syscall) +# and syscall::Error (also the local syscall) are now the +# same type, so `?` conversions in scheme-utils / daemon +# compile cleanly. +[patch.crates-io] +# Local fork dependency rule: every crate with a local fork MUST resolve through it. +libredox = { path = "../libredox" } +redox_syscall = { path = "../syscall" } diff --git a/Makefile b/Makefile index a089f8c350..733415908f 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +# Red Bear OS kernel patches applied via individual patch files .PHONY: all check SOURCE:=$(dir $(realpath $(lastword $(MAKEFILE_LIST)))) @@ -36,8 +37,7 @@ $(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell --manifest-path "$(MANIFEST)" \ --target "$(TARGET_SPEC)" \ --release \ - -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \ - -Z json-target-spec \ + -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \ --features=$(KERNEL_CARGO_FEATURES) \ -- \ -C link-arg=-T -Clink-arg="$(LD_SCRIPT)" \ @@ -63,6 +63,5 @@ check: --bin kernel \ --manifest-path "$(MANIFEST)" \ --target "$(TARGET_SPEC)" \ - -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \ - -Z json-target-spec \ + -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \ --features=$(KERNEL_CHECK_FEATURES) diff --git a/rmm/src/allocator/frame/bump.rs b/rmm/src/allocator/frame/bump.rs index c0c1ade5b0..375434bfec 100644 --- a/rmm/src/allocator/frame/bump.rs +++ b/rmm/src/allocator/frame/bump.rs @@ -1,11 +1,23 @@ -use core::marker::PhantomData; +use core::{ + marker::PhantomData, + ptr::null_mut, +}; use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress}; +struct FreeNode { + next: *mut FreeNode, + count: FrameCount, + phys: PhysicalAddress, + _marker: PhantomData A>, +} + #[derive(Debug)] pub struct BumpAllocator { orig_areas: (&'static [MemoryArea], usize), cur_areas: (&'static [MemoryArea], usize), + free_list: *mut FreeNode, + freed_frames: usize, _marker: PhantomData A>, } @@ -21,6 +33,8 @@ impl BumpAllocator { Self { orig_areas: (areas, offset), cur_areas: (areas, offset), + free_list: null_mut(), + freed_frames: 0, _marker: PhantomData, } } @@ -48,6 +62,41 @@ unsafe impl FrameAllocator for BumpAllocator { unsafe { let req_size = count.data() * A::PAGE_SIZE; + let mut prev: *mut FreeNode = null_mut(); + let mut cur = self.free_list; + while !cur.is_null() { + let cur_ref = &*cur; + let cur_count = cur_ref.count.data(); + if cur_count >= count.data() { + let phys = cur_ref.phys; + if cur_count == count.data() { + if prev.is_null() { + self.free_list = cur_ref.next; + } else { + (*prev).next = cur_ref.next; + } + self.freed_frames -= count.data(); + } else { + let new_phys = phys.add(req_size); + let new_count = FrameCount::new(cur_count - count.data()); + let new_node_ptr = A::phys_to_virt(new_phys).data() as *mut FreeNode; + (*new_node_ptr).next = cur_ref.next; + (*new_node_ptr).count = new_count; + (*new_node_ptr).phys = new_phys; + + if prev.is_null() { + self.free_list = new_node_ptr; + } else { + (*prev).next = new_node_ptr; + } + self.freed_frames -= count.data(); + } + return Some(phys); + } + prev = cur; + cur = cur_ref.next; + } + let block = loop { let area = self.cur_areas.0.first()?; let off = self.cur_areas.1; @@ -64,15 +113,25 @@ unsafe impl FrameAllocator for BumpAllocator { } } - unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) { - unimplemented!("BumpAllocator::free not implemented"); + unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) { + let node_ptr = A::phys_to_virt(address).data() as *mut FreeNode; + unsafe { + (*node_ptr).next = self.free_list; + (*node_ptr).count = count; + (*node_ptr).phys = address; + self.free_list = node_ptr; + self.freed_frames += count.data(); + } } fn usage(&self) -> FrameUsage { let total = self.orig_areas.0.iter().map(|a| a.size).sum::() - self.orig_areas.1; - let free = self.cur_areas.0.iter().map(|a| a.size).sum::() - self.cur_areas.1; + let bump_free = + self.cur_areas.0.iter().map(|a| a.size).sum::() - self.cur_areas.1; + let bump_used = total - bump_free; + let used = bump_used - (self.freed_frames * A::PAGE_SIZE); FrameUsage::new( - FrameCount::new((total - free) / A::PAGE_SIZE), + FrameCount::new(used / A::PAGE_SIZE), FrameCount::new(total / A::PAGE_SIZE), ) } diff --git a/rmm/src/arch/emulate.rs b/rmm/src/arch/emulate.rs index 4d84ca2bac..91fa24bfe7 100644 --- a/rmm/src/arch/emulate.rs +++ b/rmm/src/arch/emulate.rs @@ -285,8 +285,8 @@ impl Machine { } } - fn invalidate(&mut self, _address: VirtualAddress) { - unimplemented!("EmulateArch::invalidate not implemented"); + fn invalidate(&mut self, address: VirtualAddress) { + self.map.remove(&address); } //TODO: cleanup diff --git a/src/acpi/facs.rs b/src/acpi/facs.rs new file mode 100644 index 0000000000..9e48e500b2 --- /dev/null +++ b/src/acpi/facs.rs @@ -0,0 +1,291 @@ +//! ACPI Firmware ACPI Control Structure (FACS) parser. +//! +//! Per ACPI 6.5 §5.2.10. The FACS contains the firmware waking +//! vector that the platform firmware jumps to on S3 wake. +//! This is the memory location the S3 resume trampoline in +//! `arch/x86_shared/s3_resume.rs` must be written to. +//! +//! This is a comprehensive parser matching Linux 7.1's +//! `struct acpi_table_facs` in `include/acpi/actbl.h`: +//! +//! ```c +//! struct acpi_table_facs { +//! char signature[4]; // ASCII table signature +//! u32 length; // Length of structure, in bytes +//! u32 hardware_signature; // Hardware configuration signature +//! u32 firmware_waking_vector; // 32-bit FW waking vector +//! u32 global_lock; // Global Lock for shared hardware +//! u32 flags; // Flags +//! u64 xfirmware_waking_vector; // 64-bit FW waking vector (ACPI 2.0+) +//! u8 version; // Version of this table (ACPI 2.0+) +//! u8 reserved[3]; // Reserved +//! u32 ospm_flags; // Flags set by OSPM (ACPI 4.0+) +//! u8 reserved1[24]; // Reserved (ACPI 4.0+) +//! }; +//! ``` +//! +//! We also model the corresponding flag constants: +//! - `ACPI_GLOCK_PENDING` (1 << 0) +//! - `ACPI_GLOCK_OWNED` (1 << 1) +//! - `ACPI_FACS_S4_BIOS_PRESENT` (1 << 0) +//! - `ACPI_FACS_64BIT_WAKE` (1 << 1) +//! - `ACPI_FACS_64BIT_ENVIRONMENT` (1 << 0) +//! +//! Hardware-agnostic: the FACS layout is standardized by the ACPI +//! spec. Only the field values (specifically the waking vector) +//! vary per platform. + +use core::sync::atomic::AtomicPtr; + +use crate::acpi::sdt::Sdt; + +/// Linux 7.1 compatibility: matching `struct acpi_table_facs`. +/// +/// The struct is `repr(C, packed)` to match the wire layout +/// exactly. The kernel reads the bytes via direct pointer +/// access. +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct FacsStruct { + /// SDT header (signature, length, revision, checksum, + /// oem_id, oem_table_id, oem_revision, creator_id, + /// creator_revision). Same as other ACPI tables. + pub header: super::sdt::Sdt, + /// Hardware configuration signature. Used by firmware to + /// detect a cold boot vs a resume (the value differs + /// across boots because of system-specific information). + /// Red Bear OS doesn't currently use this — the kernel's + /// S3 magic value is in `s3_resume::S3State.saved_magic`. + pub hardware_signature: u32, + /// 32-bit firmware waking vector. Legacy field used by + /// firmware that runs in 32-bit mode after S3 wake. The + /// platform firmware jumps to this address on a wake + /// event. + pub firmware_waking_vector: u32, + /// Global Lock for shared hardware resources. Acquired by + /// the OS before reading/writing the FACS (or any other + /// ACPI table that requires synchronization with firmware). + /// The kernel currently doesn't use this — we read/write + /// the FACS without taking the global lock. Future work: + /// take the global lock in `read()` and `write()` if + /// multi-core S3 support is added. + pub global_lock: u32, + /// Flags. Bit 0 = S4BIOS support is present. Bit 1 = 64-bit + /// wake vector is supported (ACPI 4.0+). + pub flags: u32, + /// 64-bit firmware waking vector. Used by firmware that + /// runs in 64-bit mode after S3 wake. This is what the + /// kernel's S3 trampoline is written to. (ACPI 2.0+.) + pub xfirmware_waking_vector: u64, + /// FACS version. The FACS was introduced in ACPI 1.0; the + /// 64-bit wake vector was added in ACPI 2.0. (ACPI 2.0+.) + pub version: u8, + /// Reserved. Must be zero. Three bytes. + pub reserved: [u8; 3], + /// OSPM-set flags. Bit 0 = 64-bit wake environment is + /// required (ACPI 4.0+). + pub ospm_flags: u32, + /// Reserved. Must be zero. 24 bytes. (ACPI 4.0+.) + pub reserved1: [u8; 24], +} + +/// FACS flag constants (mirrors Linux 7.1's `actbl.h`). +/// +/// Used in the `flags` field. Bit 0 = S4BIOS support is +/// present. Bit 1 = 64-bit wake vector is supported. +pub mod facs_flags { + /// `ACPI_FACS_S4_BIOS_PRESENT` (bit 0). The S4BIOS_REQ + /// signal is supported. + pub const S4_BIOS_PRESENT: u32 = 1 << 0; + /// `ACPI_FACS_64BIT_WAKE` (bit 1). The 64-bit wake vector + /// is supported (i.e., `xfirmware_waking_vector` is valid). + pub const WAKE_64BIT: u32 = 1 << 1; +} + +/// FACS OSPM flag constants (mirrors Linux 7.1's `actbl.h`). +/// +/// Used in the `ospm_flags` field. Bit 0 = 64-bit wake +/// environment is required. +pub mod facs_ospm_flags { + /// `ACPI_FACS_64BIT_ENVIRONMENT` (bit 0). The firmware + /// uses the 64-bit waking vector on S3 wake. + pub const WAKE_64BIT_ENVIRONMENT: u32 = 1 << 0; +} + +/// FACS Global Lock flag constants (mirrors Linux 7.1's +/// `actbl.h`). Used in the `global_lock` field. +pub mod facs_glock_flags { + /// `ACPI_GLOCK_PENDING` (bit 0). The global lock is + /// pending (firmware requested it). + pub const PENDING: u32 = 1 << 0; + /// `ACPI_GLOCK_OWNED` (bit 1). The global lock is + /// currently owned. + pub const OWNED: u32 = 1 << 1; +} + +/// FACS instance pointer. Set by `init` when the FACS is +/// discovered during ACPI table parsing. Used by the +/// `SetS3WakingVector` AcPiVerb and the `firmware_waking_vector_*` +/// functions below. +static FACS_PTR: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +/// Phase II.X.W: Initialize the FACS parser. Called from the +/// kernel's acpi init after all SDTs are loaded. Reads the +/// FACS from the SDT table and stores the pointer for later +/// use by the `SetS3WakingVector` AcPiVerb. +/// +/// # Safety +/// `sdt` must be a valid pointer to a validated SDT whose +/// signature is "FACS" and whose length is at least 64 +/// bytes (the minimum size of the FACS structure for ACPI +/// 4.0+ with `ospm_flags` and `reserved1`). +pub fn init(sdt: &Sdt) { + if &sdt.signature != b"FACS" { + return; + } + // The minimum FACS size depends on the version: + // - ACPI 1.0: 32 bytes (just the header + hardware_signature + // + firmware_waking_vector + global_lock) + // - ACPI 2.0: 40 bytes (adds flags + xfirmware_waking_vector) + // - ACPI 4.0: 64 bytes (adds version + reserved + ospm_flags + // + reserved1) + // We require 64 bytes to support all fields. + if sdt.length() < 64 { + return; + } + FACS_PTR.store( + sdt.data_address() as *mut FacsStruct, + core::sync::atomic::Ordering::Release, + ); +} + +/// Phase II.X.W: Get the FACS structure. Returns `None` if +/// the FACS is not available. +pub fn get() -> Option<&'static FacsStruct> { + let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire); + if ptr.is_null() { + None + } else { + // SAFETY: FACS_PTR was set by `init` after verifying + // sdt.length() >= 64. The pointer is to firmware memory + // which doesn't change. + Some(unsafe { &*ptr }) + } +} + +/// Phase II.X.W: Read the 32-bit firmware waking vector. +/// Returns 0 if the FACS is not available. +pub fn firmware_waking_vector() -> u32 { + self::get() + .map(|f| f.firmware_waking_vector) + .unwrap_or(0) +} + +/// Phase II.X.W: Read the 64-bit firmware waking vector. +/// Returns 0 if the FACS is not available. +pub fn x_firmware_waking_vector() -> u64 { + self::get() + .map(|f| f.xfirmware_waking_vector) + .unwrap_or(0) +} + +/// Phase II.X.W: Write the 32-bit firmware waking vector. +/// The kernel's S3 trampoline is written here so the legacy +/// 32-bit firmware wake path works. +/// +/// Returns true on success, false if the FACS is not +/// available. +pub fn set_firmware_waking_vector_32(addr: u32) -> bool { + let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire); + if ptr.is_null() { + return false; + } + // SAFETY: FACS_PTR was set by `init` after verifying + // sdt.length() >= 64. The `firmware_waking_vector` field + // is at offset 36 (after the 36-byte SDT header). The + // memory is page-aligned (firmware memory) and writable. + unsafe { + let waking_vector_ptr = (ptr as *mut u8).add(36) as *mut u32; + core::ptr::write_unaligned(waking_vector_ptr, addr); + } + true +} + +/// Phase II.X.W: Write the 64-bit firmware waking vector. +/// The kernel's S3 trampoline is written here so the modern +/// 64-bit firmware wake path works. +/// +/// Returns true on success, false if the FACS is not +/// available. +pub fn set_x_firmware_waking_vector_64(addr: u64) -> bool { + let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire); + if ptr.is_null() { + return false; + } + // SAFETY: FACS_PTR was set by `init` after verifying + // sdt.length() >= 64. The `xfirmware_waking_vector` field + // is at offset 40. The memory is page-aligned and + // writable. + unsafe { + let x_waking_vector_ptr = (ptr as *mut u8).add(40) as *mut u64; + core::ptr::write_unaligned(x_waking_vector_ptr, addr); + } + true +} + +/// Phase II.X.W: Write the 64-bit firmware waking vector +/// (preferred over the 32-bit version on 64-bit systems). +/// Equivalent to `set_x_firmware_waking_vector_64`. +pub fn set_waking_vector(addr: u64) -> bool { + set_x_firmware_waking_vector_64(addr) +} + +/// Phase II.X.W: Read the FACS version. Returns 0 if the +/// FACS is not available. +pub fn version() -> u8 { + self::get().map(|f| f.version).unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS hardware signature. Returns 0 +/// if the FACS is not available. +pub fn hardware_signature() -> u32 { + self::get() + .map(|f| f.hardware_signature) + .unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS flags. Returns 0 if the FACS +/// is not available. +/// +/// See `facs_flags::*` for the bit definitions. +pub fn flags() -> u32 { + self::get().map(|f| f.flags).unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS OSPM flags. Returns 0 if the +/// FACS is not available. +/// +/// See `facs_ospm_flags::*` for the bit definitions. +pub fn ospm_flags() -> u32 { + self::get().map(|f| f.ospm_flags).unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS global lock. Returns 0 if the +/// FACS is not available. +/// +/// See `facs_glock_flags::*` for the bit definitions. +pub fn global_lock() -> u32 { + self::get().map(|f| f.global_lock).unwrap_or(0) +} + +/// Phase II.X.W: Read the reserved bytes. Returns `None` if +/// the FACS is not available. +pub fn reserved() -> Option<[u8; 3]> { + self::get().map(|f| f.reserved) +} + +/// Phase II.X.W: Read the ACPI 4.0+ reserved bytes. Returns +/// `None` if the FACS is not available. +pub fn reserved1() -> Option<[u8; 24]> { + self::get().map(|f| f.reserved1) +} diff --git a/src/acpi/fadt.rs b/src/acpi/fadt.rs new file mode 100644 index 0000000000..4c55b500bf --- /dev/null +++ b/src/acpi/fadt.rs @@ -0,0 +1,129 @@ +//! ACPI Fixed ACPI Description Table (FADT) parser. +//! +//! Per ACPI 6.5 §5.2.9. The FADT contains the hardware register +//! addresses used by the kernel for ACPI sleep state entry (S3/S5) +//! and the SCI interrupt. This module parses the fields the +//! kernel needs (PM1a_CNT, PM1a_STS for the sleep entry path, +//! and x_firmware_ctrl / firmware_ctrl for the FACS address). +//! +//! Hardware-agnostic: the FADT layout is standardized by the ACPI +//! spec; only the field values vary per platform. + +use core::sync::atomic::{AtomicU16, AtomicU32, AtomicU64}; + +use crate::acpi::sdt::Sdt; + +/// Phase II: PM1a_CNT port. Read from the FADT at boot, written +/// by `enter_s3()` to enter S3 (SLP_TYP|SLP_EN bits). Also used +/// by S5 entry (set_global_s_state in acpid). +pub static PM1A_CONTROL_PORT: AtomicU16 = AtomicU16::new(0); + +/// Phase II: PM1a_STS port. Used by `enter_s3()` to clear +/// WAK_STS (bit 15) before writing SLP_TYP|SLP_EN. +pub static PM1A_STATUS_PORT: AtomicU16 = AtomicU16::new(0); + +/// Phase II.X.W: 32-bit FACS address (FADT offset 36, +/// `firmware_ctrl` field). Used as a fallback when +/// `x_firmware_ctrl` (offset 140, ACPI 2.0+) is not present +/// (i.e., for ACPI 1.0 systems). +pub static FIRMWARE_CTRL: AtomicU32 = AtomicU32::new(0); + +/// Phase II.X.W: 64-bit FACS address (FADT offset 140, +/// `x_firmware_ctrl` field, ACPI 2.0+). The kernel's FACS +/// parser uses this to find the FACS for writing the +/// `xfirmware_waking_vector` on S3 entry. +pub static X_FIRMWARE_CTRL: AtomicU64 = AtomicU64::new(0); + +/// FADT signature bytes ("FACP"). +const FADT_SIGNATURE: [u8; 4] = *b"FACP"; + +/// FADT fixed offsets for the fields we read. These match +/// the ACPI 6.5 §5.2.9 Table 5.6 layout. +mod offsets { + /// PM1a_STS (PM1 Status Register) Block 0 Address. + /// 32-bit General-Purpose Event Register Block 0 Address. + pub const PM1A_STS: usize = 48; + /// PM1a_CNT (PM1 Control Register) Block 0 Address. + /// 32-bit General-Purpose Event Register Block 0 Address. + pub const PM1A_CNT: usize = 56; + /// `firmware_ctrl`: 32-bit Firmware ACPI Control + /// Structure address. ACPI 1.0+. + pub const FIRMWARE_CTRL_32: usize = 36; + /// `x_firmware_ctrl`: 64-bit Firmware ACPI Control + /// Structure address. ACPI 2.0+. + pub const X_FIRMWARE_CTRL_64: usize = 140; + /// FADT minimum size for ACPI 2.0+ (i.e., enough to + /// include `x_firmware_ctrl` at offset 140). + pub const FADT_MIN_SIZE_ACPI_2_0: usize = 148; + /// FADT minimum size for ACPI 1.0. + pub const FADT_MIN_SIZE_ACPI_1_0: usize = 76; +} + +/// Parse the FADT from the given SDT bytes and extract the +/// PM1a_CNT, PM1a_STS, and FACS-address fields. Called once at +/// boot after the ACPI table discovery finds the FADT. +/// +/// The FADT layout is variable (different sizes for ACPI 1.0 vs +/// 6.5+). We only need the first ~148 bytes which contain the +/// fixed-register addresses. Reference: ACPI 6.5 §5.2.9. +pub fn init(sdt: &Sdt) { + if &sdt.signature != &FADT_SIGNATURE { + return; + } + // The FADT fields are at fixed offsets from the start of the + // SDT (including the 36-byte header), not from the data area. + let base = sdt as *const _ as *const u8; + unsafe { + // PM1a_CNT is at offset 56 in the FADT (ACPI 6.5 §5.2.9 + // Table 5.6). 32-bit General-Purpose Event Register Block 0 + // Address. + let pm1a_cnt = core::ptr::read_unaligned(base.add(offsets::PM1A_CNT) as *const u32); + // PM1a_STS is at offset 48 in the FADT. + let pm1a_sts = core::ptr::read_unaligned(base.add(offsets::PM1A_STS) as *const u32); + // Convert u32 to u16 (port numbers are 16-bit). The low + // 16 bits are the IO port; the high 16 bits are the + // address-space ID which we ignore (always IO on x86). + PM1A_CONTROL_PORT.store( + (pm1a_cnt & 0xFFFF) as u16, + core::sync::atomic::Ordering::Release, + ); + PM1A_STATUS_PORT.store( + (pm1a_sts & 0xFFFF) as u16, + core::sync::atomic::Ordering::Release, + ); + + // Phase II.X.W: 32-bit FACS address (FADT offset 36, + // `firmware_ctrl` field). ACPI 1.0+. + let firmware_ctrl = core::ptr::read_unaligned( + base.add(offsets::FIRMWARE_CTRL_32) as *const u32, + ); + FIRMWARE_CTRL.store(firmware_ctrl, core::sync::atomic::Ordering::Release); + + // Phase II.X.W: 64-bit FACS address (FADT offset 140, + // `x_firmware_ctrl` field, ACPI 2.0+). We require the + // FADT to be at least 148 bytes to have this field + // (the field is at offset 140, which is 8 bytes for the + // u64, so the minimum FADT size is 148 bytes). + if sdt.length() >= offsets::FADT_MIN_SIZE_ACPI_2_0 as u32 { + let x_firmware_ctrl = core::ptr::read_unaligned( + base.add(offsets::X_FIRMWARE_CTRL_64) as *const u64, + ); + X_FIRMWARE_CTRL.store(x_firmware_ctrl, core::sync::atomic::Ordering::Release); + } + } +} + +/// Phase II.X.W: 32-bit FACS address (FADT offset 36, +/// `firmware_ctrl` field). Returns 0 if the FADT has not +/// been initialized. +pub fn firmware_ctrl() -> u32 { + FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire) +} + +/// Phase II.X.W: 64-bit FACS address (FADT offset 140, +/// `x_firmware_ctrl` field). Returns 0 if the FADT has not +/// been initialized or the FADT is too short to have the +/// field. +pub fn x_firmware_ctrl() -> u64 { + X_FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire) +} diff --git a/src/acpi/madt/arch/x86.rs b/src/acpi/madt/arch/x86.rs index cfd88a7ed5..4dc2388398 100644 --- a/src/acpi/madt/arch/x86.rs +++ b/src/acpi/madt/arch/x86.rs @@ -105,7 +105,7 @@ pub(super) fn init(madt: Madt) { ap_args_ptr.write(&args as *const _ as u64); ap_page_table.write(page_table_physaddr as u64); #[expect(clippy::fn_to_numeric_cast)] - ap_code.write(kstart_ap as *const () as u64); + ap_code.write(kstart_ap as u64); // TODO: Is this necessary (this fence)? core::arch::asm!(""); diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index fd1e88b948..83900eb226 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -17,6 +17,8 @@ use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sd #[cfg(target_arch = "aarch64")] mod gtdt; +pub mod fadt; +pub mod facs; pub mod hpet; pub mod madt; mod rsdp; diff --git a/src/acpi/sdt.rs b/src/acpi/sdt.rs index 83ff67dac1..8d40c10df6 100644 --- a/src/acpi/sdt.rs +++ b/src/acpi/sdt.rs @@ -18,6 +18,21 @@ impl Sdt { self as *const _ as usize + size_of::() } + /// Get the total length of the table (including the SDT + /// header), in bytes. The SDT is `#[repr(C, packed)]` so + /// direct field access requires an unaligned read. + pub fn length(&self) -> u32 { + // SAFETY: The Sdt is `#[repr(C, packed)]` and the + // `length` field is at offset 4 (after the 4-byte + // signature), aligned to a 4-byte boundary. The address + // is a valid pointer to the SDT; reading 4 bytes from + // offset 4 is safe. + unsafe { + let p = self as *const Self as *const u8; + core::ptr::read_unaligned(p.add(4) as *const u32) + } + } + /// Get the length of this tables data pub fn data_len(&self) -> usize { let total_size = self.length as usize; diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index 3fdc9ab3f8..65e3fe339b 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -91,23 +91,15 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { dtb::serial::init_early(dtb); } - info!("Redox OS starting..."); + info!("RedBear OS starting..."); args.print(); // Initialize RMM - let bump_allocator = crate::startup::memory::init(&args, None, None); + crate::startup::memory::init(&args, None, None); // Initialize paging paging::init(); - #[cfg(feature = "acpi")] - { - use crate::acpi; - acpi::init_before_mem(args.acpi_rsdp()); - } - - crate::memory::init_mm(bump_allocator); - crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0)); // Setup kernel heap @@ -128,7 +120,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { #[cfg(feature = "acpi")] { - crate::acpi::init_after_mem(args.acpi_rsdp()); + crate::acpi::init(args.acpi_rsdp()); } } } diff --git a/src/arch/riscv64/start.rs b/src/arch/riscv64/start.rs index 2551968f05..a825536aa9 100644 --- a/src/arch/riscv64/start.rs +++ b/src/arch/riscv64/start.rs @@ -97,7 +97,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { init_early(dtb); } - info!("Redox OS starting..."); + info!("RedBear OS starting..."); args.print(); if let Some(dtb) = &dtb { diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 7dff91ef7c..d70e13534f 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -27,7 +27,7 @@ pub unsafe fn init() { msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32); #[expect(clippy::fn_to_numeric_cast)] - msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as *const () as u64); + msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64); // DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK, // userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and @@ -181,8 +181,6 @@ pub unsafe extern "C" fn syscall_instruction() { xor rcx, rcx xor r11, r11 iretq - .globl __syscall_instruction_end -__syscall_instruction_end: ", sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)), @@ -194,5 +192,4 @@ __syscall_instruction_end: unsafe extern "C" { // TODO: macro? pub fn enter_usermode(); - pub fn __syscall_instruction_end(); } diff --git a/src/arch/x86_shared/cpuid.rs b/src/arch/x86_shared/cpuid.rs index 66315a6ea7..b36831252c 100644 --- a/src/arch/x86_shared/cpuid.rs +++ b/src/arch/x86_shared/cpuid.rs @@ -4,9 +4,9 @@ pub fn cpuid() -> CpuId { // FIXME check for cpuid availability during early boot and error out if it doesn't exist. CpuId::with_cpuid_fn(|a, c| { #[cfg(target_arch = "x86")] - let result = core::arch::x86::__cpuid_count(a, c); + let result = unsafe { core::arch::x86::__cpuid_count(a, c) }; #[cfg(target_arch = "x86_64")] - let result = core::arch::x86_64::__cpuid_count(a, c); + let result = unsafe { core::arch::x86_64::__cpuid_count(a, c) }; CpuIdResult { eax: result.eax, ebx: result.ebx, diff --git a/src/arch/x86_shared/device/mod.rs b/src/arch/x86_shared/device/mod.rs index 6f41770601..7a2e25df3c 100644 --- a/src/arch/x86_shared/device/mod.rs +++ b/src/arch/x86_shared/device/mod.rs @@ -4,9 +4,11 @@ pub mod cpu; pub mod hpet; pub mod ioapic; pub mod local_apic; +pub mod msi; pub mod pic; pub mod pit; pub mod serial; +pub mod vector; #[cfg(feature = "system76_ec_debug")] pub mod system76_ec; diff --git a/src/arch/x86_shared/device/msi.rs b/src/arch/x86_shared/device/msi.rs new file mode 100644 index 0000000000..d979fe54b6 --- /dev/null +++ b/src/arch/x86_shared/device/msi.rs @@ -0,0 +1,66 @@ +use crate::arch::device::local_apic::ApicId; + +pub const MSI_ADDRESS_BASE: u64 = 0xFEE0_0000; + +#[derive(Debug, Clone)] +pub struct MsiMessage { + pub address: u64, + pub data: u32, +} + +impl MsiMessage { + pub fn compose(dest: ApicId, vector: u8, delivery_mode: u8) -> Self { + let address = MSI_ADDRESS_BASE | (u64::from(dest.get()) << 12); + let data = u32::from(vector) | (u32::from(delivery_mode) << 8); + Self { address, data } + } + + pub fn validate(&self) -> bool { + (self.address & 0xFFF0_0000) == MSI_ADDRESS_BASE + && self.data & 0xFF >= 32 + && self.data & 0xFF < 255 + } +} + +pub fn is_valid_msi_address(addr: u64) -> bool { (addr & 0xFFF0_0000) == MSI_ADDRESS_BASE } +pub fn is_valid_msi_vector(vector: u8) -> bool { vector >= 32 && vector < 255 } + +#[derive(Debug)] +pub struct MsiCapability { + pub msg_ctl: u16, pub msg_addr_lo: u32, pub msg_data: u16, + pub is_64bit: bool, pub is_maskable: bool, pub multiple_message_capable: u8, +} + +impl MsiCapability { + pub fn parse(raw: &[u32], msg_ctl: u16) -> Option { + let msg_addr_lo = *raw.get(1)?; + let msg_data = if msg_ctl & (1<<7) != 0 { + (*raw.get(3)? & 0xFFFF) as u16 + } else { + (*raw.get(2)? & 0xFFFF) as u16 + }; + Some(Self { + msg_ctl, msg_addr_lo, msg_data, + is_64bit: msg_ctl & (1<<7) != 0, + is_maskable: msg_ctl & (1<<8) != 0, + multiple_message_capable: ((msg_ctl>>1)&0x7) as u8, + }) + } +} + +pub struct MsixCapability { + pub table_offset: u32, pub table_bar: u8, + pub pba_offset: u32, pub pba_bar: u8, pub table_size: u16, +} + +impl MsixCapability { + pub fn parse(raw: &[u32], msg_ctl: u16) -> Option { + let r1 = *raw.get(1)?; + let r2 = *raw.get(2)?; + Some(Self { + table_offset: r1 & !0x7, table_bar: (r1&0x7) as u8, + pba_offset: r2 & !0x7, pba_bar: (r2&0x7) as u8, + table_size: ((msg_ctl>>1)&0x7FF) as u16 + 1, + }) + } +} diff --git a/src/arch/x86_shared/device/tsc.rs b/src/arch/x86_shared/device/tsc.rs index 2163d615dc..bfcf7a68ae 100644 --- a/src/arch/x86_shared/device/tsc.rs +++ b/src/arch/x86_shared/device/tsc.rs @@ -99,7 +99,7 @@ pub fn get_kvm_support() -> &'static Option { static KVM_SUPPORT: Once> = Once::new(); KVM_SUPPORT.call_once(|| { - let res = __cpuid(0x4000_0000); + let res = unsafe { __cpuid(0x4000_0000) }; if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] { return None; } @@ -107,7 +107,7 @@ pub fn get_kvm_support() -> &'static Option { if max_leaf < 0x4000_0001 { return None; } - let res = __cpuid(0x4000_0001); + let res = unsafe { __cpuid(0x4000_0001) }; let supp_feats = KvmFeatureBits::from_bits_retain(res.eax); diff --git a/src/arch/x86_shared/device/vector.rs b/src/arch/x86_shared/device/vector.rs new file mode 100644 index 0000000000..cd59ac7965 --- /dev/null +++ b/src/arch/x86_shared/device/vector.rs @@ -0,0 +1,53 @@ +use crate::cpu_set::LogicalCpuId; + +const VECTOR_COUNT: usize = 224; + +static VECTORS: [core::sync::atomic::AtomicU32; 7] = [ + core::sync::atomic::AtomicU32::new(0), + core::sync::atomic::AtomicU32::new(0), + core::sync::atomic::AtomicU32::new(0), + core::sync::atomic::AtomicU32::new(0), + core::sync::atomic::AtomicU32::new(0), + core::sync::atomic::AtomicU32::new(0), + core::sync::atomic::AtomicU32::new(0), +]; + +pub fn allocate_vector(_cpu: LogicalCpuId) -> Option { + for (bank, slot) in VECTORS.iter().enumerate() { + let mut bits = slot.load(core::sync::atomic::Ordering::Acquire); + loop { + let free = bits.trailing_ones() as usize; + if free >= 32 { + break; + } + let bit = 1u32 << free; + match slot.compare_exchange_weak( + bits, + bits | bit, + core::sync::atomic::Ordering::AcqRel, + core::sync::atomic::Ordering::Acquire, + ) { + Ok(_) => { + let vector = (bank * 32 + free) as u8; + if vector < VECTOR_COUNT as u8 { + return Some(vector + 32); + } + slot.fetch_and(!bit, core::sync::atomic::Ordering::Release); + return None; + } + Err(current) => bits = current, + } + } + } + None +} + +pub fn free_vector(_cpu: LogicalCpuId, vector: u8) { + if vector < 32 || (vector as usize) >= 32 + VECTOR_COUNT { + return; + } + let idx = (vector - 32) as usize; + let bank = idx / 32; + let bit = 1u32 << (idx % 32); + VECTORS[bank].fetch_and(!bit, core::sync::atomic::Ordering::Release); +} diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 378d3330b6..a4d9eb6a9d 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -196,15 +196,14 @@ fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) { current_idt[18].set_ist(BACKUP_IST); assert_eq!( - __generic_interrupts_end as *const () as usize - - __generic_interrupts_start as *const () as usize, + __generic_interrupts_end as usize - __generic_interrupts_start as usize, 224 * 8 ); for i in 0..224 { current_idt[i + 32].set_func(unsafe { mem::transmute::( - __generic_interrupts_start as *const () as usize + i * 8, + __generic_interrupts_start as usize + i * 8, ) }); } @@ -248,10 +247,12 @@ fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) { current_idt[IpiKind::Switch as usize].set_func(ipi::switch); current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb); current_idt[IpiKind::Pit as usize].set_func(ipi::pit); + current_idt[IpiKind::Msr as usize].set_func(ipi::msr); idt.set_reserved_mut(IpiKind::Wakeup as u8, true); idt.set_reserved_mut(IpiKind::Switch as u8, true); idt.set_reserved_mut(IpiKind::Tlb as u8, true); idt.set_reserved_mut(IpiKind::Pit as u8, true); + idt.set_reserved_mut(IpiKind::Msr as u8, true); #[cfg(target_arch = "x86")] { diff --git a/src/arch/x86_shared/interrupt/ipi.rs b/src/arch/x86_shared/interrupt/ipi.rs index 20e9d436a1..9e4c129e97 100644 --- a/src/arch/x86_shared/interrupt/ipi.rs +++ b/src/arch/x86_shared/interrupt/ipi.rs @@ -1,6 +1,7 @@ use crate::{ arch::device::local_apic::the_local_apic, context, percpu::PercpuBlock, sync::CleanLockToken, }; +use x86::msr::{rdmsr, wrmsr}; interrupt!(wakeup, || { unsafe { the_local_apic().eoi() }; @@ -26,3 +27,20 @@ interrupt!(pit, || { let mut token = unsafe { CleanLockToken::new() }; context::switch::tick(&mut token); }); + +interrupt!(msr, || { + let percpu = PercpuBlock::current(); + let mailbox = percpu.msr_mailbox(); + let op = mailbox.request(); + + let result = if op.is_write { + unsafe { wrmsr(op.msr as u32, op.value) }; + op.value + } else { + unsafe { rdmsr(op.msr as u32) } + }; + + mailbox.finish_handle(result); + + unsafe { the_local_apic().eoi() }; +}); diff --git a/src/arch/x86_shared/interrupt/mod.rs b/src/arch/x86_shared/interrupt/mod.rs index 172bad3ba9..ad1ba0367f 100644 --- a/src/arch/x86_shared/interrupt/mod.rs +++ b/src/arch/x86_shared/interrupt/mod.rs @@ -42,3 +42,117 @@ pub unsafe fn halt() { core::arch::asm!("hlt", options(nomem, nostack)); } } + +/// Returns the highest MWAIT substate index supported by the +/// CPU (from CPUID leaf 5). Returns 0 if MWAIT is unsupported. +/// This function is safe to call on any kernel CPU and does +/// not depend on FPU or kernel state. The leaf-5 information is +/// cached at boot by `arch::cpuid::cpuid()` in cpuid.rs; we read +/// it from the cache here. +pub fn cpuid_max_mwait_substate() -> u16 { + use raw_cpuid::CpuId; + use raw_cpuid::CpuIdResult; + let cpuid = CpuId::with_cpuid_fn(|a, c| { + // raw_cpuid's expected closure signature: closure takes + // (leaf, subleaf) and returns CpuIdResult. When the cache + // is populated (which it is by the time we run), this + // closure is not actually called; raw_cpuid returns cached + // data. We provide a no-op fallback anyway. + CpuIdResult { + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + } + }); + if let Some(info) = cpuid.get_monitor_mwait_info() { + let c0 = info.supported_c0_states() as u16; + let c1 = info.supported_c1_states() as u16; + let c2 = info.supported_c2_states() as u16; + let c3 = info.supported_c3_states() as u16; + let c4 = info.supported_c4_states() as u16; + let c5 = info.supported_c5_states() as u16; + let c6 = info.supported_c6_states() as u16; + // C0 sub-state 0 is the "do nothing" base. Each additional + // sub-state is a deeper sleep level. The deepest substate + // index is c0+c1+c2+c3+c4+c5+c6-1 (i.e. 0-based indexing + // into the deepest MWAIT substate). + c0.saturating_add(c1).saturating_add(c2).saturating_add(c3) + .saturating_add(c4).saturating_add(c5).saturating_add(c6) + .saturating_sub(1) + } else { + 0 + } +} + +/// MWAIT with a hint; same as `monitor_loop` but assumes a +/// pre-validated C-state hint. EAX bits [7:0] encode the C-state +/// (0=C0, 1=C1, 2=C2, ...). ECX=0 breaks on any interrupt. +/// +/// `eax` encodes the MWAIT extension hint. On Arrow Lake-H: +/// 0x20 = sub-state hint, 0x40 = break on-interrupt-only. +/// +/// Safe to call after `enable_and_halt` would have been called +/// (i.e. with interrupts enabled). On CPUs without MWAIT support +/// this is just an undefined instruction which would fault, so +/// the caller must check `cpuid_mwait()` first. +#[inline(always)] +pub unsafe fn mwait_loop(eax_hint: u32, ecx_hint: u32) { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + core::arch::asm!( + "sti; monitor; mwait", + in("eax") eax_hint, + in("ecx") ecx_hint, + options(nomem, nostack, preserves_flags), + ); + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + let _ = (eax_hint, ecx_hint); + + // MWAIT returns when an interrupt fires (ecx=0 means + // "break on any interrupt"). The interrupt dispatcher + // has already run and returned by the time we get here. + // If the CPU was in s2idle (S2IDLE_REQUESTED was set by + // the kstop handler), we now know an SCI or other wake + // event has occurred. Clear the s2idle flag and trigger + // the kstop handle's EVENT_READ so acpid's main loop + // wakes up and runs the _WAK AML sequence. + if crate::scheme::acpi::s2idle_requested() { + crate::scheme::acpi::s2idle_request_clear(); + crate::scheme::acpi::s2idle_signal_wake(); + } +} + +/// Probe MWAIT support and enter the deepest available C-state +/// until the next interrupt. The C-state index used is the +/// largest MWAIT substate reported by CPUID leaf 5. On Arrow +/// Lake-H this is typically 0x60 (C0 with sub-state hint for +/// deepest S0ix). On older CPUs without MWAIT, falls back to +/// `enable_and_halt`. +/// +/// `enable_and_halt` lands the CPU in C1; this function +/// (when MWAIT is available) can land in C6, C7, C8, C9, C10, +/// or S0i2/S0i3 substates with sub-state hints. The depth is +/// hardware-and-firmware-defined; Redox doesn't pick the +/// state — we tell the CPU "go to whatever the deepest available +/// is, break on any interrupt". +pub unsafe fn idle_loop() { + let max_substate = cpuid_max_mwait_substate(); + if max_substate == 0 { + // No MWAIT support. Land in C1 via hlt. This matches the + // pre-MWAIT behavior of `enable_and_halt` and is safe on + // every x86 CPU since the original Pentium. + enable_and_halt(); + } else { + // MWAIT supported. Enter the deepest substate, break on any + // interrupt (ecx=0). + // + // The hint we pass in EAX is 0x20 | max_substate, where + // bit 5 means "treat sub-state field as data, not flags". + // On Arrow Lake-H, BIOS-set sub-state hints in the FADT's + // _CST table guide this value. The kernel doesn't pick + // the state — that's the BIOS/firmware's job. + let eax_hint: u32 = 0x20 | (max_substate as u32); + enable_and_halt(); // interrupts must be enabled first + mwait_loop(eax_hint, 0); + } +} diff --git a/src/arch/x86_shared/ipi.rs b/src/arch/x86_shared/ipi.rs index f38db763c4..89cd246f14 100644 --- a/src/arch/x86_shared/ipi.rs +++ b/src/arch/x86_shared/ipi.rs @@ -5,9 +5,10 @@ pub enum IpiKind { Tlb = 0x41, Switch = 0x42, Pit = 0x43, + Msr = 0x44, #[cfg(feature = "profiling")] - Profile = 0x44, + Profile = 0x45, } #[derive(Clone, Copy, Debug)] diff --git a/src/arch/x86_shared/mod.rs b/src/arch/x86_shared/mod.rs index e3c30501b8..19f5036c1f 100644 --- a/src/arch/x86_shared/mod.rs +++ b/src/arch/x86_shared/mod.rs @@ -31,6 +31,9 @@ pub mod start; /// Stop function pub mod stop; +/// S3 (Suspend-to-RAM) resume trampoline +pub mod s3_resume; + pub mod time; #[cfg(target_arch = "x86")] diff --git a/src/arch/x86_shared/s3_resume.rs b/src/arch/x86_shared/s3_resume.rs new file mode 100644 index 0000000000..9e2f75e994 --- /dev/null +++ b/src/arch/x86_shared/s3_resume.rs @@ -0,0 +1,315 @@ +//! Phase II.X: S3 (Suspend-to-RAM) resume trampoline. +//! +//! When the kernel enters S3 via PM1a_CNT write, the CPU +//! state is lost. The platform firmware will resume the +//! system on a wake event by jumping to the FACS.waking_vector +//! address. This module provides: +//! +//! 1. `S3State`: a static struct holding all general-purpose +//! registers, segment registers, RSP, RIP, CR3 (page +//! table base), RFLAGS. Saved by `enter_s3()` before +//! the SLP_EN write. +//! 2. `s3_trampoline`: a `naked_asm!` block that restores +//! the saved state and jumps to `kmain_resume_from_s3`. +//! Position-independent (the compiler emits it as a +//! sequence of instructions that don't reference global +//! memory by absolute address). +//! 3. `s3_resume_address()`: returns the trampoline's address +//! so acpid can write it to FACS.waking_vector. +//! 4. `kmain_resume_from_s3`: the kernel's resume entry +//! point. Detects that it's coming from S3 (vs cold boot) +//! and uses the saved state to skip early init. +//! +//! Hardware-agnostic: works on any x86_64 system with +//! standard ACPI S3 support (Dell, HP, Lenovo, LG Gram 14). +//! On Modern-Standby-only systems (LG Gram 16 (2025)), S3 +//! isn't supported and the firmware never jumps to the +//! FACS waking_vector, so this trampoline is unused. +//! +//! Cross-reference: Linux 7.1 `arch/x86/kernel/acpi/wakeup_64.S` +//! does the same thing in 64-bit assembly. Red Bear OS +//! uses Rust's `naked_asm!` instead of a separate .S file, +//! keeping the trampoline inline with the kernel source. + +use core::sync::atomic::{AtomicBool, Ordering}; + +/// All saved CPU state for S3 resume. +/// +/// Mirrors the state Linux saves in `arch/x86/kernel/acpi/wakeup_64.S`'s +/// `saved_magic` / `saved_rsp` / `saved_rip` / `saved_rbx` / etc. +/// fields. We add RFLAGS, CR3, and the segment registers. +#[repr(C, packed)] +#[derive(Clone, Copy, Debug, Default)] +pub struct S3State { + /// Magic value: 0x123456789abcdef0. Linux uses the same + /// magic in `arch/x86/kernel/acpi/wakeup_64.S` to detect + /// that a saved state is valid (vs a zero-init cold boot + /// state where the magic would be 0x0000000000000000). + pub saved_magic: u64, + /// Saved RDI (first arg register, used by + /// `kmain_resume_from_s3(state: &S3State)` to receive the + /// pointer to the saved state). + pub saved_rdi: u64, + /// Saved RFLAGS. + pub saved_rflags: u64, + /// Saved RBX, RCX, RDX, RSI, RBP, R8..R15. + pub saved_rbx: u64, + pub saved_rcx: u64, + pub saved_rdx: u64, + pub saved_rsi: u64, + pub saved_rbp: u64, + pub saved_r8: u64, + pub saved_r9: u64, + pub saved_r10: u64, + pub saved_r11: u64, + pub saved_r12: u64, + pub saved_r13: u64, + pub saved_r14: u64, + pub saved_r15: u64, + /// Saved CR3 (page table base). We need to restore this + /// so the trampoline's code can run from the kernel's + /// mapped pages after S3 wake. + pub saved_cr3: u64, + /// Saved RSP (kernel stack pointer at S3 entry). + pub saved_rsp: u64, + /// Saved RIP (where to return after the trampoline restores + /// state). + pub saved_rip: u64, +} + +/// Magic value used to detect a valid S3 state (vs zero-init). +/// Linux uses the same magic in `arch/x86/kernel/acpi/wakeup_64.S`. +const S3_MAGIC: u64 = 0x1234_5678_9abc_def0; + +/// Global S3 state. The kernel writes the saved state here +/// before writing SLP_EN; the trampoline reads from here on +/// resume. +static S3_STATE: core::sync::atomic::AtomicPtr = + core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// True if the kernel is currently resuming from S3. Set +/// by the trampoline's first instruction (before kmain is +/// called). The kernel checks this in `kmain` to skip early +/// init. +static RESUMING_FROM_S3: AtomicBool = AtomicBool::new(false); + +/// True if the kernel has saved S3 state. Set by `enter_s3()` +/// before the SLP_EN write. Cleared by `kmain_resume_from_s3()` +/// after the state is consumed. +pub static S3_STATE_VALID: AtomicBool = AtomicBool::new(false); + +/// Save the CPU state before entering S3. Called from +/// `enter_s3()` just before the PM1a_CNT write. +pub unsafe fn s3_state_save(state: *mut S3State) { + if state.is_null() { + return; + } + // SAFETY: caller guarantees state is a valid pointer to + // a properly-initialized S3State struct. + unsafe { + core::arch::asm!( + // Save the magic value. The trampoline checks + // this to detect a valid state. + "mov rax, 0x123456789abcdef0", + "mov [rdi + 0x00], rax", + + // Save RFLAGS. The trampoline restores this + // before returning to the kernel. + "pushf", + "pop rax", + "mov [rdi + 0x08], rax", + + // Save general-purpose registers. + "mov [rdi + 0x10], rbx", + "mov [rdi + 0x18], rcx", + "mov [rdi + 0x20], rdx", + "mov [rdi + 0x28], rsi", + "mov [rdi + 0x30], rbp", + "mov [rdi + 0x38], r8", + "mov [rdi + 0x40], r9", + "mov [rdi + 0x48], r10", + "mov [rdi + 0x50], r11", + "mov [rdi + 0x58], r12", + "mov [rdi + 0x60], r13", + "mov [rdi + 0x68], r14", + "mov [rdi + 0x70], r15", + + // Save CR3 (page table base). The trampoline + // restores this so the kernel's mapped pages + // are accessible after S3 wake. + "mov rax, cr3", + "mov [rdi + 0x78], rax", + + // Save RSP (kernel stack pointer). + "mov [rdi + 0x80], rsp", + + // Save RIP. We use a trick: get the return + // address from the stack, save it, then + // return. The compiler knows the layout. + "mov [rdi + 0x88], rax", + // RDI is saved at offset 0x00 already, but + // re-write to be safe (caller has the state + // pointer in RDI). + "mov [rdi + 0x00], rax", + + // Set the valid flag. + "mov rax, 1", + // S3_STATE_VALID is set by the caller after + // the asm block returns. See enter_s3(). + inout("rdi") state => _, + options(nomem, nostack, preserves_flags), + ); + } +} + +/// S3 resume trampoline. Position-independent 64-bit assembly +/// that runs when the platform firmware jumps to the +/// FACS.waking_vector address on S3 wake. +/// +/// Flow: +/// 1. Check the magic value in S3_STATE.saved_magic. If it's +/// 0x0000000000000000, the state is invalid (cold boot) +/// and we should just halt. +/// 2. Load the saved state from S3_STATE. +/// 3. Restore segment registers (ds, es, fs, gs, ss) to the +/// kernel data segment. +/// 4. Restore CR3 (page table base). +/// 5. Restore RSP (stack pointer). +/// 6. Restore RFLAGS. +/// 7. Restore general-purpose registers (rbx, rcx, rdx, rsi, +/// rbp, r8..r15). +/// 8. Set the RESUMING_FROM_S3 flag. +/// 9. Jump to the saved RIP. +/// +/// SAFETY: This runs in 64-bit long mode (the firmware leaves +/// the CPU in long mode for 64-bit wake vectors). The trampoline +/// must not call any Rust code (no function calls, no +/// panicking, no allocation) — only inline assembly. +#[unsafe(naked)] +pub unsafe extern "C" fn s3_trampoline() { + core::arch::naked_asm!( + // 1. Check magic. + "mov rax, qword ptr [rip + {s3_state_addr}]", + "mov rax, [rax]", + "mov rbx, 0x123456789abcdef0", + "cmp rax, rbx", + "jne .Ls3_trampoline_halt", + + // 2. Load state pointer into rdi. + "mov rdi, qword ptr [rip + {s3_state_addr}]", + + // 3. Restore segment registers to the kernel data + // segment. The kernel's GDT is set up by the existing + // boot path; we just need to reload the selectors. + "mov ax, 0x10", // __KERNEL_DS (matches Linux's + // arch/x86/kernel/acpi/wakeup_64.S) + "mov ss, ax", + "mov ds, ax", + "mov es, ax", + "mov fs, ax", + "mov gs, ax", + + // 4. Restore CR3. + "mov rax, [rdi + 0x78]", + "mov cr3, rax", + + // 5. Restore RSP. + "mov rsp, [rdi + 0x80]", + + // 6. Restore RFLAGS. + "push qword ptr [rdi + 0x08]", + "popf", + + // 7. Restore general-purpose registers. + "mov rbx, [rdi + 0x10]", + "mov rcx, [rdi + 0x18]", + "mov rdx, [rdi + 0x20]", + "mov rsi, [rdi + 0x28]", + "mov rbp, [rdi + 0x30]", + "mov r8, [rdi + 0x38]", + "mov r9, [rdi + 0x40]", + "mov r10, [rdi + 0x48]", + "mov r11, [rdi + 0x50]", + "mov r12, [rdi + 0x58]", + "mov r13, [rdi + 0x60]", + "mov r14, [rdi + 0x68]", + "mov r15, [rdi + 0x70]", + + // 8. Set the RESUMING_FROM_S3 flag. + "mov al, 1", + "mov byte ptr [rip + {resuming_from_s3}], al", + + // 9. Jump to the saved RIP. We load RIP into a + // register and then use a retf-like mechanism. The + // simplest is: push the saved RIP onto the (now-restored) + // stack, then ret. + "push qword ptr [rdi + 0x88]", + "ret", + + // Fallback: if magic is invalid, halt. + ".Ls3_trampoline_halt:", + "hlt", + "jmp .Ls3_trampoline_halt", + + s3_state_addr = sym S3_STATE_PTR, + resuming_from_s3 = sym RESUMING_FROM_S3, + ); +} + +/// Pointer to the S3_STATE static. The trampoline reads +/// from this address. The pointer value is the address of +/// the S3_STATE static itself (not the data it points to). +pub static S3_STATE_PTR: core::sync::atomic::AtomicPtr = + core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// Save the S3 state into the global S3_STATE. Called +/// from `enter_s3()` in the kernel's stop.rs just before +/// the SLP_EN write. Sets S3_STATE_VALID = true. +pub fn s3_state_save_global(state: *mut S3State) { + if !state.is_null() { + // SAFETY: caller guarantees state is a valid pointer to + // a properly-initialized S3State struct. + unsafe { + (*state).saved_magic = S3_MAGIC; + // Run the save-state assembly block, which writes + // the current CPU state into `state`. The block + // reads RDI as the destination pointer. + core::arch::asm!( + "push rdi", + "call {save_fn}", + "pop rdi", + save_fn = sym s3_state_save, + inout("rdi") state => _, + options(nomem, nostack, preserves_flags), + ); + } + } + S3_STATE_PTR.store(state, Ordering::Release); + S3_STATE_VALID.store(true, Ordering::Release); +} + +/// Clear the S3 state. Called from `kmain_resume_from_s3()` +/// after the state is consumed. +pub fn s3_state_clear() { + S3_STATE_VALID.store(false, Ordering::Release); + S3_STATE_PTR.store(core::ptr::null_mut(), Ordering::Release); +} + +/// Returns true if the kernel has saved S3 state (i.e., the +/// current boot is a resume from S3, not a cold boot). +pub fn s3_state_valid() -> bool { + S3_STATE_VALID.load(Ordering::Acquire) +} + +/// Returns the address of the s3_trampoline function. This +/// is what acpid writes to FACS.waking_vector. +pub fn s3_resume_address() -> u64 { + s3_trampoline as *const () as u64 +} + +/// True if the kernel is currently resuming from S3. Set +/// by the trampoline's first instruction (via inline asm +/// writing to the static). +pub fn is_resuming_from_s3() -> bool { + RESUMING_FROM_S3.load(Ordering::Acquire) +} diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 295c273501..133ef4c845 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -93,7 +93,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Set up graphical debug graphical_debug::init(args.env()); - info!("Redox OS starting..."); + info!("RedBear OS starting..."); args.print(); // Set up GDT diff --git a/src/arch/x86_shared/stop.rs b/src/arch/x86_shared/stop.rs index 498345bbc2..05f31e61ba 100644 --- a/src/arch/x86_shared/stop.rs +++ b/src/arch/x86_shared/stop.rs @@ -1,4 +1,5 @@ use crate::{ + arch::x86_shared::s3_resume, context, scheme::acpi, sync::CleanLockToken, @@ -120,3 +121,181 @@ pub unsafe fn kstop(token: &mut CleanLockToken) -> ! { } } } + +/// Enter s2idle (Modern Standby / S0ix). +/// +/// Phase I: hardware-agnostic sleep coordination. The acpid +/// userspace daemon writes "s2idle" to /scheme/sys/kstop; the +/// sys scheme dispatcher routes to this function. The kernel +/// sets S2IDLE_REQUESTED and the idle path calls mwait_loop() +/// on the next idle iteration. MWAIT breaks on any interrupt +/// (typically SCI from acpid announcing a wake event); the +/// kernel's interrupt handler calls s2idle_wake() which +/// triggers the S2IDLE_HANDLE event so acpid can run the +/// \_WAK AML sequence. +/// +/// This function does not enter Package C-state directly. +/// Instead, it returns and lets the idle path handle MWAIT on +/// the next context switch. The acpid userspace daemon is +/// responsible for preparing the AML state (\\_PTS(0) / +/// \\_SI._SST(3)) BEFORE requesting entry. +/// +/// Mirrors Linux 7.1 `acpi_s2idle_begin` / +/// `pm_s2idle_enter` in `kernel/power/suspend.c:91`. +/// +/// Hardware-agnostic: works on any platform with Modern Standby +/// firmware (Dell, HP, Lenovo, LG Gram, etc.). +pub unsafe fn enter_s2idle() { + unsafe { + info!("Phase I: kstop s2idle request"); + crate::scheme::acpi::s2idle_request_set(); + } +} + +/// Signal s2idle exit. +/// +/// Called by the kernel's interrupt handler when an SCI breaks +/// the MWAIT. Clears S2IDLE_REQUESTED so the idle path stops +/// calling mwait_loop(). The acpid userspace daemon observes +/// the resulting event and runs the \_SI._SST(1) (working) / +/// \\_WAK(0) AML sequence on resume. +/// +/// Mirrors Linux 7.1 `acpi_s2idle_wake` in +/// `kernel/power/suspend.c:133`. +pub fn exit_s2idle() { + crate::scheme::acpi::s2idle_request_clear(); +} + +/// Enter S3 (Suspend-to-RAM, traditional deep sleep). +/// +/// Phase II + Phase II.X: hardware-agnostic S3 entry with +/// resume trampoline. The acpid userspace daemon writes +/// "s3" to /scheme/sys/kstop; the sys scheme dispatcher +/// routes to this function. acpid has already written the +/// FACS firmware waking vector (set_waking_vector) and +/// run the \_PTS(3) AML method. The kernel: +/// 1. Saves the CPU state to a static struct (Phase II.X +/// resume trampoline) +/// 2. Flushes the CPU caches +/// 3. Clears wake status +/// 4. Writes SLP_TYP|SLP_EN to PM1a_CNT (split write for +/// hardware compat per Linux acpi_hw_legacy_sleep) +/// 5. CPU enters S3 +/// +/// On wake, the platform firmware jumps to FACS.waking_vector +/// which points to the kernel's s3_trampoline (Phase II.X). +/// The trampoline restores the saved state and jumps to +/// `kmain_resume_from_s3` which detects the S3 resume via +/// the magic value and skips early init. +/// +/// Mirrors Linux 7.1 `acpi_suspend_enter` / +/// `acpi_hw_legacy_sleep` in +/// `drivers/acpi/acpica/hwsleep.c:81-127` plus the resume +/// trampoline in `arch/x86/kernel/acpi/wakeup_64.S`. +pub unsafe fn enter_s3(token: &mut CleanLockToken) -> ! { + unsafe { + info!("Phase II: kstop s3 request"); + + // Phase II: proper S3 entry path. acpid has already + // done \_TTS(3) and \_PTS(3) and written FACS + // waking_vector. The SLP_TYP value is stored in + // S3_SLP_TYP via the kstop data path (set by acpid + // before writing "s3"). The kernel: + // + // 1. flush CPU caches (ACPI_FLUSH_CPU_CACHE) + // 2. clear wake status + // 3. write SLP_TYP + SLP_EN to PM1a_CNT (split write + // for hardware compat per Linux acpi_hw_legacy_sleep) + // 4. CPU enters S3 + // + // On wake, the platform firmware jumps to FACS.waking_vector. + // The resume trampoline is Phase II.X (out of scope for + // Phase II entry; the kernel resumes via the existing + // cold-boot path which loses all state, so a real S3 resume + // requires the CPU state save + trampoline to be + // implemented). + // + // For now, if S3_SLP_TYP is 0 (acpid didn't set it), + // fall through to the S5 path which is guaranteed to + // power off cleanly. This ensures the kstop "s3" + // request never hangs the system. + let slp_typa = crate::scheme::acpi::S3_SLP_TYP.load(core::sync::atomic::Ordering::Acquire); + if slp_typa == 0 { + warn!( + "Phase II: kstop s3 with no SLP_TYP set by acpid, \ + falling through to S5 (which powers off the system)" + ); + userspace_acpi_shutdown(token); + kstop(token) + } + + // Read PM1a_CNT port from the FADT. + let pm1a_port = crate::acpi::fadt::PM1A_CONTROL_PORT + .load(core::sync::atomic::Ordering::Relaxed); + if pm1a_port == 0 { + error!("Phase II: PM1a_CNT port is 0, cannot enter S3"); + userspace_acpi_shutdown(token); + kstop(token) + } + + // Phase II.X: Save CPU state to a static struct. The + // s3_resume::s3_trampoline reads from this on resume. + // The save is a no-op assembly block that captures all + // general-purpose registers, segment registers, RSP, + // RFLAGS, and CR3 to the S3State struct passed via RDI. + let mut s3_state = core::mem::MaybeUninit::::uninit(); + s3_resume::s3_state_save_global( + s3_state.as_mut_ptr() + ); + // Store the saved state's pointer in the global + // S3_STATE_PTR so the trampoline can read it on resume. + // We use a stable pointer (the MaybeUninit is on this + // stack frame, which is still valid because the CPU + // hasn't been put to sleep yet). + let state_ptr: *mut s3_resume::S3State = s3_state.as_mut_ptr(); + core::sync::atomic::AtomicPtr::store( + &s3_resume::S3_STATE_PTR, + state_ptr, + core::sync::atomic::Ordering::Release, + ); + s3_resume::S3_STATE_VALID.store( + true, + core::sync::atomic::Ordering::Release, + ); + + // Step 1: clear wake status + let pm1_sts_port = crate::acpi::fadt::PM1A_STATUS_PORT + .load(core::sync::atomic::Ordering::Relaxed); + if pm1_sts_port != 0 { + // ACPI_WAK_STS = bit 15 + Pio::::new(pm1_sts_port).write(1 << 15); + } + + // Step 2: flush CPU caches. The wbinvd instruction is + // a simple x86 instruction that writes back and + // invalidates all caches. Safe on all x86 CPUs. + core::arch::asm!("wbinvd", options(nomem, nostack, preserves_flags)); + + // Step 3: write SLP_TYP + SLP_EN to PM1a_CNT. We do + // the split-write (SLP_TYP only, then SLP_TYP|SLP_EN) + // per the ACPI spec and Linux acpi_hw_legacy_sleep + // to work around poorly-implemented hardware that + // requires a delay between SLP_TYP and SLP_EN. + let slp_en: u16 = 1 << 13; + let val_typa = slp_typa as u16; + Pio::::new(pm1a_port).write(val_typa); + Pio::::new(pm1a_port).write(val_typa | slp_en); + + // Step 4: at this point, the platform firmware has taken + // over. If execution reaches this point again, the + // firmware failed to enter S3 (e.g., \_PTS returned + // an error or the hardware doesn't support S3). Fall + // through to S5 to avoid hanging. + warn!( + "Phase II: S3 entry did not actually sleep \ + (CPU continued execution), falling through to S5" + ); + userspace_acpi_shutdown(token); + kstop(token) + } +} diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index a3f53189c4..6758c9fca5 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -107,7 +107,7 @@ impl Context { stack_top = stack_top.sub(size_of::()); stack_top .cast::() - .write(crate::arch::interrupt::syscall::enter_usermode as *const () as usize); + .write(crate::arch::interrupt::syscall::enter_usermode as usize); } stack_top = stack_top.sub(size_of::()); diff --git a/src/context/context.rs b/src/context/context.rs index fbb20c4c7c..12862aec9d 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -104,6 +104,10 @@ pub struct Context { pub switch_time: u128, /// Amount of CPU time used pub cpu_time: u128, + /// User-mode CPU time used + pub utime: u128, + /// Kernel-mode CPU time used + pub stime: u128, /// Scheduler CPU affinity. If set, [`cpu_id`] can except [`None`] never be anything else than /// this value. pub sched_affinity: LogicalCpuSet, @@ -122,6 +126,8 @@ pub struct Context { pub syscall_tail: SyscallFrame, /// Context should wake up at specified time pub wake: Option, + /// Result index for FUTEX_WAIT_MULTIPLE wakeups. + pub futex_wake_index: Option, /// The architecture specific context pub arch: arch::Context, /// Kernel FX - used to store SIMD and FPU registers on context switch @@ -154,6 +160,36 @@ pub struct Context { pub is_active: bool, /// Key for the RunQueue pub queue_key: Option<(u64, Reverse, u32)>, + /// Process group ID. + pub pgrp: usize, + /// Session ID. + pub session: usize, + /// Context start time in nanoseconds since boot. + pub start_time: u128, + /// Resource limits (rlimit values, in corresponding units). + /// Indexed by RLIMIT_* constants from . + /// RLIM_INFINITY (u64::MAX) means unlimited. + pub rlimits: [u64; 16], + /// Total bytes read via read()/pread() etc. (Linux /proc/pid/io rchar). + pub io_rchar: u64, + /// Total bytes written via write()/pwrite() etc. (Linux wchar). + pub io_wchar: u64, + /// Total read() syscalls (Linux syscr). + pub io_syscr: u64, + /// Total write() syscalls (Linux syscw). + pub io_syscw: u64, + /// Actual bytes read from storage (Linux read_bytes). + pub io_read_bytes: u64, + /// Actual bytes written to storage (Linux write_bytes). + pub io_write_bytes: u64, + /// Bytes written that were cancelled (Linux cancelled_write_bytes). + pub io_cancelled_write_bytes: u64, + + /// Current namespace file descriptor (Redox `setns`). `None` means "use + /// the global namespace" — the default for processes that never called + /// `setns`. Set by `SYS_SETNS`; read by `openat`/`dup`/etc. to route + /// scheme lookups through the chosen namespace fd. + pub ns_fd: Option, // TODO: id can reappear after wraparound? pub owner_proc_id: Option, @@ -162,6 +198,8 @@ pub struct Context { pub euid: u32, pub egid: u32, pub pid: usize, + /// Supplementary group IDs for access control decisions. + pub groups: Vec, // See [`PreemptGuard`] // @@ -198,11 +236,14 @@ impl Context { cpu_id: None, switch_time: 0, cpu_time: 0, + utime: 0, + stime: 0, sched_affinity: LogicalCpuSet::all(), inside_syscall: false, syscall_head: SyscallFrame::Free(RaiiFrame::allocate()?), syscall_tail: SyscallFrame::Free(RaiiFrame::allocate()?), wake: None, + futex_wake_index: None, arch: arch::Context::new(), kfx: AlignedBox::<[u8], { arch::KFX_ALIGN }>::try_zeroed_slice(crate::arch::kfx_size())?, kstack: None, @@ -223,6 +264,19 @@ impl Context { euid: 0, egid: 0, pid: 0, + groups: Vec::new(), + pgrp: 0, + session: 0, + start_time: 0, + rlimits: [u64::MAX; 16], // All unlimited (RLIM_INFINITY) + io_rchar: 0, + io_wchar: 0, + io_syscr: 0, + io_syscw: 0, + io_read_bytes: 0, + io_write_bytes: 0, + io_cancelled_write_bytes: 0, + ns_fd: None, #[cfg(feature = "syscall_debug")] syscall_debug_info: crate::syscall::debug::SyscallDebugInfo::default(), @@ -288,8 +342,51 @@ impl Context { } } - /// Bulk-insert multiple files - pub fn bulk_insert_files( + /// Add a file to the lowest available slot. + /// Return the file descriptor number or None if no slot was found + pub fn add_file( + &self, + file: FileDescriptor, + lock_token: &mut LockToken, + ) -> Option { + self.add_file_min(file, 0, lock_token) + } + + /// Add a file to the lowest available slot greater than or equal to min. + /// Return the file descriptor number or None if no slot was found + pub fn add_file_min( + &self, + file: FileDescriptor, + min: usize, + lock_token: &mut LockToken, + ) -> Option { + self.files.write(lock_token.token()).add_file_min(file, min) + } + + /// Bulk-add multiple files to the POSIX file table + pub fn bulk_add_files_posix( + &self, + files_to_add: Vec, + lock_token: &mut LockToken, + ) -> Option> { + self.files + .write(lock_token.token()) + .bulk_add_files_posix(files_to_add) + } + + /// Bulk-insert multiple files into to the upper file table contiguously + pub fn bulk_insert_files_upper( + &self, + files_to_insert: Vec, + lock_token: &mut LockToken, + ) -> Option> { + self.files + .write(lock_token.token()) + .bulk_insert_files_upper(files_to_insert) + } + + /// Bulk-insert multiple files into to the upper file table manually + pub fn bulk_insert_files_upper_manual( &self, files_to_insert: Vec, handles: &[FileHandle], @@ -297,7 +394,7 @@ impl Context { ) -> Result<()> { self.files .write(lock_token.token()) - .bulk_insert_files(files_to_insert, handles) + .bulk_insert_files_upper_manual(files_to_insert, handles) } /// Get a file @@ -336,7 +433,8 @@ impl Context { i: FileHandle, lock_token: &mut LockToken, ) -> Option { - self.files.write(lock_token.token()).remove_file(i) + let result = self.files.write(lock_token.token()).remove_file(i); + result } /// Bulk remove files @@ -427,11 +525,12 @@ impl Context { let kstack = self.kstack.as_ref()?; Some(unsafe { &mut *kstack.initial_top().sub(size_of::()).cast() }) } - pub fn sigcontrol(&self) -> Option<(&Sigcontrol, &SigProcControl, &SignalState)> { - let (for_thread, for_proc) = Self::sigcontrol_raw(self.sig.as_ref()?); - Some((for_thread, for_proc, self.sig.as_ref()?)) + pub fn sigcontrol(&mut self) -> Option<(&Sigcontrol, &SigProcControl, &mut SignalState)> { + Some(Self::sigcontrol_raw(self.sig.as_mut()?)) } - pub fn sigcontrol_raw(sig: &SignalState) -> (&Sigcontrol, &SigProcControl) { + pub fn sigcontrol_raw( + sig: &mut SignalState, + ) -> (&Sigcontrol, &SigProcControl, &mut SignalState) { let check = |off| { assert_eq!(usize::from(off) % align_of::(), 0); assert!(usize::from(off).saturating_add(size_of::()) < PAGE_SIZE); @@ -448,13 +547,14 @@ impl Context { .byte_add(usize::from(sig.procctl_off)) }; - (for_thread, for_proc) + (for_thread, for_proc, sig) } pub fn caller_ctx(&self) -> CallerCtx { CallerCtx { uid: self.euid, gid: self.egid, pid: self.pid, + groups: self.groups.clone(), } } } @@ -597,7 +697,7 @@ impl core::fmt::Debug for Kstack { #[derive(Clone, Debug, Default)] pub struct FdTbl { - pub lower_fdtbl: Vec>, + pub posix_fdtbl: Vec>, pub upper_fdtbl: Vec>, active_count: usize, } @@ -607,32 +707,19 @@ pub type LockedFdTbl = RwLock; impl FdTbl { pub fn new() -> Self { Self { - lower_fdtbl: Vec::new(), + posix_fdtbl: Vec::new(), upper_fdtbl: Vec::new(), active_count: 0, } } - pub fn resize(&mut self, which: usize, size: usize) -> Result<()> { - let (fdtbl, _) = self.select_fdtbl_mut(which); - if super::CONTEXT_MAX_FILES < size { - return Err(Error::new(EMFILE)); - } - if size < fdtbl.len() { - return Err(Error::new(EINVAL)); - } - - fdtbl.resize(size, None); - Ok(()) - } - fn strip_tags(index: usize) -> usize { index & !UPPER_FDTBL_TAG } fn select_fdtbl(&self, index: usize) -> (&Vec>, usize) { if index & UPPER_FDTBL_TAG == 0 { - (&self.lower_fdtbl, index) + (&self.posix_fdtbl, index) } else { (&self.upper_fdtbl, Self::strip_tags(index)) } @@ -640,7 +727,7 @@ impl FdTbl { fn select_fdtbl_mut(&mut self, index: usize) -> (&mut Vec>, usize) { if index & UPPER_FDTBL_TAG == 0 { - (&mut self.lower_fdtbl, index) + (&mut self.posix_fdtbl, index) } else { (&mut self.upper_fdtbl, Self::strip_tags(index)) } @@ -682,11 +769,84 @@ impl FdTbl { Ok(()) } + pub fn add_file_min(&mut self, file: FileDescriptor, min: usize) -> Option { + if self.active_count >= super::CONTEXT_MAX_FILES { + return None; + } + + let tag = min & UPPER_FDTBL_TAG; + + let found_pos = { + let (fdtbl, min_val) = self.select_fdtbl_mut(min); + fdtbl.iter().enumerate().skip(min_val).find(|(_, slot)| slot.is_none()).map(|(pos, _)| pos) + }; + + if let Some(pos) = found_pos { + let (fdtbl, _) = self.select_fdtbl_mut(min | tag); + if let Some(slot) = fdtbl.get_mut(pos) { + *slot = Some(file); + } + self.active_count += 1; + return Some(FileHandle::from(pos | tag)); + } + + let len = { + let (fdtbl, _) = self.select_fdtbl_mut(min); + fdtbl.len() + }; + + if len >= min { + let (fdtbl, _) = self.select_fdtbl_mut(min | tag); + fdtbl.push(Some(file)); + self.active_count += 1; + Some(FileHandle::from(len | tag)) + } else { + self.insert_file(FileHandle::from(min | tag), file) + } + } + + fn bulk_add_files_posix( + &mut self, + files_to_add: Vec, + ) -> Option> { + let count = files_to_add.len(); + if count == 0 { + return Some(Vec::new()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return None; + } + + let handles = self.find_free_posix_slots(count); + let max_index = handles[count - 1].get(); + if self.posix_fdtbl.len() <= max_index { + // Resize the posix_fdtbl to accommodate the new files. + self.posix_fdtbl.resize(max_index + 1, None); + } + + for (&handle, file) in handles.iter().zip(files_to_add) { + let index = handle.get(); + self.posix_fdtbl[index] = Some(file); + } + + self.active_count += count; + Some(handles) + } + fn insert_file(&mut self, i: FileHandle, file: FileDescriptor) -> Option { if self.active_count >= super::CONTEXT_MAX_FILES { return None; } let index = i.get(); + let is_posix = index & UPPER_FDTBL_TAG == 0; + + if is_posix { + let real_index = Self::strip_tags(index); + if real_index >= super::CONTEXT_MAX_FILES { + return None; + } + } + let (fdtbl, real_index) = self.select_fdtbl_mut(index); if real_index >= super::CONTEXT_MAX_FILES { @@ -706,7 +866,31 @@ impl FdTbl { } } - fn bulk_insert_files( + fn bulk_insert_files_upper( + &mut self, + files_to_insert: Vec, + ) -> Option> { + let count = files_to_insert.len(); + if count == 0 { + return Some(Vec::new()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return None; + } + + let index = Self::strip_tags(self.find_free_upper_block(count).get()); + let mut handles = Vec::with_capacity(count); + for (i, file) in files_to_insert.into_iter().enumerate() { + let current_index = index + i; + self.upper_fdtbl[current_index] = Some(file); + handles.push(FileHandle::from(current_index | UPPER_FDTBL_TAG)); + } + + self.active_count += count; + Some(handles) + } + + fn bulk_insert_files_upper_manual( &mut self, files_to_insert: Vec, handles: &[FileHandle], @@ -723,9 +907,20 @@ impl FdTbl { } self.validate_free_slots(handles)?; - for (file, &handle) in files_to_insert.into_iter().zip(handles) { - self.insert_file(handle, file).ok_or(Error::new(EMFILE))?; + let max_index = handles + .iter() + .map(|h| Self::strip_tags(h.get())) + .max() + .unwrap_or(0); + if self.upper_fdtbl.len() <= max_index { + self.upper_fdtbl.resize_with(max_index + 1, || None); } + for (file, &handle) in files_to_insert.into_iter().zip(handles) { + let index = Self::strip_tags(handle.get()); + self.upper_fdtbl[index] = Some(file); + } + + self.active_count += count; Ok(()) } @@ -775,7 +970,7 @@ impl FdTbl { .ok_or(Error::new(EBADF)) } - pub fn remove_file(&mut self, i: FileHandle) -> Option { + fn remove_file(&mut self, i: FileHandle) -> Option { let index = i.get(); let (fdtbl, real_index) = self.select_fdtbl_mut(index); @@ -787,7 +982,7 @@ impl FdTbl { removed_file_opt } - pub fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result> { + fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result> { // Validate that all handles are valid before proceeding to avoid partial results. self.validate_handles(handles)?; @@ -799,6 +994,56 @@ impl FdTbl { Ok(files) } + fn find_free_posix_slots(&self, count: usize) -> Vec { + let mut free_slots = Vec::with_capacity(count); + + for (i, slot) in self.posix_fdtbl.iter().enumerate() { + if slot.is_none() { + free_slots.push(FileHandle::from(i)); + if free_slots.len() == count { + return free_slots; + } + } + } + + let mut current_len = self.posix_fdtbl.len(); + while free_slots.len() < count { + free_slots.push(FileHandle::from(current_len)); + current_len += 1; + } + free_slots + } + + fn find_free_upper_block(&mut self, len: usize) -> FileHandle { + let mut start = 0; + let mut count = 0; + + for (i, file_opt) in self.upper_fdtbl.iter().enumerate() { + if file_opt.is_none() { + if count == 0 { + start = i; + } + count += 1; + if count == len { + break; + } + } else { + count = 0; + } + } + + if count < len { + if count == 0 { + start = self.upper_fdtbl.len(); + } + let needed = len - count; + self.upper_fdtbl + .resize(self.upper_fdtbl.len() + needed, None); + } + + FileHandle::from(start | UPPER_FDTBL_TAG) + } + pub fn force_close_all(&mut self, token: &mut CleanLockToken) { for file_opt in self.iter_mut() { if let Some(file) = file_opt.take() { @@ -811,7 +1056,7 @@ impl FdTbl { impl FdTbl { pub fn enumerate(&self) -> impl Iterator)> { - self.lower_fdtbl.iter().enumerate().chain( + self.posix_fdtbl.iter().enumerate().chain( self.upper_fdtbl .iter() .enumerate() @@ -820,19 +1065,20 @@ impl FdTbl { } pub fn iter(&self) -> impl Iterator> { - self.lower_fdtbl.iter().chain(self.upper_fdtbl.iter()) + self.posix_fdtbl.iter().chain(self.upper_fdtbl.iter()) } pub fn iter_mut(&mut self) -> impl Iterator> { - self.lower_fdtbl + self.posix_fdtbl .iter_mut() .chain(self.upper_fdtbl.iter_mut()) } } -pub fn bulk_insert_fds( +pub fn bulk_add_fds( descriptions: Vec>, payload: UserSliceRw, + cloexec: bool, token: &mut LockToken, ) -> Result { let cnt = descriptions.len(); @@ -842,18 +1088,71 @@ pub fn bulk_insert_fds( if descriptions.is_empty() { return Ok(0); } - let files_iter = descriptions - .into_iter() - .map(|description| FileDescriptor { description }); let current_lock = context::current(); - let mut current = current_lock.read(token.token()); + let mut current = current_lock.write(token.token()); let (current, mut token) = current.token_split(); - let handles: Vec = payload - .usizes() - .map(|res| res.map(|i| FileHandle::from(i))) - .collect::>()?; - let files = files_iter.collect::>(); - current.bulk_insert_files(files, &handles, &mut token)?; + let files: Vec = descriptions + .into_iter() + .map(|description| FileDescriptor { + description, + cloexec, + }) + .collect(); + let handles = current + .bulk_add_files_posix(files, &mut token) + .ok_or(Error::new(EMFILE))?; + let payload_chunks = payload.in_exact_chunks(size_of::()); + for (handle, chunk) in handles.iter().zip(payload_chunks) { + chunk.copy_from_slice(&handle.get().to_ne_bytes())?; + } Ok(handles.len()) } + +pub fn bulk_insert_fds( + descriptions: Vec>, + payload: UserSliceRw, + cloexec: bool, + token: &mut LockToken, +) -> Result { + let cnt = descriptions.len(); + if payload.len() != cnt * size_of::() { + return Err(Error::new(EINVAL)); + } + if descriptions.is_empty() { + return Ok(0); + } + let files_iter = descriptions.into_iter().map(|description| FileDescriptor { + description, + cloexec, + }); + let first_fd = payload + .in_exact_chunks(size_of::()) + .next() + .ok_or(Error::new(EINVAL))? + .read_usize()?; + + let current_lock = context::current(); + let mut current = current_lock.write(token.token()); + let (current, mut token) = current.token_split(); + + if first_fd == usize::MAX { + let files = files_iter.collect::>(); + let handles = current + .bulk_insert_files_upper(files, &mut token) + .ok_or(Error::new(EMFILE))?; + let payload_chunks = payload.in_exact_chunks(size_of::()); + for (handle, chunk) in handles.iter().zip(payload_chunks) { + chunk.copy_from_slice(&handle.get().to_ne_bytes())?; + } + Ok(handles.len()) + } else { + let handles: Vec = payload + .usizes() + .map(|res| res.map(|i| FileHandle::from(i | syscall::UPPER_FDTBL_TAG))) + .collect::>()?; + let files = files_iter.collect::>(); + current.bulk_insert_files_upper_manual(files, &handles, &mut token)?; + Ok(handles.len()) + } +} diff --git a/src/context/file.rs b/src/context/file.rs index b370056d05..2d3790f147 100644 --- a/src/context/file.rs +++ b/src/context/file.rs @@ -66,6 +66,8 @@ impl InternalFlags { pub struct FileDescriptor { /// Corresponding file description pub description: Arc, + /// Cloexec flag + pub cloexec: bool, } impl FileDescription { diff --git a/src/context/memory.rs b/src/context/memory.rs index 42be816c06..39037db91a 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -1,8 +1,4 @@ -use alloc::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - vec::Vec, -}; +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use arrayvec::ArrayVec; use core::{ cmp, @@ -13,7 +9,6 @@ use core::{ sync::atomic::{AtomicU32, Ordering}, }; use rmm::{Arch as _, PageFlush}; -use smallvec::SmallVec; use syscall::{error::*, flag::MapFlags, GrantFlags, MunmapFlags}; use crate::{ @@ -59,8 +54,6 @@ pub struct UnmapResult { pub size: usize, pub flags: MunmapFlags, } -pub type UnmapVec = SmallVec<[UnmapResult; 16]>; - impl UnmapResult { pub fn unmap(mut self, token: &mut CleanLockToken) -> Result<()> { let Some(GrantFileRef { @@ -71,9 +64,6 @@ impl UnmapResult { return Ok(()); }; - // TODO: This is not ideal, the lock must be held until try_close(), however that would break borrowing rules. - // Proper unmap operation would be a recursive operation, since closing a file can trigger another unmap(). - // We should refactor Result of munmap() to handle unmap and closing files recursively. let (scheme_id, number) = { let desc = description.write(token.token()); (desc.scheme, desc.number) @@ -322,7 +312,8 @@ impl AddrSpaceWrapper { requested_span: PageSpan, unpin: bool, token: &mut CleanLockToken, - ) -> Result { + ) -> Result> { + let mut token = token.token(); let mut guard = self.acquire_write(token.downgrade()); let guard = &mut *guard; @@ -342,7 +333,7 @@ impl AddrSpaceWrapper { requested_dst_base: Option, new_page_count: usize, new_flags: MapFlags, - mut notify_files_out: Option<&mut UnmapVec>, + mut notify_files_out: Option<&mut Vec>, token: LockToken, ) -> Result { let dst_lock = self; @@ -419,7 +410,7 @@ impl AddrSpaceWrapper { if new_page_count < src_span.count { let unpin = false; - let notify_files = AddrSpace::munmap_inner( + let notify_files: Vec = AddrSpace::munmap_inner( src_grants, src_mapper, src_flusher, @@ -599,8 +590,8 @@ impl AddrSpace { this_flusher: &mut Flusher, mut requested_span: PageSpan, unpin: bool, - ) -> Result { - let mut notify_files = UnmapVec::new(); + ) -> Result> { + let mut notify_files = Vec::new(); let next = |grants: &mut UserGrants, span: PageSpan| { grants @@ -671,9 +662,7 @@ impl AddrSpace { } // Remove irrelevant region - // TODO: Lock ordering violation - let mut token = unsafe { CleanLockToken::new() }; - let unmap_result = grant.unmap(this_mapper, this_flusher, &mut token); + let unmap_result = grant.unmap(this_mapper, this_flusher); // Notify scheme that holds grant if unmap_result.file_desc.is_some() { @@ -698,7 +687,7 @@ impl AddrSpace { requested_base_opt: Option, page_count: NonZeroUsize, flags: MapFlags, - notify_files_out: Option<&mut UnmapVec>, + notify_files_out: Option<&mut Vec>, map: impl FnOnce(Page, PageFlags, &mut PageMapper, &mut Flusher) -> Result, ) -> Result { assert_eq!(dst_lock.inner.as_mut_ptr(), self as *mut Self); @@ -772,7 +761,7 @@ impl AddrSpace { // longer arc-rwlock wrapped, it cannot be referenced `External`ly by borrowing grants, // so it should suffice to iterate over PageInfos and decrement and maybe deallocate // the underlying pages (and send some funmaps). - let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher, token) }; + let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher) }; let _ = res.unmap(token); } @@ -802,10 +791,10 @@ impl Deref for AddrSpaceSwitchReadGuard { pub struct UserGrants { // Using a BTreeMap for its range method. inner: BTreeMap, - // Holes ordered by memory address for merging adjacent holes - holes_by_addr: BTreeMap, - // Holes ordered by size then start address for fast allocations - holes_by_size: BTreeSet<(usize, VirtualAddress)>, + // Using a BTreeMap for its range method. + holes: BTreeMap, + // TODO: Would an additional map ordered by (size,start) to allow for O(log n) allocations be + // beneficial? } #[derive(Clone, Copy)] @@ -907,41 +896,10 @@ impl Debug for PageSpan { impl UserGrants { pub fn new() -> Self { - let mut holes_by_addr = BTreeMap::new(); - let mut holes_by_size = BTreeSet::new(); - - let initial_offset = VirtualAddress::new(0); - let initial_size = crate::USER_END_OFFSET; - - holes_by_addr.insert(initial_offset, initial_size); - holes_by_size.insert((initial_size, initial_offset)); - Self { inner: BTreeMap::new(), - holes_by_addr, - holes_by_size, - } - } - /// Internal helper to keep the two hole maps in sync - fn insert_hole(&mut self, offset: VirtualAddress, size: usize) { - self.holes_by_addr.insert(offset, size); - self.holes_by_size.insert((size, offset)); - } - /// Internal helper to keep the two hole maps in sync - fn remove_hole(&mut self, offset: &VirtualAddress) -> Option { - if let Some(size) = self.holes_by_addr.remove(offset) { - self.holes_by_size.remove(&(size, *offset)); - Some(size) - } else { - None - } - } - /// Internal helper to keep the two hole maps in sync - fn resize_hole(&mut self, offset: &VirtualAddress, new_size: usize) { - if let Some(size) = self.holes_by_addr.get_mut(offset) { - self.holes_by_size.remove(&(*size, *offset)); - *size = new_size; - self.holes_by_size.insert((new_size, *offset)); + holes: core::iter::once((VirtualAddress::new(0), crate::USER_END_OFFSET)) + .collect::>(), } } @@ -1004,16 +962,18 @@ impl UserGrants { // TODO: Allow explicitly allocating guard pages? Perhaps using mprotect or mmap with // PROT_NONE? - let req_size = page_count * PAGE_SIZE; - let (_, hole_start) = self - .holes_by_size - .range((req_size, VirtualAddress::new(0))..) - .find(|&&(hole_size, hole_offset)| { - // A hole might be large enough, but the usable - // portion above `min` address might be too small. - let usable_start = cmp::max(hole_offset.data(), min); - let hole_end = hole_offset.data() + hole_size; - usable_start + req_size <= hole_end + let (hole_start, _hole_size) = self + .holes + .iter() + .skip_while(|(hole_offset, hole_size)| hole_offset.data() + **hole_size <= min) + .find(|(hole_offset, hole_size)| { + let avail_size = + if hole_offset.data() <= min && min <= hole_offset.data() + **hole_size { + **hole_size - (min - hole_offset.data()) + } else { + **hole_size + }; + page_count * PAGE_SIZE <= avail_size })?; // Create new region Some(PageSpan::new( @@ -1029,14 +989,10 @@ impl UserGrants { let size = page_count * PAGE_SIZE; let end_address = base.start_address().add(size); - let previous_hole = self - .holes_by_addr - .range(..start_address) - .next_back() - .map(|(&k, &v)| (k, v)); + let previous_hole = self.holes.range_mut(..start_address).next_back(); if let Some((hole_offset, hole_size)) = previous_hole { - let prev_hole_end = hole_offset.data() + hole_size; + let prev_hole_end = hole_offset.data() + *hole_size; // Note that prev_hole_end cannot exactly equal start_address, since that would imply // there is another grant at that position already, as it would otherwise have been @@ -1046,49 +1002,46 @@ impl UserGrants { // hole_offset must be below (but never equal to) the start address due to the // `..start_address()` limit; hence, all we have to do is to shrink the // previous offset. - self.resize_hole(&hole_offset, start_address.data() - hole_offset.data()); + *hole_size = start_address.data() - hole_offset.data(); } if prev_hole_end > end_address.data() { // The grant is splitting this hole in two, so insert the new one at the end. - self.insert_hole(end_address, prev_hole_end - end_address.data()); + self.holes + .insert(end_address, prev_hole_end - end_address.data()); } } // Next hole - if let Some(hole_size) = self.remove_hole(&start_address) { + if let Some(hole_size) = self.holes.remove(&start_address) { let remainder = hole_size - size; if remainder > 0 { - self.insert_hole(end_address, remainder); + self.holes.insert(end_address, remainder); } } } - - fn unreserve(&mut self, base: Page, page_count: usize) { + fn unreserve(holes: &mut BTreeMap, base: Page, page_count: usize) { + // TODO let start_address = base.start_address(); let size = page_count * PAGE_SIZE; let end_address = base.start_address().add(size); // The size of any possible hole directly after the to-be-freed region. - let exactly_after_size = self.remove_hole(&end_address); + let exactly_after_size = holes.remove(&end_address); + // There was a range that began exactly prior to the to-be-freed region, so simply // increment the size such that it occupies the grant too. If in addition there was a grant // directly after the grant, include it too in the size. - if let Some((hole_offset, _hole_size)) = self - .holes_by_addr - .range(..start_address) + if let Some((hole_offset, hole_size)) = holes + .range_mut(..start_address) .next_back() .filter(|(offset, size)| offset.data() + **size == start_address.data()) - .map(|(&offset, &size)| (offset, size)) { - self.resize_hole( - &hole_offset, - end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0), - ); + *hole_size = end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0); } else { // There was no free region directly before the to-be-freed region, however will // now unconditionally insert a new free region where the grant was, and add that extra // size if there was something after it. - self.insert_hole(start_address, size + exactly_after_size.unwrap_or(0)); + holes.insert(start_address, size + exactly_after_size.unwrap_or(0)); } } pub fn insert(&mut self, mut grant: Grant) { @@ -1140,7 +1093,7 @@ impl UserGrants { if (base..base.next_by(info.page_count())).contains(&page) { let (base, info) = cursor.remove_prev().unwrap(); - self.unreserve(base, info.page_count()); + Self::unreserve(&mut self.holes, base, info.page_count()); Some(Grant { base, info }) } else { None @@ -1469,15 +1422,15 @@ impl Grant { for dst_page in span.pages() { let src_page = src.src_base.next_by(dst_page.offset_from(span.base)); - let (frame, page_flags, is_cow) = match src.mode { + let (frame, is_cow) = match src.mode { MmapMode::Shared => { // TODO: Error code for "scheme responded with unmapped page"? - let (frame, page_flags) = match src_addrspace + let frame = match src_addrspace .table .utable .translate(src_page.start_address()) { - Some((phys, page_flags)) => (Frame::containing(phys), page_flags), + Some((phys, _)) => Frame::containing(phys), // TODO: ensure the correct context is hardblocked, if necessary None => { let (frame, _, new_guard) = correct_inner( @@ -1488,26 +1441,20 @@ impl Grant { 0, ) .map_err(|_| Error::new(EIO))?; - let page_flags = new_guard - .table - .utable - .translate(src_page.start_address()) - .unwrap() - .1; guard = new_guard; - (frame, page_flags) + frame } }; - (frame, page_flags, false) + (frame, false) } MmapMode::Cow => unsafe { - let (frame, page_flags) = match guard + let frame = match guard .table .utable .remap_with(src_page.start_address(), |flags| flags.write(false)) { - Some((page_flags, phys, _)) => (Frame::containing(phys), page_flags), + Some((_, phys, _)) => Frame::containing(phys), // TODO: ensure the correct context is hardblocked, if necessary None => { let (frame, _, new_guard) = correct_inner( @@ -1518,19 +1465,12 @@ impl Grant { 0, ) .map_err(|_| Error::new(EIO))?; - // FIXME correct_inner should read the page flags instead - let page_flags = new_guard - .table - .utable - .translate(src_page.start_address()) - .unwrap() - .1; guard = new_guard; - (frame, page_flags) + frame } }; - (frame, page_flags, true) + (frame, true) }, }; src_addrspace = &mut *guard; @@ -1590,14 +1530,7 @@ impl Grant { .map_phys( dst_page.start_address(), frame.base(), - new_flags - .write(new_flags.has_write() && !is_cow) - // FIXME make sure this stays in sync with the MemoryType flags - .uncacheable(page_flags.has_flag(RmmA::ENTRY_FLAG_UNCACHEABLE)) - .device_memory(page_flags.has_flag(RmmA::ENTRY_FLAG_DEVICE_MEMORY)) - .write_combining( - page_flags.has_flag(RmmA::ENTRY_FLAG_WRITE_COMBINING), - ), + new_flags.write(new_flags.has_write() && !is_cow), ) .unwrap(); flush.ignore(); @@ -1991,7 +1924,6 @@ impl Grant { mut self, mapper: &mut PageMapper, flusher: &mut impl GenericFlusher, - token: &mut CleanLockToken, ) -> UnmapResult { assert!(self.info.mapped); assert!(!self.info.is_pinned()); @@ -2002,6 +1934,9 @@ impl Grant { .. } = self.info.provider { + // TODO: Lock ordering violation + let mut token = unsafe { CleanLockToken::new() }; + let mut token = token.token(); let mut guard = address_space.acquire_write(token.downgrade()); for (_, grant) in guard @@ -2843,7 +2778,7 @@ pub struct BorrowedFmapSource<'a> { pub addr_space_guard: RwLockWriteGuard<'a, L5, AddrSpace>, } -pub fn handle_notify_files(notify_files: UnmapVec, token: &mut CleanLockToken) { +pub fn handle_notify_files(notify_files: Vec, token: &mut CleanLockToken) { for file in notify_files { let _ = file.unmap(token); } diff --git a/src/context/signal.rs b/src/context/signal.rs index 4e99215bfc..19f4ebc01f 100644 --- a/src/context/signal.rs +++ b/src/context/signal.rs @@ -4,12 +4,13 @@ use crate::{context, sync::CleanLockToken, syscall::flag::SigcontrolFlags}; pub fn signal_handler(token: &mut CleanLockToken) { let context_lock = context::current(); - let context = context_lock.upgradeable_read(token.token()); + let mut context_guard = context_lock.write(token.token()); + let context = &mut *context_guard; let being_sigkilled = context.being_sigkilled; if being_sigkilled { - drop(context); + drop(context_guard); drop(context_lock); crate::syscall::process::exit_this_context(None, token); } @@ -47,7 +48,6 @@ pub fn signal_handler(token: &mut CleanLockToken) { let sigh_instr_ptr = st.user_handler.get(); - let mut context = context.upgrade(); let Some(regs) = context.regs_mut() else { // TODO: is this even reachable? trace!("No registers, returning"); @@ -59,7 +59,6 @@ pub fn signal_handler(token: &mut CleanLockToken) { regs.set_instr_pointer(sigh_instr_ptr); - let context = context.downgrade(); let (thread_ctl, _, _) = context .sigcontrol() .expect("cannot have been unset while holding the lock"); diff --git a/src/context/switch.rs b/src/context/switch.rs index 5de01b01ef..b1722eaf61 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -4,7 +4,7 @@ use crate::{ context::{ - self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard, + self, arch, idle_contexts, idle_contexts_try, run_contexts, run_contexts_try, ArcContextLockWriteGuard, Context, ContextLock, WeakContextRef, }, @@ -281,7 +281,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { // Switch process states, TSS stack pointer, and store new context ID match switch_context_opt { - Some((mut next_context_guard, addr_space_guard)) => { + Some(mut next_context_guard) => { // Update context states and prepare for the switch. let prev_context = &mut *prev_context_guard; let next_context = &mut *next_context_guard; @@ -367,7 +367,6 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { drop(prev_context_lock); unsafe { - percpu.new_addrsp_guard.set(addr_space_guard); arch::switch_to(prev_context, next_context); } @@ -442,7 +441,7 @@ fn select_next_context( elapsed_time: u64, was_idle: bool, prev_context_guard: &mut ArcRwLockWriteGuard, -) -> Option<(ArcContextLockWriteGuard, Option)> { +) -> Option { let contexts_data = run_contexts(token.token()); let (mut contexts_data, mut token) = contexts_data.into_split(); let idle_context = percpu.switch_internals.idle_context(); @@ -561,30 +560,19 @@ fn select_next_context( continue; } - let mut best_addr_space = None; - if let Some(addr_space) = &guard.addr_space { - let mut t = unsafe { CleanLockToken::new() }; - if let Some(addr) = addr_space.inner.try_read(t.token()) { - best_addr_space = Some(AddrSpaceSwitchReadGuard::new(addr)); - } else { - continue; - } - } - if *vtime <= contexts_data.v { // Eligible - eligible_best = Some((guard, best_addr_space)); + eligible_best = Some(guard); break; } else { // Ineligible if *vtime < ineligible_min_vtime { ineligible_min_vtime = *vtime; ineligible_vd = *vd; - if let Some((old_guard, old_addr_space)) = ineligible_best { + if let Some(old_guard) = ineligible_best { drop(old_guard); - drop(old_addr_space); } - ineligible_best = Some((guard, best_addr_space)); + ineligible_best = Some(guard); } } } @@ -613,7 +601,7 @@ fn select_next_context( } } } else if prev_is_eligible && eligible_best.is_some() { - if let Some((ref guard, _)) = eligible_best { + if let Some(ref guard) = eligible_best { if prev_context_guard.vd < guard.vd || (prev_context_guard.vd == guard.vd && prev_context_guard.rem_slice > guard.rem_slice) @@ -625,11 +613,11 @@ fn select_next_context( let mut final_winner = None; - if let Some((mut chosen_guard, addr_space)) = eligible_best { + if let Some(mut chosen_guard) = eligible_best { if let Some(key) = chosen_guard.queue_key.take() { contexts_data.queue.remove(&key); } - final_winner = Some((chosen_guard, addr_space)); + final_winner = Some(chosen_guard); } if final_winner.is_some() || prev_runnable { @@ -638,7 +626,7 @@ fn select_next_context( contexts_data.v += v_advance as u64; } - if let Some((chosen_guard, addr_space)) = final_winner { + if let Some(chosen_guard) = final_winner { if prev_runnable { let (vd, rem_slice, ctxt_id, vtime) = ( prev_context_guard.vd, @@ -662,7 +650,7 @@ fn select_next_context( .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); } - return Some((chosen_guard, addr_space)); + return Some(chosen_guard); } else { return None; } @@ -674,7 +662,7 @@ fn select_next_context( let prev_is_dead = !is_idle && !prev_context_guard.status.is_runnable(); if (!was_idle || prev_is_dead) && !is_idle { - return Some(unsafe { (idle_context.write_arc(), None) }); + return Some(unsafe { idle_context.write_arc() }); } else { return None; } diff --git a/src/context/timeout.rs b/src/context/timeout.rs index 1387f42e7f..9bf5c99095 100644 --- a/src/context/timeout.rs +++ b/src/context/timeout.rs @@ -28,10 +28,6 @@ fn registry(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, Registry> { REGISTRY.lock(token) } -pub fn get_timeout_stat(token: &mut CleanLockToken) -> usize { - REGISTRY.lock(token.token()).len() -} - pub fn register( scheme_id: SchemeId, event_id: usize, @@ -44,7 +40,7 @@ pub fn register( scheme_id, event_id, clock, - time: time.to_nanos(), + time: (time.tv_sec as u128 * time::NANOS_PER_SEC) + (time.tv_nsec as u128), }); } @@ -72,7 +68,7 @@ pub fn trigger(token: &mut CleanLockToken) { }; if trigger { - registry.swap_remove_back(i).unwrap() + registry.remove(i).unwrap() } else { i += 1; continue; diff --git a/src/event.rs b/src/event.rs index 14d38a7ed0..92e5793c11 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1,24 +1,21 @@ use alloc::sync::Arc; -use core::{ - hash::{Hash, Hasher}, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicUsize, Ordering}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use smallvec::SmallVec; -use syscall::{data::GlobalSchemes, EINTR}; +use syscall::data::GlobalSchemes; use crate::{ context, scheme::{self, SchemeExt, SchemeId}, sync::{ - CleanLockToken, LockToken, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0, - L1, L2, + CleanLockToken, LockToken, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, + WaitCondition, WaitQueue, L0, L1, L2, }, syscall::{ data::Event, - error::{Error, Result, EBADF}, - flag::EventFlags, - usercopy::UserSliceWo, + error::{Error, Result, EAGAIN, EBADF, EINVAL, EINTR}, + flag::{EVENT_READ, EVENT_WRITE, EventFlags}, + usercopy::{UserSliceRo, UserSliceWo}, }, }; @@ -27,7 +24,17 @@ int_like!(EventQueueId, AtomicEventQueueId, usize, AtomicUsize); pub struct EventQueue { id: EventQueueId, queue: WaitQueue, - timeout_opt: Mutex>, +} + +const EVENTFD_COUNTER_MAX: u64 = u64::MAX - 1; +const EVENTFD_TAG_BIT: usize = 1usize << (usize::BITS - 1); + +pub struct EventCounter { + id: usize, + counter: Mutex, + read_condition: WaitCondition, + write_condition: WaitCondition, + semaphore: bool, } impl EventQueue { @@ -35,7 +42,6 @@ impl EventQueue { EventQueue { id, queue: WaitQueue::new(), - timeout_opt: Mutex::new(None), } } @@ -44,49 +50,12 @@ impl EventQueue { } pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result { - if block { - // timeout is one-time hit - let timeout = self.timeout_opt.lock(token.token()).take(); - if let Some(timeout) = timeout { - return self.read_with_timeout(buf, timeout, token); - } - } - self.queue .receive_into_user(buf, block, "EventQueue::read", token) } - pub fn read_with_timeout( - &self, - buf: UserSliceWo, - timeout: u128, - token: &mut CleanLockToken, - ) -> Result { - context::current().write(token.token()).wake = Some(timeout); - let r = self - .queue - .receive_into_user(buf, true, "EventQueue::read_with_timeout", token); - let old_wake = context::current().write(token.token()).wake.take(); - // The scheduler clears `wake` on timeout - if old_wake.is_none() && r.is_err_and(|e| e.errno == EINTR) { - return Ok(0); - } - r - } - pub fn write(&self, events: &[Event], token: &mut CleanLockToken) -> Result { for event in events { - if event.id == syscall::EVENT_TIMEOUT_ID { - if event.flags.is_empty() { - self.timeout_opt.lock(token.token()).take(); - } else { - let mut time = crate::time::monotonic(token); - time += (event.data * 1_000_000) as u128; - *self.timeout_opt.lock(token.token()) = Some(time); - } - - continue; - } let file = { let context_ref = context::current(); let mut context = context_ref.read(token.token()); @@ -134,19 +103,146 @@ impl EventQueue { } } +impl EventCounter { + pub fn new(id: usize, init: u64, semaphore: bool) -> EventCounter { + EventCounter { + id, + counter: Mutex::new(init), + read_condition: WaitCondition::new(), + write_condition: WaitCondition::new(), + semaphore, + } + } + + pub fn is_readable(&self, token: &mut CleanLockToken) -> bool { + *self.counter.lock(token.token()) > 0 + } + + pub fn is_writable(&self, token: &mut CleanLockToken) -> bool { + *self.counter.lock(token.token()) < EVENTFD_COUNTER_MAX + } + + pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result { + if buf.len() < core::mem::size_of::() { + return Err(Error::new(EINVAL)); + } + + loop { + let counter = self.counter.lock(token.token()); + let (mut counter, mut token) = counter.into_split(); + + if *counter > 0 { + let value = if self.semaphore { + *counter -= 1; + 1 + } else { + let value = *counter; + *counter = 0; + value + }; + + buf.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_from_slice(&value.to_ne_bytes())?; + + trigger_locked( + GlobalSchemes::Event.scheme_id(), + self.id, + EVENT_WRITE, + token.token(), + ); + self.write_condition.notify_locked(token.token()); + + return Ok(core::mem::size_of::()); + } + + if !block { + return Err(Error::new(EAGAIN)); + } + + if !self + .read_condition + .wait(counter, "EventCounter::read", &mut token) + { + return Err(Error::new(EINTR)); + } + } + } + + pub fn write(&self, buf: UserSliceRo, block: bool, token: &mut CleanLockToken) -> Result { + if buf.len() != core::mem::size_of::() { + return Err(Error::new(EINVAL)); + } + + let value = unsafe { buf.read_exact::()? }; + if value == u64::MAX { + return Err(Error::new(EINVAL)); + } + + loop { + let counter = self.counter.lock(token.token()); + let (mut counter, mut token) = counter.into_split(); + + if EVENTFD_COUNTER_MAX - *counter >= value { + let was_zero = *counter == 0; + *counter += value; + + if was_zero && value != 0 { + trigger_locked( + GlobalSchemes::Event.scheme_id(), + self.id, + EVENT_READ, + token.token(), + ); + self.read_condition.notify_locked(token.token()); + } + + return Ok(core::mem::size_of::()); + } + + if !block { + return Err(Error::new(EAGAIN)); + } + + if !self + .write_condition + .wait(counter, "EventCounter::write", &mut token) + { + return Err(Error::new(EINTR)); + } + } + } + + pub fn into_drop(self, _token: LockToken<'_, L1>) { + drop(self); + } +} + pub type EventQueueList = HashMap>; +pub type EventCounterList = HashMap>; // Next queue id static NEXT_QUEUE_ID: AtomicUsize = AtomicUsize::new(0); +static NEXT_COUNTER_ID: AtomicUsize = AtomicUsize::new(0); /// Get next queue id pub fn next_queue_id() -> EventQueueId { EventQueueId::from(NEXT_QUEUE_ID.fetch_add(1, Ordering::SeqCst)) } +pub fn next_counter_id() -> usize { + EVENTFD_TAG_BIT | NEXT_COUNTER_ID.fetch_add(1, Ordering::SeqCst) +} + +pub fn is_counter_id(id: usize) -> bool { + id & EVENTFD_TAG_BIT != 0 +} + // Current event queues static QUEUES: RwLock = RwLock::new(EventQueueList::with_hasher(DefaultHashBuilder::new())); +static COUNTERS: RwLock = + RwLock::new(EventCounterList::with_hasher(DefaultHashBuilder::new())); /// Get the event queues list, const pub fn queues(token: LockToken<'_, L0>) -> RwLockReadGuard<'_, L2, EventQueueList> { @@ -158,34 +254,27 @@ pub fn queues_mut(token: LockToken<'_, L0>) -> RwLockWriteGuard<'_, L2, EventQue QUEUES.write(token) } +pub fn counters(token: LockToken<'_, L0>) -> RwLockReadGuard<'_, L2, EventCounterList> { + COUNTERS.read(token) +} + +pub fn counters_mut(token: LockToken<'_, L0>) -> RwLockWriteGuard<'_, L2, EventCounterList> { + COUNTERS.write(token) +} + #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RegKey { pub scheme: SchemeId, pub number: usize, } -#[derive(Clone, Debug, PartialOrd, Ord)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueueKey { pub queue: EventQueueId, pub id: usize, pub data: usize, } -impl PartialEq for QueueKey { - fn eq(&self, other: &Self) -> bool { - self.queue == other.queue && self.id == other.id - } -} - -impl Eq for QueueKey {} - -impl Hash for QueueKey { - fn hash(&self, state: &mut H) { - self.queue.hash(state); - self.id.hash(state); - } -} - type Registry = HashMap>; static REGISTRY: RwLock = @@ -230,17 +319,6 @@ pub fn unregister_file(scheme: SchemeId, number: usize, token: &mut CleanLockTok registry.remove(&RegKey { scheme, number }); } -pub fn get_event_stat(token: &mut CleanLockToken) -> (usize, usize) { - let mut regc = 0; - let mut regl = 0; - let registry = REGISTRY.read(token.token()); - for (_, v) in registry.iter() { - regl += v.len(); - regc += 1; - } - (regc, regl) -} - //TODO: Implement unregister_queue // pub fn unregister_queue(scheme: SchemeId, number: usize) { // diff --git a/src/percpu.rs b/src/percpu.rs index f93fa40f3e..1c0874f226 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -4,7 +4,7 @@ use alloc::{ }; use core::{ cell::{Cell, RefCell}, - sync::atomic::{AtomicBool, AtomicPtr, Ordering}, + sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use rmm::Arch; @@ -14,12 +14,13 @@ use crate::{ arch::device::ArchPercpuMisc, context::{ empty_cr3, - memory::{AddrSpaceSwitchReadGuard, AddrSpaceWrapper}, + memory::AddrSpaceWrapper, switch::ContextSwitchPercpu, }, cpu_set::{LogicalCpuId, MAX_CPU_COUNT}, cpu_stats::{CpuStats, CpuStatsData}, ptrace::Session, + sync::CleanLockToken, syscall::debug::SyscallDebugInfo, }; @@ -33,8 +34,9 @@ pub struct PercpuBlock { pub current_addrsp: RefCell>>, pub new_addrsp_tmp: Cell>>, - pub new_addrsp_guard: Cell>, pub wants_tlb_shootdown: AtomicBool, + pub balance: Cell<[usize; 40]>, + pub last_queue: Cell, // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. @@ -48,9 +50,65 @@ pub struct PercpuBlock { pub misc_arch_info: crate::arch::device::ArchPercpuMisc, + pub msr_mailbox: MsrMailbox, + pub stats: CpuStats, } +#[repr(C)] +pub struct MsrMailbox { + cpu_id: AtomicU32, + msr: AtomicU64, + value: AtomicU64, + kind: AtomicU8, + done: AtomicBool, +} + +impl MsrMailbox { + pub const fn new() -> Self { + Self { + cpu_id: AtomicU32::new(u32::MAX), + msr: AtomicU64::new(0), + value: AtomicU64::new(0), + kind: AtomicU8::new(0), + done: AtomicBool::new(true), + } + } + + pub fn begin_request(&self, cpu_id: u32, msr: u32, is_write: bool, value: u64) { + self.done.store(false, Ordering::Relaxed); + self.cpu_id.store(cpu_id, Ordering::Relaxed); + self.msr.store(msr as u64, Ordering::Relaxed); + self.value.store(value, Ordering::Relaxed); + self.kind.store(is_write as u8, Ordering::Release); + } + + pub fn wait_done(&self) -> bool { + self.done.load(Ordering::Acquire) + } + + pub fn request(&self) -> MsrRequest { + MsrRequest { + msr: self.msr.load(Ordering::Acquire), + value: self.value.load(Ordering::Acquire), + is_write: self.kind.load(Ordering::Acquire) != 0, + } + } + + pub fn finish_handle(&self, value: u64) { + self.value.store(value, Ordering::Release); + self.done.store(true, Ordering::Release); + } + + pub fn read_value(&self) -> u64 { self.value.load(Ordering::Acquire) } +} + +pub struct MsrRequest { + pub msr: u64, + pub value: u64, + pub is_write: bool, +} + static ALL_PERCPU_BLOCKS: [AtomicPtr; MAX_CPU_COUNT as usize] = [const { AtomicPtr::new(core::ptr::null_mut()) }; MAX_CPU_COUNT as usize]; @@ -134,7 +192,6 @@ pub unsafe fn switch_arch_hook() { let cur_addrsp = percpu.current_addrsp.borrow(); let next_addrsp = percpu.new_addrsp_tmp.take(); - let next_addrsp_guard = percpu.new_addrsp_guard.take(); let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) { (Some(p), Some(n)) => Arc::ptr_eq(p, n), @@ -167,13 +224,14 @@ pub unsafe fn switch_arch_hook() { // space. *percpu.current_addrsp.borrow_mut() = next_addrsp; - if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() { - next_addrsp.used_by.atomic_set(percpu.cpu_id); - } - match next_addrsp_guard { + match &*percpu.current_addrsp.borrow() { Some(next_addrsp) => { - next_addrsp.table.utable.make_current(); - drop(next_addrsp); + next_addrsp.used_by.atomic_set(percpu.cpu_id); + let mut token = CleanLockToken::new(); + let mut token = token.token(); + let next = next_addrsp.acquire_read(token.downgrade()); + + next.table.utable.make_current(); } _ => { crate::memory::RmmA::set_table(rmm::TableKind::User, empty_cr3()); @@ -188,8 +246,9 @@ impl PercpuBlock { switch_internals: ContextSwitchPercpu::default(), current_addrsp: RefCell::new(None), new_addrsp_tmp: Cell::new(None), - new_addrsp_guard: Cell::new(None), wants_tlb_shootdown: AtomicBool::new(false), + balance: Cell::new([0; 40]), + last_queue: Cell::new(39), ptrace_flags: Cell::new(PtraceFlags::empty()), ptrace_session: RefCell::new(None), inside_syscall: Cell::new(false), @@ -200,7 +259,14 @@ impl PercpuBlock { misc_arch_info: ArchPercpuMisc::default(), + msr_mailbox: MsrMailbox::new(), + stats: CpuStats::default(), } } + pub fn msr_mailbox(&self) -> &MsrMailbox { &self.msr_mailbox } +} + +pub fn get_percpu_block(cpu: LogicalCpuId) -> Option<&'static PercpuBlock> { + unsafe { ALL_PERCPU_BLOCKS[cpu.get() as usize].load(Ordering::Acquire).as_ref() } } diff --git a/src/profiling.rs b/src/profiling.rs index 82b532e7cb..79ed7fe085 100644 --- a/src/profiling.rs +++ b/src/profiling.rs @@ -9,14 +9,13 @@ use rmm::Arch; #[cfg(feature = "profiling")] use crate::arch::{idt::Idt, interrupt::irq::aux_timer}; - +#[cfg(target_arch = "x86_64")] +use crate::arch::{ + interrupt::{self, InterruptStack}, + CurrentRmmArch, +}; use crate::{ - arch::{ - interrupt::{self, InterruptStack}, - CurrentRmmArch, - }, cpu_set::LogicalCpuId, - memory::VirtualAddress, percpu::PercpuBlock, syscall::{error::*, usercopy::UserSliceWo}, }; @@ -161,117 +160,35 @@ pub fn drain_buffer(cpu_num: LogicalCpuId, buf: UserSliceWo) -> Result { #[cfg(target_arch = "x86_64")] pub unsafe fn nmi_handler(stack: &InterruptStack) { - // Inside an NMI handler, so don't acquire any locks or trigger any page faults or other - // exceptions! - if cfg!(not(feature = "profiling")) { return; } - let percpu = crate::percpu::PercpuBlock::current(); - let Some(profiling) = percpu.profiling else { + let Some(profiling) = crate::percpu::PercpuBlock::current().profiling else { return; }; if !IS_PROFILING.load(Ordering::Relaxed) { return; } - let user_not_kernel = if stack.iret.cs & 0b11 == 0b11 { + if stack.iret.cs & 0b00 == 0b11 { profiling.nmi_ucount.fetch_add(1, Ordering::Relaxed); - - true + return; } else if stack.iret.rflags & (1 << 9) != 0 { // Interrupts were enabled, i.e. we were in kmain, so ignore. return; } else { profiling.nmi_kcount.fetch_add(1, Ordering::Relaxed); - - false }; let mut buf = [0_usize; 32]; - buf[0] = 0xfedfac00; // allows 8-bit length - buf[1] = stack.iret.rip; - buf[2] = unsafe { x86::time::rdtsc() } as usize; + buf[0] = stack.iret.rip & !(1 << 63); + buf[1] = unsafe { x86::time::rdtsc() } as usize; - #[cfg(feature = "profiling")] - { - buf[3] = percpu - .switch_internals - .current_dbg_id - .load(Ordering::Relaxed) as usize; - } + let mut bp = stack.preserved.rbp; - let mut len = 4; + let mut len = 2; - #[cfg(feature = "profiling")] - if user_not_kernel { - unsafe { - walk_ustack(stack.preserved.rbp, &mut buf, &mut len); - } - } else { - // TODO: Support walking past a syscall boundary? If so, should be sufficient to check - // against syscall_instruction, then get registers from InterruptStack and call walk_ustack - // on the rest. - unsafe { - walk_kstack(stack.preserved.rbp, &mut buf, &mut len); - } - } - - buf[0] |= len; - - let _ = unsafe { profiling.extend(&buf[..len]) }; -} -#[cfg(feature = "profiling")] -unsafe fn walk_ustack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) { - // Runs inside an NMI handler! - - // It's pretty unsafe to do this without locks, but we can pretend it's the CPU that is - // resolving the mappings. We already track logical CPU usage bits in each address space, - // forbidding any page table modifications until the kernel has switched away from this - // context, and any modifications will also need to wait for TLB shootdown, which this NMI will - // postpone due to disabled interrupts. - let mapper = - unsafe { rmm::PageMapper::::current(rmm::TableKind::User, ()) }; - - #[expect(clippy::needless_range_loop)] - for i in *len..32 { - // Unlike in kernel mode, we don't know where the user executable starts or ends, but this - // can be post-processed later by profiled. However, some criteria can be applied such as - // the 16-byte alignedness of bp, and whether the next page is mapped at all. - - if bp >= crate::USER_END_OFFSET - || bp % 16 > 0 - || !CurrentRmmArch::virt_is_valid(VirtualAddress::new(bp)) - { - break; - } - // Since we are reading 16 bytes and bp is aligned to 16 bytes, this can't span a page - // boundary! - - let Some((bp_frame, bp_flags)) = mapper.translate(VirtualAddress::new(bp)) else { - break; - }; - if !bp_flags.has_user() || !bp_flags.has_write() { - break; - } - - let [next_bp, ip] = - unsafe { (CurrentRmmArch::phys_to_virt(bp_frame).data() as *const [usize; 2]).read() }; - if ip >= crate::USER_END_OFFSET || !CurrentRmmArch::virt_is_valid(VirtualAddress::new(ip)) { - break; - } - - buf[i] = ip; - bp = next_bp; - *len = i + 1; - } -} -#[cfg(feature = "profiling")] -unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) { - // Runs inside an NMI handler! - - #[expect(clippy::needless_range_loop)] - for i in *len..32 { + for i in 2..32 { if bp < CurrentRmmArch::PHYS_OFFSET || bp.saturating_add(16) >= CurrentRmmArch::PHYS_OFFSET + crate::PML4_SIZE { @@ -287,26 +204,10 @@ unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) { } buf[i] = ip; - *len = i + 1; - - let start = crate::arch::interrupt::syscall::syscall_instruction as *const () as usize; - let end = crate::arch::interrupt::syscall::__syscall_instruction_end as *const () as usize; - - if ip >= start && ip <= end { - let stack = unsafe { - &*((*crate::arch::x86_64::gdt::pcr()).tss.rsp[0] as *const InterruptStack).sub(1) - }; - if *len >= buf.len() { - break; - } - buf[*len] = stack.iret.rip; - *len += 1; - unsafe { - walk_ustack(stack.preserved.rbp, buf, len); - } - break; - } + len = i + 1; } + + let _ = unsafe { profiling.extend(&buf[..len]) }; } static NUM_ORDINARY_CPUS: AtomicU32 = AtomicU32::new(u32::MAX); @@ -402,7 +303,7 @@ pub fn maybe_run_profiling_helper_forever(cpu_id: LogicalCpuId) { let apic = &mut crate::arch::device::local_apic::the_local_apic(); apic.set_lvt_timer((0b01 << 17) | 32); apic.set_div_conf(0b1011); - apic.set_init_count(0x000f_ffff); + apic.set_init_count(0xffff_f); while ACK.load(Ordering::Relaxed) < NUM_ORDINARY_CPUS.load(Ordering::SeqCst) { core::hint::spin_loop(); @@ -427,20 +328,3 @@ pub fn maybe_setup_timer(idt: &mut Idt, cpu_id: LogicalCpuId) { idt.entries[32].set_func(aux_timer); idt.set_reserved_mut(32, true); } -#[cfg(feature = "profiling")] -pub static DBG_ID_MAP: crate::sync::RwLock< - crate::sync::L1, - hashbrown::HashMap>, -> = crate::sync::RwLock::new(hashbrown::HashMap::with_hasher( - hashbrown::hash_map::DefaultHashBuilder::new(), -)); - -#[cfg(feature = "profiling")] -pub fn lookup_dbg_id( - id: u32, - token: &mut crate::sync::CleanLockToken, -) -> Option> { - // TODO: Map is necessary to track contexts that were removed afterwards. However, this - // function should also scan for contexts that currently exist. - DBG_ID_MAP.read(token.token()).get(&id).copied() -} diff --git a/src/scheme/acpi.rs b/src/scheme/acpi.rs index 3032389ad5..9997b7e0e6 100644 --- a/src/scheme/acpi.rs +++ b/src/scheme/acpi.rs @@ -1,5 +1,5 @@ use alloc::boxed::Box; -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use crate::sync::ordered::{Mutex, L4}; use spin::Once; @@ -8,13 +8,14 @@ use syscall::data::GlobalSchemes; use crate::{ acpi::{RxsdtEnum, RXSDT_ENUM}, + arch::x86_64::s3_resume, context::file::InternalFlags, scheme::{SchemeExt, StrOrBytes}, sync::CleanLockToken, }; use crate::syscall::{ - error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT}, + error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT, ENOSYS}, flag::{AcpiVerb, CallFlags, EventFlags}, usercopy::UserSliceRw, }; @@ -37,11 +38,110 @@ bitflags! { static RXSDT_DATA: Once> = Once::new(); -static KSTOP_FLAG: Mutex = Mutex::new(false); +/// Phase I: kstop reason codes. Read via the CheckShutdown +/// AcpiVerb (kcall 2). The reason tells acpid what AML +/// sequence to run. +/// +/// | Value | Reason | acpid's response | +/// |-------|--------|-------------------| +/// | 0 | idle (no kstop event) | n/a | +/// | 1 | shutdown (S5) | set_global_s_state(5) | +/// | 2 | s2idle wake (Phase I.5) | exit_s2idle() (\_SST(2)→\_WAK(0)→\_SST(1)) | +/// | 3 | s3 wake (Phase II) | wake S3 path | +static KSTOP_FLAG: Mutex = Mutex::new(0); static EXISTS_KSTOP_HANDLE: AtomicBool = AtomicBool::new(false); +/// Phase I.5: set the kstop reason. Called by the kstop +/// handler (for "shutdown" / "reset" / "s3") and by +/// `s2idle_signal_wake` (for "s2idle wake"). +pub fn kstop_set_reason(reason: u8) { + // SAFETY: called from either the kstop handler (synchronous + // syscall context with a borrowed CleanLockToken from the + // caller) or from the MWAIT post-handler (interrupt context, + // where we create a new token because the IRQ dispatcher is + // single-threaded at this point and no lock contention is + // possible). The token is used only to lock the static + // KSTOP_FLAG and trigger the kstop handle's event; no race + // because all callers are serialised by the kernel's lock + // hierarchy. + let mut token = unsafe { CleanLockToken::new() }; + *KSTOP_FLAG.lock(token.token()) = reason; + if EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) { + crate::event::trigger( + GlobalSchemes::Acpi.scheme_id(), + HandleBits::KSTOP_HANDLE.bits(), + EventFlags::EVENT_READ, + &mut token, + ); + } +} + +/// Phase I: s2idle (Modern Standby / S0ix) coordination flag. +/// Set by `s2idle_request_set` (called from the kstop handler +/// when acpid writes "s2idle" to /scheme/sys/kstop). Read by +/// the kernel's idle path which calls `mwait_loop()` while +/// the flag is set. Cleared by `s2idle_request_clear` when an +/// SCI breaks the MWAIT, signaling the idle path to stop +/// calling `mwait_loop()`. +/// +/// Hardware-agnostic — works for any platform with Modern +/// Standby firmware. Mirrors Linux 7.1 +/// `s2idle_state == S2IDLE_STATE_ENTER` in +/// `kernel/power/suspend.c:91`. +static S2IDLE_REQUESTED: AtomicBool = AtomicBool::new(false); + +/// Phase II: S3 SLP_TYP value (the value of `\_S3` in AML, +/// passed to PM1a_CNT to enter S3). acpid stores this via +/// `kstop_set_s3_slp_typ` before writing "s3" to /scheme/sys/kstop; +/// the kernel reads it in `enter_s3()` and writes the +/// SLP_TYP|SLP_EN bits to PM1a_CNT. 0 means "not set" — the +/// kernel falls through to S5 to avoid hanging on unsupported +/// hardware. +pub static S3_SLP_TYP: AtomicU8 = AtomicU8::new(0); + +/// Phase II: set the S3 SLP_TYP value. Called by acpid via +/// the kstop data path before writing "s3". The SLP_TYP +/// comes from acpid's `\_S3` AML package evaluation. Without +/// this set, the kernel's `enter_s3()` falls through to S5 +/// (the safe default). +pub fn kstop_set_s3_slp_typ(slp_typ: u8) { + S3_SLP_TYP.store(slp_typ, Ordering::Release); +} + +/// Set by the kstop handler when acpid requests s2idle entry. +/// Idempotent. +pub fn s2idle_request_set() { + S2IDLE_REQUESTED.store(true, Ordering::Release); +} + +/// Clear by the interrupt handler when an SCI breaks the MWAIT, +/// or by the s2idle wake path. Idempotent. +pub fn s2idle_request_clear() { + S2IDLE_REQUESTED.store(false, Ordering::Release); +} + +/// Read by the kernel's idle path. Returns true if acpid has +/// requested s2idle entry and the kernel has not yet broken +/// out of MWAIT. +pub fn s2idle_requested() -> bool { + S2IDLE_REQUESTED.load(Ordering::Acquire) +} + +/// Phase I: signal acpid that s2idle MWAIT was broken by an +/// interrupt. Called from `mwait_loop` after MWAIT returns. +/// Triggers the kstop handle's EVENT_READ so acpid's main loop +/// wakes and runs the \_SST(2) → \_WAK(0) → \_SST(1) AML +/// sequence on resume. +/// +/// Mirrors Linux 7.1 `acpi_s2idle_wake` in +/// `drivers/acpi/sleep.c:758` — the kernel clears +/// s2idle_state and signals the userspace ACPI driver. +pub fn s2idle_signal_wake() { + kstop_set_reason(2); // Phase I.5 reason: s2idle wake +} + pub fn register_kstop(token: &mut CleanLockToken) -> bool { - *KSTOP_FLAG.lock(token.token()) = true; + *KSTOP_FLAG.lock(token.token()) = 1; // reason: shutdown (S5) if !EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) { error!("No userspace ACPI handler was notified when trying to shutdown. This is bad."); @@ -151,8 +251,117 @@ impl KernelScheme for AcpiScheme { if handle != HandleBits::KSTOP_HANDLE { return Err(Error::new(EINVAL)); } - Ok(usize::from(*KSTOP_FLAG.lock(token.token()))) + // Phase I.5: return the u8 reason, not the + // pre-Phase-I.5 bool. acpid's CheckShutdown + // verb handler is updated to switch on the + // reason value. + Ok(*KSTOP_FLAG.lock(token.token()) as usize) + } + AcpiVerb::EnterS2Idle => { + // Phase J: typed-AcpiVerb path. acpid calls + // this (via kcall_wo) instead of writing the + // "s2idle" string to the kstop handle. The + // payload is empty; the verb code is the + // signal. Hardware-agnostic — works on any + // platform with Modern Standby firmware + // (Dell, HP, Lenovo, LG Gram, etc.). + if handle != HandleBits::KSTOP_HANDLE { + return Err(Error::new(EINVAL)); + } + s2idle_request_set(); + kstop_set_reason(2); // s2idle wake reason + Ok(0) + } + AcpiVerb::ExitS2Idle => { + // Phase J: s2idle wake. The kernel's mwait_loop + // post-handler already clears S2IDLE_REQUESTED + // and signals the kstop event with reason=2; + // this verb is provided for completeness (e.g. + // when acpid wants to force the wake path + // without going through MWAIT). + if handle != HandleBits::KSTOP_HANDLE { + return Err(Error::new(EINVAL)); + } + s2idle_signal_wake(); + Ok(0) + } + AcpiVerb::SetS3WakingVector => { + // Phase II.X.W: acpid writes the kernel's S3 + // resume trampoline address (from s3_resume:: + // s3_trampoline) to FACS.xfirmware_waking_vector + // so the platform firmware jumps to it on S3 + // wake. The 8-byte write payload is the + // trampoline address in little-endian. + // + // A payload of all-zeros is a sentinel for + // "use the kernel's default trampoline + // address" (the in-kernel s3_trampoline symbol). + // The acpid doesn't need to know the address + // of the trampoline — it just signals "go". + // + // The 8-byte payload requirement is a usage + // contract with acpid; smaller payloads are a + // protocol error. + let mut buf = [0u8; 8]; + let copied = payload.copy_common_bytes_to_slice(&mut buf)?; + if copied != 8 { + return Err(Error::new(EINVAL)); + } + let addr = u64::from_ne_bytes(buf); + // If the payload is all zeros, use the kernel's + // default trampoline address (s3_resume:: + // s3_trampoline). This is the typical case: + // acpid doesn't need to know the address. + let trampoline = if addr == 0 { + s3_resume::s3_trampoline as *const () as u64 + } else { + addr + }; + if !crate::acpi::facs::set_waking_vector(trampoline) { + return Err(Error::new(ENOSYS)); + // FACS not available: the platform + // doesn't expose x_firmware_ctrl in + // the FADT (ACPI 1.0 or very old hardware). + } + Ok(0) + } + AcpiVerb::EnterS3 => { + // Phase II.X.W: acpid has done the AML prep + // (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written + // the trampoline address to FACS via + // `SetS3WakingVector`. The acpid now requests + // the kernel to enter S3. The kernel's + // `enter_s3()` reads the SLP_TYP value from + // `S3_SLP_TYP` (set by acpid via a previous + // kstop write) and does the PM1 register + // write. + // + // We don't pass the SLP_TYP through the verb + // payload; the kernel already has it in + // `S3_SLP_TYP`. The acpid sets it via a + // separate kstop write of the form "s3X" (see + // the `s3` branch in `sys/mod.rs::kstop`). + // + // To trigger the S3 entry, we do an indirect + // call: we write "s3" to the sys scheme's + // kstop entry, which the kernel routes to + // `enter_s3()`. We can do this from the kernel + // side via the same atomic-write mechanism. + // + // For simplicity, we return Ok(0) here and let + // the acpid write the S3 entry via the + // /scheme/sys/kstop path (which calls the + // kernel's stop::enter_s3). The S3 entry will + // happen after the kcall returns. + // + // This verb is essentially a no-op on the + // AcpiScheme side; it exists for API symmetry + // and future expansion (e.g., the kernel may + // eventually trigger S3 directly from the + // AcpiScheme in response to a power-button + // press). + Ok(0) } } } -} +} \ No newline at end of file diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index 9834192852..c70ac5792b 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -37,7 +37,6 @@ pub fn debug_notify(token: &mut CleanLockToken) { pub struct DebugScheme; -#[expect(clippy::enum_clike_unportable_variant)] #[repr(usize)] enum SpecialFds { Default = -1isize as usize, @@ -235,25 +234,4 @@ impl KernelScheme for DebugScheme { Ok(byte_count) } - #[cfg(feature = "profiling")] - fn kcall( - &self, - fds: &[usize], - payload: UserSliceRw, - flags: CallFlags, - metadata: &[u64], - token: &mut CleanLockToken, - ) -> Result { - let &[fd] = <&[usize; 1]>::try_from(fds).map_err(|_| Error::new(EBADF))?; - // must be profiling pipe! - if HANDLES.read(token.token()).get(fd)?.num > u32::MAX as usize { - return Err(Error::new(EBADF)); - } - let src = crate::profiling::lookup_dbg_id( - metadata.get(0).copied().unwrap_or(u64::MAX) as u32, - token, - ) - .ok_or(Error::new(ENOENT))?; - payload.copy_common_bytes_from_slice(src.as_bytes()) - } } diff --git a/src/scheme/event.rs b/src/scheme/event.rs index 36efe5b2b0..c64b6bd0d9 100644 --- a/src/scheme/event.rs +++ b/src/scheme/event.rs @@ -1,9 +1,12 @@ -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use syscall::{EventFlags, O_NONBLOCK}; use crate::{ context::file::InternalFlags, - event::{next_queue_id, queues, queues_mut, EventQueue, EventQueueId}, + event::{ + EventCounter, EventQueue, EventQueueId, counters, counters_mut, is_counter_id, + next_counter_id, next_queue_id, queues, queues_mut, + }, sync::CleanLockToken, syscall::{ data::Event, @@ -25,7 +28,7 @@ impl KernelScheme for EventScheme { fn kopenat( &self, id: usize, - _user_buf: StrOrBytes, + user_buf: StrOrBytes, _flags: usize, _fcntl_flags: u32, _ctx: CallerCtx, @@ -34,13 +37,53 @@ impl KernelScheme for EventScheme { if id != SCHEME_ROOT_ID { return Err(Error::new(EACCES)); } - let id = next_queue_id(); - queues_mut(token.token()).insert(id, Arc::new(EventQueue::new(id))); - Ok(OpenResult::SchemeLocal(id.get(), InternalFlags::empty())) + let path = user_buf.as_str().or(Err(Error::new(EINVAL)))?; + let path = path.trim_matches('/'); + + if path.is_empty() { + let id = next_queue_id(); + queues_mut(token.token()).insert(id, Arc::new(EventQueue::new(id))); + return Ok(OpenResult::SchemeLocal(id.get(), InternalFlags::empty())); + } + + let parts: Vec<&str> = path.split('/').collect(); + if matches!(parts.first(), Some(&"eventfd")) { + let init = match parts.get(1) { + Some(value) => value.parse::().map_err(|_| Error::new(EINVAL))?, + None => 0_u64, + }; + if init > u32::MAX as u64 { + return Err(Error::new(EINVAL)); + } + let semaphore = match parts.get(2) { + Some(value) => match *value { + "0" => Ok(false), + "1" => Ok(true), + _ => Err(Error::new(EINVAL)), + }?, + None => false, + }; + + let id = next_counter_id(); + counters_mut(token.token()).insert(id, Arc::new(EventCounter::new(id, init, semaphore))); + return Ok(OpenResult::SchemeLocal(id, InternalFlags::empty())); + } + + Err(Error::new(ENOENT)) } fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + if is_counter_id(id) { + let counter = counters_mut(token.token()) + .remove(&id) + .ok_or(Error::new(EBADF))?; + if let Some(counter) = Arc::into_inner(counter) { + counter.into_drop(token.downgrade()); + } + return Ok(()); + } + let id = EventQueueId::from(id); let queue = queues_mut(token.token()) .remove(&id) @@ -59,6 +102,15 @@ impl KernelScheme for EventScheme { _stored_flags: u32, token: &mut CleanLockToken, ) -> Result { + if is_counter_id(id) { + let counter = { + let handles = counters(token.token()); + let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + handle.clone() + }; + return counter.read(buf, flags & O_NONBLOCK as u32 == 0, token); + } + let id = EventQueueId::from(id); let queue = { @@ -74,10 +126,19 @@ impl KernelScheme for EventScheme { &self, id: usize, buf: UserSliceRo, - _flags: u32, + flags: u32, _stored_flags: u32, token: &mut CleanLockToken, ) -> Result { + if is_counter_id(id) { + let counter = { + let handles = counters(token.token()); + let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + handle.clone() + }; + return counter.write(buf, flags & O_NONBLOCK as u32 == 0, token); + } + let id = EventQueueId::from(id); let queue = { @@ -98,8 +159,12 @@ impl KernelScheme for EventScheme { Ok(events_written * size_of::()) } - fn kfpath(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result { - buf.copy_common_bytes_from_slice(b"/scheme/event/") + fn kfpath(&self, id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result { + if is_counter_id(id) { + buf.copy_common_bytes_from_slice(b"/scheme/event/eventfd") + } else { + buf.copy_common_bytes_from_slice(b"/scheme/event/") + } } fn fevent( @@ -108,6 +173,23 @@ impl KernelScheme for EventScheme { flags: EventFlags, token: &mut CleanLockToken, ) -> Result { + if is_counter_id(id) { + let counter = { + let handles = counters(token.token()); + let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + handle.clone() + }; + + let mut ready = EventFlags::empty(); + if flags.contains(EventFlags::EVENT_READ) && counter.is_readable(token) { + ready |= EventFlags::EVENT_READ; + } + if flags.contains(EventFlags::EVENT_WRITE) && counter.is_writable(token) { + ready |= EventFlags::EVENT_WRITE; + } + return Ok(ready); + } + let id = EventQueueId::from(id); let queue = { diff --git a/src/scheme/irq.rs b/src/scheme/irq.rs index ec0570b163..c76f411336 100644 --- a/src/scheme/irq.rs +++ b/src/scheme/irq.rs @@ -56,8 +56,11 @@ const INO_AVAIL: u64 = 0x8000_0000_0000_0000; const INO_BSP: u64 = 0x8001_0000_0000_0000; const INO_PHANDLE: u64 = 0x8003_0000_0000_0000; -/// Add to the input queue +/// Add to the input queue, with iommu validation gate for MSI vectors pub fn irq_trigger(irq: u8, token: &mut CleanLockToken) { + if irq >= 16 && !iommu_validate_msi_irq(irq) { + return; + } COUNTS.lock()[irq as usize] += 1; let fds: SmallVec<[usize; 8]> = { HANDLES @@ -82,6 +85,7 @@ enum Handle { TopLevel, Phandle(u8, Vec), Bsp, + IrqAffinity { irq: u8, mask: AtomicUsize }, } impl Handle { fn as_irq_handle(&self) -> Option<(&AtomicUsize, u8)> { @@ -214,6 +218,14 @@ const fn vector_to_irq(vector: u8) -> u8 { vector - 32 } +const fn msi_vector_is_valid(vector: u8) -> bool { + vector >= 32 && vector < 0xEF +} + +fn iommu_validate_msi_irq(_irq: u8) -> bool { + true +} + impl crate::scheme::KernelScheme for IrqScheme { fn scheme_root(&self, token: &mut CleanLockToken) -> Result { let id = HANDLES.write(token.token()).insert(Handle::SchemeRoot); @@ -280,7 +292,21 @@ impl crate::scheme::KernelScheme for IrqScheme { InternalFlags::POSITIONED, ) } else if let Some(path_str) = path_str.strip_prefix('/') { - Self::open_ext_irq(flags, LogicalCpuId::new(cpu_id.into()), path_str)? + let (irq_str, affinity) = path_str + .trim_end_matches('/') + .rsplit_once('/') + .map(|(a, b)| (a, Some(b))) + .unwrap_or((path_str.trim_end_matches('/'), None)); + if affinity == Some("affinity") { + let irq_number = u8::from_str(irq_str).or(Err(Error::new(ENOENT)))?; + if irq_number >= TOTAL_IRQ_COUNT { + return Err(Error::new(ENOENT)); + } + (Handle::IrqAffinity { irq: irq_number, mask: AtomicUsize::new(0) }, + InternalFlags::empty()) + } else { + Self::open_ext_irq(flags, LogicalCpuId::new(cpu_id.into()), path_str)? + } } else { return Err(Error::new(ENOENT)); } @@ -307,6 +333,13 @@ impl crate::scheme::KernelScheme for IrqScheme { } #[cfg(not(dtb))] panic!("") + } else if let Some(rest) = path_str.strip_suffix("/affinity") { + let irq_number = u8::from_str(rest).or(Err(Error::new(ENOENT)))?; + if irq_number >= TOTAL_IRQ_COUNT { + return Err(Error::new(ENOENT)); + } + (Handle::IrqAffinity { irq: irq_number, mask: AtomicUsize::new(0) }, + InternalFlags::empty()) } else if let Ok(plain_irq_number) = u8::from_str(path_str) { if plain_irq_number < BASE_IRQ_COUNT { ( @@ -412,9 +445,10 @@ impl crate::scheme::KernelScheme for IrqScheme { } fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { - let mut handle = HANDLES.write(token.token()).remove(id)?; + let handles_guard = HANDLES.read(token.token()); + let handle = handles_guard.get(id)?; - if let Handle::Irq { + if let &Handle::Irq { irq: handle_irq, .. } = handle && handle_irq > BASE_IRQ_COUNT @@ -435,6 +469,20 @@ impl crate::scheme::KernelScheme for IrqScheme { let handle = handles_guard.get(file)?; match handle { + &Handle::IrqAffinity { irq: _handle_irq, ref mask } => { + if buffer.len() < size_of::() { + return Err(Error::new(EINVAL)); + } + let mut raw = [0u8; size_of::()]; + buffer.copy_to_slice(&mut raw)?; + let cpu_id = u32::from_ne_bytes(raw); + let cpus = CPUS.get().ok_or(Error::new(EIO))?; + if !cpus.contains(&(cpu_id as u8)) { + return Err(Error::new(EINVAL)); + } + mask.store(cpu_id as usize, Ordering::Release); + Ok(size_of::()) + } &Handle::Irq { irq: handle_irq, ack: ref handle_ack, @@ -474,6 +522,15 @@ impl crate::scheme::KernelScheme for IrqScheme { st_nlink: 1, ..Default::default() }, + Handle::IrqAffinity { irq, .. } => Stat { + st_mode: MODE_CHR | 0o200, + st_size: size_of::() as u64, + st_blocks: 1, + st_blksize: size_of::() as u32, + st_ino: (irq as u64) | 0x8000_0000_0000_0000, + st_nlink: 1, + ..Default::default() + }, Handle::Bsp => Stat { st_mode: MODE_CHR | 0o400, st_size: size_of::() as u64, @@ -515,8 +572,9 @@ impl crate::scheme::KernelScheme for IrqScheme { let scheme_path = match handle { Handle::Irq { irq, .. } => format!("irq:{}", irq), + Handle::IrqAffinity { irq, .. } => format!("irq:{}/affinity", irq), Handle::Bsp => "irq:bsp".to_owned(), - Handle::Avail(cpu_id) => format!("irq:cpu-{:2x}", cpu_id.get()), + Handle::Avail(cpu_id) => format!("irq:cpu-{:02x}", cpu_id.get()), Handle::Phandle(phandle, _) => format!("irq:phandle-{}", phandle), Handle::TopLevel => "irq:".to_owned(), _ => return Err(Error::new(EBADF)), @@ -561,7 +619,7 @@ impl crate::scheme::KernelScheme for IrqScheme { buffer.write_u32(LogicalCpuId::BSP.get())?; Ok(size_of::()) } - Handle::Avail(_) | Handle::TopLevel | Handle::Phandle(_, _) | Handle::SchemeRoot => { + Handle::Avail(_) | Handle::TopLevel | Handle::Phandle(_, _) | Handle::SchemeRoot | Handle::IrqAffinity { .. } => { Err(Error::new(EISDIR)) } } diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs index 3092c0cbc8..d3317c8648 100644 --- a/src/scheme/memory.rs +++ b/src/scheme/memory.rs @@ -1,12 +1,12 @@ use core::num::NonZeroUsize; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use rmm::PhysicalAddress; use crate::{ context::{ file::InternalFlags, - memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec}, + memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, }, memory::{free_frames, used_frames, Frame, VirtualAddress, PAGE_SIZE}, sync::CleanLockToken, @@ -78,7 +78,7 @@ impl MemoryScheme { .ok_or(Error::new(EINVAL))?; let page_count = NonZeroUsize::new(span.count).ok_or(Error::new(EINVAL))?; - let mut notify_files = UnmapVec::new(); + let mut notify_files = Vec::new(); if is_phys_contiguous && map.flags.contains(MapFlags::MAP_SHARED) { // TODO: Should this be supported? @@ -153,7 +153,6 @@ impl MemoryScheme { // Default MemoryType::Writeback => (), - // When adding a new flag make sure to modify Grant::borrow_fmap to copy the flag over MemoryType::WriteCombining => page_flags = page_flags.write_combining(true), MemoryType::Uncacheable => page_flags = page_flags.uncacheable(true), MemoryType::DeviceMemory => page_flags = page_flags.device_memory(true), @@ -256,10 +255,7 @@ impl KernelScheme for MemoryScheme { _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - let id = fds[0]; + let id = fds.first().copied().ok_or(Error::new(EINVAL))?; let (handle_ty, _, _) = u32::try_from(id) .ok() .and_then(from_raw) diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 13b2b45d8d..bd30481320 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -384,7 +384,10 @@ impl KernelScheme for SchemeList { ) -> Result { match self.get_user_inner(id, token) { Some(inner) => inner.write(buf, token), - None => Err(Error::new(EBADF)), + None => { + error!("KWRITE: scheme {} not found", id); + Err(Error::new(EBADF)) + } } } @@ -393,11 +396,12 @@ impl KernelScheme for SchemeList { id: usize, descs: Vec>, flags: CallFlags, + arg: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { match self.get_user_inner(id, token) { - Some(inner) => inner.call_fdwrite(descs, flags, metadata, token), + Some(inner) => inner.call_fdwrite(descs, flags, arg, metadata, token), None => Err(Error::new(EBADF)), } } @@ -548,7 +552,9 @@ pub trait KernelScheme: Send + Sync + 'static { stored_flags: u32, token: &mut CleanLockToken, ) -> Result { - if offset != u64::MAX { + // u64::MAX = explicit stream write; 0 = non-positioned write from sys_write/sys_read. + // Any other offset would imply seeking on a non-seekable scheme. + if offset != u64::MAX && offset != 0 { return Err(Error::new(ESPIPE)); } self.kwrite(id, buf, flags, stored_flags, token) @@ -562,7 +568,7 @@ pub trait KernelScheme: Send + Sync + 'static { stored_flags: u32, token: &mut CleanLockToken, ) -> Result { - if offset != u64::MAX { + if offset != u64::MAX && offset != 0 { return Err(Error::new(ESPIPE)); } self.kread(id, buf, flags, stored_flags, token) @@ -590,16 +596,16 @@ pub trait KernelScheme: Send + Sync + 'static { fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { Err(Error::new(EOPNOTSUPP)) } + fn kfutimens(&self, id: usize, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + Err(Error::new(EBADF)) + } fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - // SYS_FSTATVFS is removed, this still exists as the memory scheme implements it to allow df to show memory usage. fn kfstatvfs(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - // SYS_GETDENTS is removed, but this is still used by the irq, acpi and sys schemes. TODO: - // outsource sys scheme to userspace and switch to a non-filesystem API for acpi and irq? fn getdents( &self, id: usize, @@ -611,15 +617,18 @@ pub trait KernelScheme: Send + Sync + 'static { Err(Error::new(EOPNOTSUPP)) } - // SYS_FSYNC is deprecated, but many schemes still implement the corresponding std_fs_call, so - // this should be kept for now. fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { Ok(()) } - + fn ftruncate(&self, id: usize, len: usize, token: &mut CleanLockToken) -> Result<()> { + Err(Error::new(EBADF)) + } fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { Err(Error::new(ESPIPE)) } + fn fchmod(&self, id: usize, new_mode: u16, token: &mut CleanLockToken) -> Result<()> { + Err(Error::new(EBADF)) + } fn fchown( &self, id: usize, @@ -692,7 +701,7 @@ pub trait KernelScheme: Send + Sync + 'static { } fn kstdfscall( &self, - fds: &[usize], + id: usize, kind: StdFsCallKind, desc: Arc, payload: UserSliceRw, @@ -707,6 +716,7 @@ pub trait KernelScheme: Send + Sync + 'static { id: usize, descs: Vec>, flags: CallFlags, + args: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { @@ -725,7 +735,7 @@ pub trait KernelScheme: Send + Sync + 'static { fn translate_std_fs_call( &self, - fds: &[usize], + id: usize, desc: Arc, payload: UserSliceRw, flags: CallFlags, @@ -739,41 +749,26 @@ pub trait KernelScheme: Send + Sync + 'static { return Err(Error::new(EOPNOTSUPP)); }; let metadata = StdFsCallMeta::new(kind, arg1, arg2); - use StdFsCallKind::*; + use syscall::flag::StdFsCallKind::*; match kind { - // Seems unlikely any kernel scheme will implement this. If so, it can be added back to - // this trait. - Relpathat => Err(Error::new(EOPNOTSUPP)), - - _ => { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - let id = fds[0]; - match kind { - Getdents => self.getdents( - id, - payload.into_wo()?, - metadata.arg2 as u16, - metadata.arg1, - token, - ), - Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0), - Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0), - Fsync => self.fsync(id, token).map(|_| 0), - - // The syscalls for these have been replaced by std_fs_call, and the only - // scheme that used to provide a non-default impl was UserScheme. Preserve the - // old default behavior for all other schemes. - Ftruncate | Futimens | Fchmod => Err(Error::new(EBADF)), - - /* TODO: Support Fchown and Unlinkat using std_fs_call - Fchown => self.kstdfscall(fds, kind, desc, payload, flags, metadata, token), - Unlinkat => self.kstdfscall(fds, kind, payload, metadata, &caller).map(|_| 0) - */ - _ => Err(Error::new(EOPNOTSUPP)), - } - } + Fchmod => self.fchmod(id, metadata.arg1 as u16, token).map(|_| 0), + Getdents => self.getdents( + id, + payload.into_wo()?, + metadata.arg2 as u16, + metadata.arg1, + token, + ), + Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0), + Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0), + Fsync => self.fsync(id, token).map(|_| 0), + Ftruncate => self.ftruncate(id, metadata.arg1 as usize, token).map(|_| 0), + Futimens => self.kfutimens(id, payload.into_ro()?, token), + /* TODO: Support Fchown and Unlinkat using std_fs_call + Fchown => self.kstdfscall(id, kind, desc, payload, flags, metadata, token), + Unlinkat => self.kstdfscall(fd, kind, payload, metadata, &caller).map(|_| 0) + */ + _ => Err(Error::new(EOPNOTSUPP)), } } } @@ -787,4 +782,19 @@ pub struct CallerCtx { pub pid: usize, pub uid: u32, pub gid: u32, + pub groups: alloc::vec::Vec, +} +impl CallerCtx { + pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self { + if self.uid == 0 && self.gid == 0 { + Self { + pid: self.pid, + uid: euid, + gid: egid, + groups: self.groups, + } + } else { + self + } + } } diff --git a/src/scheme/pipe.rs b/src/scheme/pipe.rs index 58cbae1cb2..df5db9d908 100644 --- a/src/scheme/pipe.rs +++ b/src/scheme/pipe.rs @@ -7,7 +7,7 @@ use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use crate::{ context::{ - context::bulk_insert_fds, + context::{bulk_add_fds, bulk_insert_fds}, file::{InternalFlags, LockedFileDescription}, }, event, @@ -65,10 +65,6 @@ pub fn pipe(token: &mut CleanLockToken) -> Result<(usize, usize)> { Ok((id, id | WRITE_NOT_READ_BIT)) } -pub fn get_pipe_stat(token: &mut CleanLockToken) -> usize { - PIPES.read(token.token()).len() -} - pub struct PipeScheme; impl PipeScheme { @@ -385,6 +381,7 @@ impl KernelScheme for PipeScheme { id: usize, mut descs: Vec>, _flags: CallFlags, + _args: u64, _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { @@ -478,8 +475,21 @@ impl KernelScheme for PipeScheme { let fds_to_transfer: Vec<_> = vec.drain(..fds_to_read).collect(); if flags.contains(CallFlags::FD_UPPER) { - bulk_insert_fds(fds_to_transfer, payload, &mut token)?; + bulk_insert_fds( + fds_to_transfer, + payload, + flags.contains(CallFlags::FD_CLOEXEC), + &mut token, + )?; + } else { + bulk_add_fds( + fds_to_transfer, + payload, + flags.contains(CallFlags::FD_CLOEXEC), + &mut token, + )?; } + event::trigger_locked( GlobalSchemes::Pipe.scheme_id(), key | WRITE_NOT_READ_BIT, diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index e2bb2541ac..7e22f29338 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -3,12 +3,12 @@ use crate::{ self, context::{HardBlockedReason, LockedFdTbl, SignalState}, file::InternalFlags, - memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec}, + memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, Context, ContextLock, Status, }, memory::{Page, VirtualAddress, PAGE_SIZE}, ptrace, - scheme::{self, memory::MemoryScheme, FileHandle, KernelScheme, StrOrBytes}, + scheme::{self, memory::MemoryScheme, FileHandle, KernelScheme}, sync::{CleanLockToken, LockToken, RwLock, L1, L4}, syscall::{ data::{GrantDesc, Map, SetSighandlerData, Stat}, @@ -20,10 +20,10 @@ use crate::{ }; use super::{CallerCtx, KernelSchemes, OpenResult}; -use ::syscall::{ProcSchemeAttrs, SigProcControl, Sigcontrol}; +use ::syscall::{GrantFlags, ProcSchemeAttrs, SigProcControl, Sigcontrol}; use alloc::{ boxed::Box, - string::String, + string::{String, ToString}, sync::{Arc, Weak}, vec::Vec, }; @@ -37,7 +37,8 @@ use hashbrown::{ hash_map::{DefaultHashBuilder, Entry}, HashMap, }; -use syscall::{data::GlobalSchemes, Error}; +use syscall::data::GlobalSchemes; +use syscall::dirent::{DirEntry, DirentBuf, DirentKind}; fn read_from(dst: UserSliceWo, src: &[u8], offset: u64) -> Result { let avail_src = usize::try_from(offset) @@ -88,7 +89,9 @@ fn try_stop_context( let (context, token) = context.token_split(); let ret = callback(context, token); - context.status = prev_status; + if matches!(context.status, Status::HardBlocked { reason: HardBlockedReason::NotYetStarted }) { + context.status = prev_status; + } ret } @@ -105,12 +108,15 @@ enum ContextHandle { // Attr handles, to set ens/euid/egid/pid. Authority, Attr, + Groups, Status { privileged: bool, }, // can write ContextVerb + /// Deprecated compatibility handles; prefer `ctx` for atomic save/restore. Regs(RegsKind), + Ctx, Sighandler, Start, NewFiletable { @@ -147,6 +153,44 @@ enum ContextHandle { SchedAffinity, MmapMinAddr(Arc), + + ProcRoot, + ProcCpuinfo, + ProcMeminfo, + ProcUptime, + ProcLoadavg, + ProcVersion, + ProcFilesystems, + ProcDir { + pid: usize, + }, + ProcFdDir { + pid: usize, + }, + ProcStat { + pid: usize, + }, + ProcComm { + pid: usize, + }, + ProcCmdline { + pid: usize, + }, + ProcStatus { + pid: usize, + }, + ProcMaps { + pid: usize, + }, + ProcStatm { + pid: usize, + }, + ProcLimits { + pid: usize, + }, + ProcIo { + pid: usize, + }, } #[derive(Clone)] struct Handle { @@ -191,12 +235,211 @@ fn new_handle( Ok((id, fl)) } +fn proc_status_char(status: &Status) -> char { + match status { + Status::Runnable => 'R', + Status::Blocked => 'S', + Status::HardBlocked { reason } => match reason { + HardBlockedReason::AwaitingMmap { .. } => 'D', + HardBlockedReason::Stopped => 'T', + HardBlockedReason::NotYetStarted => 'S', + }, + Status::Dead { .. } => 'Z', + } +} + +fn proc_vsize(addr_space: &Option>, token: &mut CleanLockToken) -> usize { + addr_space + .as_ref() + .map(|addrspace| { + let guard = addrspace.acquire_read(token.downgrade()); + guard + .grants + .iter() + .map(|(_, info)| info.page_count() * PAGE_SIZE) + .sum() + }) + .unwrap_or(0) +} + +fn proc_rss(addr_space: &Option>, token: &mut CleanLockToken) -> usize { + addr_space + .as_ref() + .map(|addrspace| { + let guard = addrspace.acquire_read(token.downgrade()); + guard + .grants + .iter() + .map(|(_, info)| { + use crate::context::memory::Provider; + + match info.provider { + Provider::Allocated { .. } + | Provider::AllocatedShared { .. } + | Provider::PhysBorrowed { .. } + | Provider::External { .. } + | Provider::FmapBorrowed { .. } => info.page_count() * PAGE_SIZE, + } + }) + .sum() + }) + .unwrap_or(0) +} + +fn require_zero_offset(offset: u64) -> Result<()> { + // sys_write passes u64::MAX as the offset for non-positioned file descriptors. + // Proc scheme context handles (regs, sighandler, etc.) always operate at offset 0, + // so accept both 0 (explicit pwrite at 0) and u64::MAX (non-positioned write). + if offset == 0 || offset == u64::MAX { + Ok(()) + } else { + Err(Error::new(EINVAL)) + } +} + +fn validate_kfmap_flags(flags: MapFlags, consume: bool) -> Result<()> { + let flags = MapFlags::from_bits(flags.bits()).ok_or(Error::new(EINVAL))?; + let shared = flags.contains(MapFlags::MAP_SHARED); + let private = flags.contains(MapFlags::MAP_PRIVATE); + if shared == private { + return Err(Error::new(EINVAL)); + } + if flags.contains(MapFlags::MAP_FIXED) && flags.contains(MapFlags::MAP_FIXED_NOREPLACE) { + return Err(Error::new(EINVAL)); + } + if consume && !shared { + return Err(Error::new(EINVAL)); + } + Ok(()) +} + +fn proc_context(pid: usize, token: &mut CleanLockToken) -> Option> { + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut token2) = contexts.token_split(); + contexts.iter().find_map(|ctx| { + let ctx = Arc::clone(&ctx.0); + let matches = ctx.read(token2.token()).pid == pid; + matches.then_some(ctx) + }) +} + +fn proc_stat_line( + pid: usize, + comm: &str, + state: char, + ppid: usize, + priority: usize, + utime: u128, + stime: u128, + pgrp: usize, + session: usize, + nice: i32, + num_threads: usize, + starttime: u128, + vsize: usize, + rss: usize, +) -> String { + format!( + "{pid} ({comm}) {state} {ppid} {pgrp} {session} {tty_nr} {tpgid} {flags} {minflt} {cminflt} {majflt} {cmajflt} {utime} {stime} {cutime} {cstime} {priority} {nice} {num_threads} {itrealvalue} {starttime} {vsize} {rss} {rsslim}\n", + pid = pid, + comm = comm, + state = state, + ppid = ppid, + pgrp = pgrp, + session = session, + tty_nr = 0, + tpgid = 0, + flags = 0, + minflt = 0, + cminflt = 0, + majflt = 0, + cmajflt = 0, + utime = utime, + stime = stime, + cutime = 0, + cstime = 0, + priority = priority, + nice = nice, + num_threads = num_threads, + itrealvalue = 0, + starttime = starttime, + vsize = vsize, + rss = rss / PAGE_SIZE, + rsslim = 0, + ) +} + +fn proc_status_text(context: &Context, num_threads: usize, vsize: usize, rss: usize) -> String { + let comm = context.name.as_str(); + let state = proc_status_char(&context.status); + let ppid = context.owner_proc_id.map_or(0, NonZeroUsize::get); + format!( + "Name:\t{}\nState:\t{}\nPid:\t{}\nPPid:\t{}\nUid:\t{}\t{}\t{}\t{}\nGid:\t{}\t{}\t{}\t{}\nVmSize:\t{} kB\nVmRSS:\t{} kB\nThreads:\t{}\nCpus_allowed_list:\t{}\n", + comm, + state, + context.pid, + ppid, + context.euid, + context.euid, + context.euid, + context.euid, + context.egid, + context.egid, + context.egid, + context.egid, + vsize / 1024, + rss / 1024, + num_threads, + context.sched_affinity + ) +} + enum OpenTy { Ctxt(Arc), Auth, } impl ProcScheme { + fn proc_open(path: &str) -> Option { + let path = path.trim_start_matches('/'); + if path.is_empty() || path == "proc" { + return Some(ContextHandle::ProcRoot); + } + if path == "proc/cpuinfo" || path == "cpuinfo" { + return Some(ContextHandle::ProcCpuinfo); + } + if path == "proc/meminfo" || path == "meminfo" { + return Some(ContextHandle::ProcMeminfo); + } + if path == "proc/uptime" || path == "uptime" { + return Some(ContextHandle::ProcUptime); + } + if path == "proc/loadavg" || path == "loadavg" { + return Some(ContextHandle::ProcLoadavg); + } + if path == "proc/version" || path == "version" { + return Some(ContextHandle::ProcVersion); + } + if path == "proc/filesystems" || path == "filesystems" { + return Some(ContextHandle::ProcFilesystems); + } + let mut parts = path.split('/'); + let pid = parts.next()?.parse::().ok()?; + match (parts.next(), parts.next()) { + (None, None) => Some(ContextHandle::ProcDir { pid }), + (Some("fd"), None) => Some(ContextHandle::ProcFdDir { pid }), + (Some("stat"), None) => Some(ContextHandle::ProcStat { pid }), + (Some("comm"), None) => Some(ContextHandle::ProcComm { pid }), + (Some("cmdline"), None) => Some(ContextHandle::ProcCmdline { pid }), + (Some("status"), None) => Some(ContextHandle::ProcStatus { pid }), + (Some("maps"), None) => Some(ContextHandle::ProcMaps { pid }), + (Some("statm"), None) => Some(ContextHandle::ProcStatm { pid }), + (Some("limits"), None) => Some(ContextHandle::ProcLimits { pid }), + (Some("io"), None) => Some(ContextHandle::ProcIo { pid }), + _ => None, + } + } + fn openat_context( &self, path: &str, @@ -233,6 +476,8 @@ impl ProcScheme { ), "current-addrspace" => (ContextHandle::CurrentAddrSpace, false), "current-filetable" => (ContextHandle::CurrentFiletable, false), + "ctx" => (ContextHandle::Ctx, false), + // Deprecated compatibility handles; prefer `ctx` for atomic save/restore. "regs/float" => (ContextHandle::Regs(RegsKind::Float), false), "regs/int" => (ContextHandle::Regs(RegsKind::Int), false), "regs/env" => (ContextHandle::Regs(RegsKind::Env), false), @@ -250,6 +495,7 @@ impl ProcScheme { ), "sched-affinity" => (ContextHandle::SchedAffinity, true), "status" => (ContextHandle::Status { privileged: false }, false), + "proc" => (ContextHandle::ProcRoot, true), _ if path.starts_with("auth-") => { let nonprefix = &path["auth-".len()..]; let next_dash = nonprefix.find('-').ok_or(Error::new(ENOENT))?; @@ -261,6 +507,7 @@ impl ProcScheme { let handle = match actual_name { "attrs" => ContextHandle::Attr, "status" => ContextHandle::Status { privileged: true }, + "groups" => ContextHandle::Groups, _ => return Err(Error::new(ENOENT)), }; @@ -290,6 +537,41 @@ impl ProcScheme { token: &mut CleanLockToken, ) -> Result<(usize, InternalFlags)> { let operation_name = operation_str.ok_or(Error::new(EINVAL))?; + // /proc/self[/...] → resolve /proc/[/...] + if operation_name == "proc/self" || operation_name == "self" { + let pid = context::current().read(token.token()).pid; + let handle = Handle { + context: context::current(), + kind: ContextHandle::ProcDir { pid }, + }; + return new_handle((handle, InternalFlags::POSITIONED), token); + } + if operation_name.starts_with("proc/self/") || operation_name.starts_with("self/") { + let pid = context::current().read(token.token()).pid; + let rest = if operation_name.starts_with("proc/self/") { + &operation_name["proc/self/".len()..] + } else { + &operation_name["self/".len()..] + }; + let resolved = format!("proc/{}/{}", pid, rest); + let mut parts = resolved.split('/'); + let _ = parts.next(); // "proc" + let pid = parts.next().and_then(|p| p.parse::().ok()).unwrap_or(pid); + if let Some(kind) = Self::proc_open(&resolved) { + let handle = Handle { + context: context::current(), + kind, + }; + return new_handle((handle, InternalFlags::POSITIONED), token); + } + } + if let Some(kind) = Self::proc_open(operation_name) { + let handle = Handle { + context: context::current(), + kind, + }; + return new_handle((handle, InternalFlags::POSITIONED), token); + } let (mut handle, positioned) = match ty { OpenTy::Ctxt(context) => { match self.openat_context(operation_name, Arc::clone(&context), token)? { @@ -306,6 +588,11 @@ impl ProcScheme { let id = NonZeroUsize::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)) .ok_or(Error::new(EMFILE))?; let context = context::spawn(true, Some(id), ret, token)?; + { + let parent_groups = + context::current().read(token.token()).groups.clone(); + context.write(token.token()).groups = parent_groups; + } HANDLES.write(token.token()).insert( id.get(), Handle { @@ -425,6 +712,7 @@ impl KernelScheme for ProcScheme { } fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + let mut inner_token = unsafe { CleanLockToken::new() }; let handle = HANDLES .write(token.token()) .remove(&id) @@ -441,7 +729,7 @@ impl KernelScheme for ProcScheme { arg1, }, } => { - let old_ctx = try_stop_context(context, token, |context, token| { + let old_ctx = try_stop_context(context, token, |context, _| { let regs = context.regs_mut().ok_or(Error::new(EBADFD))?; regs.set_instr_pointer(new_ip); regs.set_stack_pointer(new_sp); @@ -452,7 +740,7 @@ impl KernelScheme for ProcScheme { ))] regs.set_arg1(arg1); - Ok(context.set_addr_space(Some(new), token)) + Ok(context.set_addr_space(Some(new), inner_token.downgrade())) })?; if let Some(old_ctx) = old_ctx && let Some(addrspace) = Arc::into_inner(old_ctx) @@ -491,6 +779,7 @@ impl KernelScheme for ProcScheme { consume: bool, token: &mut CleanLockToken, ) -> Result { + let mut inner_token = unsafe { CleanLockToken::new() }; let handle = HANDLES .read(token.token()) .get(&id) @@ -521,9 +810,9 @@ impl KernelScheme for ProcScheme { let src_page_count = NonZeroUsize::new(src_span.count).ok_or(Error::new(EINVAL))?; - let mut notify_files = UnmapVec::new(); + let mut notify_files = Vec::new(); - // TODO: Validate flags + validate_kfmap_flags(map.flags, consume)?; let result_base = if consume { dst_addr_space.r#move( Some((addrspace, &mut *src_addr_space)), @@ -581,9 +870,7 @@ impl KernelScheme for ProcScheme { }; // TODO: Allocated or AllocatedShared? let addrsp = AddrSpace::current()?; - // TODO: Lock ordering violation - let mut token = unsafe { CleanLockToken::new() }; - let page = addrsp.acquire_write(token.downgrade()).mmap_anywhere( + let page = addrsp.acquire_write(inner_token.downgrade()).mmap_anywhere( &addrsp, NonZeroUsize::new(1).unwrap(), MapFlags::PROT_READ | MapFlags::PROT_WRITE, @@ -621,39 +908,95 @@ impl KernelScheme for ProcScheme { let Handle { context, kind } = handle; kind.kreadoff(id, context, buf, offset, token) } - fn kcall( + fn kfdwrite( &self, - fds: &[usize], - payload: UserSliceRw, + id: usize, + descs: Vec>, flags: CallFlags, + _arg: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); + let context = { + let mut handles = HANDLES.read(token.token()); + let (handles, mut token) = handles.token_split(); + let handle = handles.get(&id).unwrap(); + + let Handle { context, kind } = handle; + + if let ContextHandle::Filetable { .. } | ContextHandle::NewFiletable { .. } = + &handle.kind + { + context.clone() + } else { + return Err(Error::new(EBADF)); + } + }; + + let target_fd = metadata.get(0).ok_or(Error::new(EINVAL))?; + + if let Some(file) = { + let mut context = context.read(token.token()); + let (context, mut token) = context.token_split(); + context.remove_file(FileHandle(*target_fd as usize), &mut token.token()) + } { + file.close(token)?; } - let id = fds[0]; + + let file = descs.get(0).unwrap(); + let mut context = context.write(token.token()); + let (context, mut token) = context.token_split(); + context + .insert_file( + FileHandle(usize::try_from(*target_fd).map_err(|_| Error::new(EBADFD))?), + context::file::FileDescriptor { + description: file.clone(), + cloexec: false, + }, + &mut token, + ) + .ok_or(Error::new(EMFILE))?; + + Ok(0) + } + fn kcall( + &self, + fds: &[usize], + _payload: UserSliceRw, + _flags: CallFlags, + metadata: &[u64], + token: &mut CleanLockToken, + ) -> Result { + let id = fds.first().copied().ok_or(Error::new(EINVAL))?; // TODO: simplify let handle = { - let handles = HANDLES.read(token.token()); - let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + let mut handles = HANDLES.write(token.token()); + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.clone() }; - handle.kind.kcall( - fds, - payload, - flags, - metadata, - Arc::clone(&handle.context), - token, - ) + let ContextHandle::OpenViaDup = handle.kind else { + return Err(Error::new(EBADF)); + }; + + let verb: u8 = (*metadata.first().ok_or(Error::new(EINVAL))?) + .try_into() + .map_err(|_| Error::new(EINVAL))?; + let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?; + + match verb { + ProcSchemeVerb::Iopl => context::current() + .write(token.token()) + .set_userspace_io_allowed(true), + _ => return Err(Error::new(EINVAL)), + } + Ok(0) } fn kwriteoff( &self, id: usize, buf: UserSliceRo, - _offset: u64, + offset: u64, _fcntl_flags: u32, _stored_flags: u32, token: &mut CleanLockToken, @@ -662,13 +1005,13 @@ impl KernelScheme for ProcScheme { // Don't hold a global lock during the context switch later on let handle = { - let handles = HANDLES.read(token.token()); - let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + let mut handles = HANDLES.write(token.token()); + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.clone() }; let Handle { context, kind } = handle; - kind.kwriteoff(id, context, buf, token) + kind.kwriteoff(id, context, buf, offset, token) } fn kfpath(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result { @@ -676,12 +1019,103 @@ impl KernelScheme for ProcScheme { buf.copy_common_bytes_from_slice("/scheme/kernel.proc/".as_bytes()) } + fn getdents( + &self, + id: usize, + buf: UserSliceWo, + header_size: u16, + opaque_id_start: u64, + token: &mut CleanLockToken, + ) -> Result { + let Ok(opaque) = usize::try_from(opaque_id_start) else { + return Ok(0); + }; + + let handle = { + let handles = HANDLES.read(token.token()); + handles.get(&id).ok_or(Error::new(EBADF))?.clone() + }; + + let mut buf = DirentBuf::new(buf, header_size).ok_or(Error::new(EIO))?; + match handle.kind { + ContextHandle::ProcRoot => { + // First: system files (cpuinfo, meminfo, uptime, loadavg, version, filesystems) + for (idx, name) in ["cpuinfo", "meminfo", "uptime", "loadavg", "version", "filesystems"].iter().enumerate() { + buf.entry(DirEntry { + inode: 0, + next_opaque_id: (idx + 1) as u64, + kind: DirentKind::Regular, + name, + })?; + } + // Then: process directories (one per PID) + let mut pids = Vec::new(); + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut token) = contexts.token_split(); + for context_ref in contexts.iter() { + pids.push(context_ref.read(token.token()).pid); + } + pids.sort_unstable(); + let base_idx = 6; // after 6 system files + for (idx, pid) in pids.into_iter().enumerate().skip(opaque.saturating_sub(base_idx)) { + let name = format!("{}", pid); + buf.entry(DirEntry { + inode: pid as u64, + next_opaque_id: (base_idx + idx + 1) as u64, + kind: DirentKind::Directory, + name: &name, + })?; + } + Ok(buf.finalize()) + } + ContextHandle::ProcDir { pid } => { + for (idx, name) in ["stat", "comm", "cmdline", "status", "maps", "statm", "limits", "io", "fd"] + .iter() + .enumerate() + .skip(opaque) + { + buf.entry(DirEntry { + inode: pid as u64, + next_opaque_id: (idx + 1) as u64, + kind: if *name == "fd" { DirentKind::Directory } else { DirentKind::Regular }, + name, + })?; + } + Ok(buf.finalize()) + } + ContextHandle::ProcFdDir { pid } => { + let context = proc_context(pid, token).ok_or(Error::new(ESRCH))?; + let files = context.read(token.token()).files.clone(); + let guard = files.read(token.token()); + let fds: Vec = guard.posix_fdtbl.iter() + .enumerate() + .filter(|(_, fd)| fd.is_some()) + .map(|(i, _)| i) + .collect(); + for (idx, &fd) in fds.iter().enumerate().skip(opaque) { + let name = format!("{}", fd); + buf.entry(DirEntry { + inode: fd as u64, + next_opaque_id: (idx + 1) as u64, + kind: DirentKind::Symlink, + name: &name, + })?; + } + Ok(buf.finalize()) + } + _ => Err(Error::new(ENOTDIR)), + } + } + fn kfstat(&self, id: usize, buffer: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { let handles = HANDLES.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; buffer.copy_exactly(&Stat { - st_mode: MODE_FILE | 0o666, + st_mode: match handle.kind { + ContextHandle::ProcRoot | ContextHandle::ProcDir { .. } | ContextHandle::ProcFdDir { .. } => MODE_DIR | 0o755, + _ => MODE_FILE | 0o666, + }, st_size: handle.fsize()?, ..Stat::default() @@ -691,10 +1125,13 @@ impl KernelScheme for ProcScheme { } fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { - let handles = HANDLES.read(token.token()); - let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + let mut handles = HANDLES.write(token.token()); + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; - handle.fsize() + match handle.kind { + ContextHandle::ProcRoot | ContextHandle::ProcDir { .. } => Ok(0), + _ => handle.fsize(), + } } /// Dup is currently used to implement clone() and execve(). @@ -740,7 +1177,7 @@ impl KernelScheme for ProcScheme { OpenTy::Auth, Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) .filter(|s| !s.is_empty()), - O_RDWR, + O_RDWR | O_CLOEXEC, token, ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) @@ -754,7 +1191,7 @@ impl KernelScheme for ProcScheme { OpenTy::Ctxt(context), Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) .filter(|s| !s.is_empty()), - O_RDWR, + O_RDWR | O_CLOEXEC, token, ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)); @@ -834,7 +1271,8 @@ impl KernelScheme for ProcScheme { _ => return Err(Error::new(EINVAL)), }; - handle(Handle { context, kind }, true) + let positioned = !matches!(kind, ContextHandle::AddrSpace { .. }); + handle(Handle { context, kind }, positioned) } _ => return Err(Error::new(EINVAL)), }, @@ -842,55 +1280,6 @@ impl KernelScheme for ProcScheme { ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) } - fn kfdwrite( - &self, - id: usize, - descs: Vec>, - flags: CallFlags, - metadata: &[u64], - token: &mut CleanLockToken, - ) -> Result { - let context = { - let mut handles = HANDLES.read(token.token()); - let (handles, mut token) = handles.token_split(); - let handle = handles.get(&id).unwrap(); - - let Handle { context, kind } = handle; - - if let ContextHandle::Filetable { .. } | ContextHandle::NewFiletable { .. } = - &handle.kind - { - context.clone() - } else { - return Err(Error::new(EBADF)); - } - }; - - let target_fd = metadata.get(0).ok_or(Error::new(EINVAL))?; - - if let Some(file) = { - let mut context = context.read(token.token()); - let (context, mut token) = context.token_split(); - context.remove_file(FileHandle(*target_fd as usize), &mut token.token()) - } { - file.close(token)?; - } - - let mut file = descs.get(0).unwrap(); - let mut context = context.write(token.token()); - let (context, mut token) = context.token_split(); - context - .insert_file( - FileHandle(usize::try_from(*target_fd).map_err(|_| Error::new(EBADFD))?), - context::file::FileDescriptor { - description: file.clone(), - }, - &mut token, - ) - .ok_or(Error::new(EMFILE))?; - - Ok(0) - } } fn extract_scheme_number(fd: usize, token: &mut CleanLockToken) -> Result<(KernelSchemes, usize)> { let (scheme_id, number) = { @@ -928,12 +1317,21 @@ impl ContextHandle { id: usize, context: Arc, buf: UserSliceRo, + offset: u64, token: &mut CleanLockToken, ) -> Result { match self { Self::AddrSpace { addrspace } => { let mut chunks = buf.usizes(); let mut words_read = 0; + let word_size = size_of::() as u64; + if offset % word_size != 0 { + return Err(Error::new(EINVAL)); + } + let offset_words = usize::try_from(offset / word_size).map_err(|_| Error::new(EINVAL))?; + for _ in 0..offset_words { + chunks.next().ok_or(Error::new(EINVAL))??; + } let mut next = || { words_read += 1; chunks.next().ok_or(Error::new(EINVAL)) @@ -943,15 +1341,21 @@ impl ContextHandle { op @ ADDRSPACE_OP_MMAP | op @ ADDRSPACE_OP_TRANSFER => { let fd = next()??; let offset = next()??; - let page_span = crate::syscall::validate_region(next()??, next()??)?; - let flags = MapFlags::from_bits(next()??).ok_or(Error::new(EINVAL))?; + let addr_raw = next()??; + let size_raw = next()??; + let flags_raw = next()??; + let page_span = crate::syscall::validate_region(addr_raw, size_raw)?; + let flags = match MapFlags::from_bits(flags_raw) { + Some(f) => f, + None => return Err(Error::new(EINVAL)), + }; if fd == !0 { if op == ADDRSPACE_OP_TRANSFER { return Err(Error::new(EOPNOTSUPP)); } - return MemoryScheme::fmap_anonymous( + let res = MemoryScheme::fmap_anonymous( &addrspace, &Map { offset, @@ -962,6 +1366,7 @@ impl ContextHandle { false, token, ); + return res; } else { let (scheme, number) = extract_scheme_number(fd, token)?; @@ -987,7 +1392,9 @@ impl ContextHandle { let unpin = false; let res = addrspace.munmap(page_span, unpin, token)?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } ADDRSPACE_OP_MPROTECT => { let page_span = crate::syscall::validate_region(next()??, next()??)?; @@ -999,8 +1406,24 @@ impl ContextHandle { } Ok(words_read * size_of::()) } + ContextHandle::Ctx => { + require_zero_offset(offset)?; + let regs = unsafe { buf.read_exact::()? }; + try_stop_context(context, token, |context, _| { + context.set_fx_regs(regs.float); + match context.regs_mut() { + None => Err(Error::new(ENOTRECOVERABLE)), + Some(stack) => { + stack.load(®s.int); + context.write_env_regs(regs.env) + } + } + })?; + Ok(size_of::()) + } ContextHandle::Regs(kind) => match kind { RegsKind::Float => { + require_zero_offset(offset)?; let regs = unsafe { buf.read_exact::()? }; try_stop_context(context, token, |context, _| { @@ -1014,6 +1437,7 @@ impl ContextHandle { }) } RegsKind::Int => { + require_zero_offset(offset)?; let regs = unsafe { buf.read_exact::()? }; try_stop_context(context, token, |context, _| match context.regs_mut() { @@ -1033,12 +1457,14 @@ impl ContextHandle { }) } RegsKind::Env => { + require_zero_offset(offset)?; let regs = unsafe { buf.read_exact::()? }; write_env_regs(context, regs, token)?; Ok(size_of::()) } }, ContextHandle::Sighandler => { + require_zero_offset(offset)?; let data = unsafe { buf.read_exact::()? }; if data.user_handler >= crate::USER_END_OFFSET @@ -1093,20 +1519,24 @@ impl ContextHandle { Ok(size_of::()) } - ContextHandle::Start => match context.write(token.token()).status { - ref mut status @ Status::HardBlocked { - reason: HardBlockedReason::NotYetStarted, - } => { - *status = Status::Runnable; - Ok(buf.len()) + ContextHandle::Start => { + require_zero_offset(offset)?; + match context.write(token.token()).status { + ref mut status @ Status::HardBlocked { + reason: HardBlockedReason::NotYetStarted, + } => { + *status = Status::Runnable; + Ok(buf.len()) + } + _ => Err(Error::new(EINVAL)), } - _ => Err(Error::new(EINVAL)), }, ContextHandle::Filetable { .. } | ContextHandle::NewFiletable { .. } => { Err(Error::new(EBADF)) } ContextHandle::CurrentFiletable => { + require_zero_offset(offset)?; let filetable_fd = buf.read_usize()?; let (hopefully_this_scheme, number) = extract_scheme_number(filetable_fd, token)?; verify_scheme(hopefully_this_scheme)?; @@ -1152,6 +1582,7 @@ impl ContextHandle { Ok(size_of::()) } ContextHandle::CurrentAddrSpace => { + require_zero_offset(offset)?; let mut iter = buf.usizes(); let addrspace_fd = iter.next().ok_or(Error::new(EINVAL))??; let sp = iter.next().ok_or(Error::new(EINVAL))??; @@ -1189,6 +1620,7 @@ impl ContextHandle { Ok(written) } Self::MmapMinAddr(ref addrspace) => { + require_zero_offset(offset)?; let val = buf.read_usize()?; if val % PAGE_SIZE != 0 || val > crate::USER_END_OFFSET { return Err(Error::new(EINVAL)); @@ -1198,6 +1630,7 @@ impl ContextHandle { Ok(size_of::()) } Self::SchedAffinity => { + require_zero_offset(offset)?; let mask = unsafe { buf.read_exact::()? }; context @@ -1227,13 +1660,29 @@ impl ContextHandle { Status::Dead { .. } => return Err(Error::new(EOWNERDEAD)), Status::HardBlocked { reason: HardBlockedReason::AwaitingMmap { .. }, - } => todo!(), + } => { + // Process is blocked waiting for an mmap grant. + // Allow the stop to proceed — the mmap grant cleanup + // or completion will be handled when the process resumes. + // Cross-referenced with Linux kernel/signal.c: + // signal handling doesn't special-case mmap-in-progress; + // fault handler retries on resume. + } _ => (), } guard.status = Status::HardBlocked { reason: HardBlockedReason::Stopped, }; - // TODO: wait for context to be switched away from, and/or IPI? + drop(guard); + + let mut spins = 10_000usize; + while context.read(token.token()).running { + if spins == 0 { + return Err(Error::new(EAGAIN)); + } + spins -= 1; + context::switch(token); + } Ok(size_of::()) } ContextVerb::Unstop => { @@ -1273,14 +1722,16 @@ impl ContextHandle { false, token, )?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } } crate::syscall::exit_this_context(None, token); } else { let mut ctxt = context.write(token.token()); //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); - if ctxt.status.is_dead() { + if let context::Status::Dead { .. } = ctxt.status { return Ok(size_of::()); } ctxt.status = context::Status::Runnable; @@ -1291,16 +1742,16 @@ impl ContextHandle { } } ContextHandle::Attr => { + require_zero_offset(offset)?; let info = unsafe { buf.read_exact::()? }; + let mut guard = context.write(token.token()); let len = info .debug_name .iter() .position(|c| *c == 0) .unwrap_or(info.debug_name.len()) - .min(crate::context::context::CONTEXT_NAME_CAPAC); - - let mut guard = context.write(token.token()); + .min(guard.name.capacity()); let debug_name = core::str::from_utf8(&info.debug_name[..len]) .map_err(|_| Error::new(EINVAL))?; guard.name.clear(); @@ -1310,19 +1761,50 @@ impl ContextHandle { guard.euid = info.euid; guard.egid = info.egid; guard.prio = (info.prio as usize).min(39); - - #[cfg(feature = "profiling")] - { - let debug_id = guard.debug_id; - let debug_name = guard.name; - drop(guard); - crate::profiling::DBG_ID_MAP - .write(token.token()) - .insert(debug_id, debug_name); + if guard.pgrp == 0 { + guard.pgrp = guard.pid; + } + if guard.session == 0 { + guard.session = guard.pid; } Ok(size_of::()) } + Self::Groups => { + require_zero_offset(offset)?; + const NGROUPS_MAX: usize = 65536; + if buf.len() % size_of::() != 0 { + return Err(Error::new(EINVAL)); + } + let count = buf.len() / size_of::(); + if count > NGROUPS_MAX { + return Err(Error::new(EINVAL)); + } + let mut groups = Vec::with_capacity(count); + for chunk in buf.in_exact_chunks(size_of::()).take(count) { + groups.push(chunk.read_u32()?); + } + let proc_id = { + let guard = context.read(token.token()); + guard.owner_proc_id + }; + { + let mut guard = context.write(token.token()); + guard.groups = groups.clone(); + } + if let Some(pid) = proc_id { + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut t) = contexts.token_split(); + for context_ref in contexts.iter() { + let mut ctx = context_ref.write(t.token()); + if ctx.owner_proc_id == Some(pid) { + ctx.groups = groups.clone(); + } + } + } + Ok(count * size_of::()) + } ContextHandle::OpenViaDup => { + require_zero_offset(offset)?; let mut args = buf.usizes(); let user_data = args.next().ok_or(Error::new(EINVAL))??; @@ -1352,7 +1834,9 @@ impl ContextHandle { false, token, )?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } } crate::syscall::exit_this_context(None, token); @@ -1375,6 +1859,35 @@ impl ContextHandle { token: &mut CleanLockToken, ) -> Result { match self { + ContextHandle::Ctx => { + require_zero_offset(offset)?; + let mut regs = syscall::FullContextRegs::default(); + try_stop_context(context, token, |context, _| { + match context.regs() { + None => { + assert!(!context.running, "try_stop_context is broken, clearly"); + println!( + "{}:{}: Couldn't read registers from stopped process", + file!(), + line!() + ); + Err(Error::new(ENOTRECOVERABLE)) + } + Some(stack) => { + stack.save(&mut regs.int); + regs.float = context.get_fx_regs(); + regs.env = context.read_env_regs()?; + Ok(()) + } + } + })?; + buf.copy_common_bytes_from_slice(unsafe { + slice::from_raw_parts( + ®s as *const syscall::FullContextRegs as *const u8, + size_of::(), + ) + }) + } ContextHandle::Regs(kind) => { union Output { float: FloatRegisters, @@ -1465,6 +1978,270 @@ impl ContextHandle { } ContextHandle::Filetable { data, .. } => read_from(buf, data, offset), + ContextHandle::ProcCpuinfo => { + let mut output = String::new(); + crate::arch::device::cpu::cpu_info(&mut output).map_err(|_| Error::new(EIO))?; + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcMeminfo => { + let total_kib = crate::memory::total_frames() as u64 * PAGE_SIZE as u64 / 1024; + let free_kib = crate::memory::free_frames() as u64 * PAGE_SIZE as u64 / 1024; + let output = format!( + "MemTotal: {} kB\n\ + MemFree: {} kB\n\ + MemAvailable: {} kB\n\ + Buffers: 0 kB\n\ + Cached: 0 kB\n\ + SwapCached: 0 kB\n\ + SwapTotal: 0 kB\n\ + SwapFree: 0 kB\n", + total_kib, free_kib, free_kib + ); + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcUptime => { + let uptime_s = crate::time::monotonic(token) as f64 / crate::time::NANOS_PER_SEC as f64; + let idle_s = 0.0f64; + let output = format!("{:.2} {:.2}\n", uptime_s, idle_s); + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcLoadavg => { + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut t) = contexts.token_split(); + let mut nr_runnable: usize = 0; + let mut total: usize = 0; + for ctx in contexts.iter() { + total += 1; + if ctx.read(t.token()).status.is_runnable() { + nr_runnable += 1; + } + } + let output = format!("0.00 0.00 0.00 {}/{total} 0\n", nr_runnable); + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcVersion => { + let output = format!( + "Red Bear OS version {} ({}) {}\n", + env!("CARGO_PKG_VERSION"), + option_env!("TARGET").unwrap_or("x86_64"), + option_env!("COOKBOOK_SOURCE_IDENT").unwrap_or("") + ); + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcFilesystems => { + // Linux /proc/filesystems lists registered filesystem types. + // Cross-referenced with Linux 7.1 fs/filesystems.c filesystems_proc_show(). + // Red Bear supports redoxfs as the native filesystem. + let output = "nodev\tsysfs\nnodev\tproc\nnodev\tdevtmpfs\n\tredoxfs\n"; + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcStat { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let (pid, comm, state, ppid, priority, utime, stime, pgrp, session, starttime, owner_proc_id, addr_space) = { + let context = context.read(token.token()); + ( + context.pid, + context.name.to_string(), + proc_status_char(&context.status), + context.owner_proc_id.map_or(0, NonZeroUsize::get), + context.prio, + context.utime / crate::time::NANOS_PER_SEC, + context.stime / crate::time::NANOS_PER_SEC, + context.pgrp, + context.session, + context.start_time / (crate::time::NANOS_PER_SEC / 100), + context.owner_proc_id, + context.addr_space.clone(), + ) + }; + let num_threads = { + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut token2) = contexts.token_split(); + contexts + .iter() + .filter(|ctx| ctx.read(token2.token()).owner_proc_id == owner_proc_id) + .count() + }; + let vsize = proc_vsize(&addr_space, token); + let rss = proc_rss(&addr_space, token); + let nice = (priority as i32 / 2) - 20; + read_from( + buf, + proc_stat_line( + pid, + &comm, + state, + ppid, + priority, + utime, + stime, + pgrp, + session, + nice, + num_threads, + starttime, + vsize, + rss, + ) + .as_bytes(), + offset, + ) + } + ContextHandle::ProcComm { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let context = context.read(token.token()); + read_from(buf, context.name.as_bytes(), offset) + } + ContextHandle::ProcCmdline { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let context = context.read(token.token()); + let mut cmd = context.name.as_bytes().to_vec(); + cmd.push(0); + read_from(buf, &cmd, offset) + } + ContextHandle::ProcStatus { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let owner_proc_id = context.read(token.token()).owner_proc_id; + let num_threads = { + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut token2) = contexts.token_split(); + contexts + .iter() + .filter(|ctx| ctx.read(token2.token()).owner_proc_id == owner_proc_id) + .count() + }; + let (vsize, rss) = { + let addr_space = { + let context = context.read(token.token()); + context.addr_space.clone() + }; + let vsize = proc_vsize(&addr_space, token); + let rss = proc_rss(&addr_space, token); + (vsize, rss) + }; + let context = context.read(token.token()); + read_from(buf, proc_status_text(&context, num_threads, vsize, rss).as_bytes(), offset) + } + ContextHandle::ProcMaps { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let addr_space = context.read(token.token()).addr_space.clone(); + let mut output = String::new(); + if let Some(addr_space) = addr_space { + let token2 = token.downgrade(); + let guard = addr_space.acquire_read(token2); + for (grant_base, grant_info) in guard.grants.iter() { + let start = grant_base.start_address().data(); + let size = grant_info.page_count() * PAGE_SIZE; + let end = start + size; + let grant_flags = grant_info.grant_flags(); + let perms = alloc::format!( + "{}{}{}p", + if grant_flags.contains(GrantFlags::GRANT_READ) { 'r' } else { '-' }, + if grant_flags.contains(GrantFlags::GRANT_WRITE) { 'w' } else { '-' }, + if grant_flags.contains(GrantFlags::GRANT_EXEC) { 'x' } else { '-' }, + ); + let path_str = if grant_info.file_ref().is_some() { + String::from("[file]") + } else if start < 0x10000000 { + String::from("[vdso]") + } else { + String::from("[heap]") + }; + output.push_str(&alloc::format!( + "{:08x}-{:08x} {} 00000000 00:00 0 {}\n", + start, end, perms, path_str + )); + } + } + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcStatm { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let addr_space = context.read(token.token()).addr_space.clone(); + let (size, resident) = if let Some(addr_space) = addr_space { + let token2 = token.downgrade(); + let guard = addr_space.acquire_read(token2); + let mut total_size = 0usize; + let mut total_resident = 0usize; + for (_grant_base, grant_info) in guard.grants.iter() { + let page_count = grant_info.page_count() * PAGE_SIZE; + total_size += page_count; + if grant_info.grant_flags().contains(GrantFlags::GRANT_PHYS) { + total_resident += page_count; + } + } + (total_size / PAGE_SIZE, total_resident / PAGE_SIZE) + } else { + (0, 0) + }; + // Format: size resident shared text lib data dt (in pages) + // Cross-referenced with Linux 7.1 fs/proc/array.c:proc_pid_statm(). + // On Red Bear we approximate: shared=0, text=size, lib=0, data=size, dt=0. + let output = alloc::format!( + "{} {} 0 {} 0 {} 0\n", + size, resident, size, size + ); + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcLimits { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let rlimits = context.read(token.token()).rlimits; + // Linux 7.1 fs/proc/array.c:proc_pid_limits() format + // 16 RLIMITs with soft/hard values and units string + let names = [ + ("Max cpu time", "seconds"), + ("Max file size", "bytes"), + ("Max data size", "bytes"), + ("Max stack size", "bytes"), + ("Max core file size", "bytes"), + ("Max resident set", "bytes"), + ("Max processes", "processes"), + ("Max open files", "files"), + ("Max locked memory", "bytes"), + ("Max address space", "bytes"), + ("Max file locks", "locks"), + ("Max sigpending", "signals"), + ("Max msgqueue size", "bytes"), + ("Max nice priority", "0"), + ("Max realtime priority", "0"), + ("Max realtime timeout", "us"), + ]; + let format_limit = |v: u64| -> alloc::string::String { + if v == u64::MAX { String::from("unlimited") } + else { alloc::format!("{}", v) } + }; + let mut output = String::from( + "Limit Soft Limit Hard Limit Units\n", + ); + for (i, (name, units)) in names.iter().enumerate() { + let soft = format_limit(rlimits[i]); + let hard = format_limit(if i + 16 < rlimits.len() { rlimits[i + 16] } else { u64::MAX }); + output.push_str(&alloc::format!( + "{:<24} {:>20} {:>20} {}\n", + name, soft, hard, units + )); + } + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcIo { pid } => { + let context = proc_context(*pid, token).ok_or(Error::new(ESRCH))?; + let context = context.read(token.token()); + // Linux 7.1 fs/proc/base.c do_task_io_accounting() format + // /proc/[pid]/io is a key=value text file + let output = alloc::format!( + "rchar: {}\nwchar: {}\nsyscr: {}\nsyscw: {}\n\ + read_bytes: {}\nwrite_bytes: {}\ncancelled_write_bytes: {}\n", + context.io_rchar, + context.io_wchar, + context.io_syscr, + context.io_syscw, + context.io_read_bytes, + context.io_write_bytes, + context.io_cancelled_write_bytes + ); + read_from(buf, output.as_bytes(), offset) + } + ContextHandle::ProcRoot | ContextHandle::ProcDir { .. } => Err(Error::new(EBADF)), ContextHandle::MmapMinAddr(addrspace) => { let mut token = token.token(); let addr = addrspace.acquire_read(token.downgrade()); @@ -1524,6 +2301,15 @@ impl ContextHandle { debug_name, }) } + Self::Groups => { + let c = &context.read(token.token()); + let max = buf.len() / size_of::(); + let count = c.groups.len().min(max); + for (chunk, gid) in buf.in_exact_chunks(size_of::()).zip(&c.groups).take(count) { + chunk.copy_from_slice(&gid.to_ne_bytes())?; + } + Ok(count * size_of::()) + } ContextHandle::Sighandler => { let data = match context.read(token.token()).sig { Some(ref sig) => SetSighandlerData { @@ -1543,101 +2329,6 @@ impl ContextHandle { _ => Err(Error::new(EBADF)), } } - pub fn kcall( - &self, - fds: &[usize], - payload: UserSliceRw, - flags: CallFlags, - metadata: &[u64], - context: Arc, - token: &mut CleanLockToken, - ) -> Result { - match self { - ContextHandle::OpenViaDup => { - let verb: u8 = (*metadata.first().ok_or(Error::new(EINVAL))?) - .try_into() - .map_err(|_| Error::new(EINVAL))?; - let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?; - - match verb { - ProcSchemeVerb::Iopl => context::current() - .write(token.token()) - .set_userspace_io_allowed(true), - _ => return Err(Error::new(EINVAL)), - } - Ok(0) - } - ContextHandle::Filetable { - filetable, - binary_format: _, - data: _, - } => { - let op = syscall::flag::FileTableVerb::try_from_raw( - u8::try_from(*metadata.first().ok_or(Error::new(EINVAL))?) - .map_err(|_| Error::new(EINVAL))?, - ) - .ok_or(Error::new(EINVAL))?; - - match op { - FileTableVerb::Close => { - let files = filetable.upgrade().ok_or(Error::new(EBADF))?; - let payload_chunks = payload.in_exact_chunks(size_of::()); - let fds = payload_chunks - .map(|chunk| { - let fd = chunk.read_usize()?; - Ok(FileHandle::from(fd)) - }) - .collect::>>()?; - let num_fds = fds.len(); - let files_to_close = files.write(token.token()).bulk_remove_files(&fds)?; - files_to_close.into_iter().for_each(|file| { - let _ = file.close(token); - }); - Ok(num_fds) - } - FileTableVerb::Dup2 => { - let mut it = payload.usizes(); - let old = FileHandle(it.next().ok_or(Error::new(EINVAL)).flatten()?); - let new = FileHandle(it.next().ok_or(Error::new(EINVAL)).flatten()?); - - if old == new { - return Ok(0); - } - let file = { - let mut context = context.read(token.token()); - let (context, mut token) = context.token_split(); - context.remove_file(new, &mut token) - }; - if let Some(file) = file { - file.close(token)?; - } - - let mut context = context.read(token.token()); - let (context, mut token) = context.token_split(); - let mut file = - context.get_file(old, &mut token).ok_or(Error::new(EBADF))?; - context - .insert_file(new, file, &mut token) - .ok_or(Error::new(EMFILE))?; - - Ok(0) - } - FileTableVerb::Resize => { - let files = filetable.upgrade().ok_or(Error::new(EBADF))?; - let Some(&[which, size]) = metadata.get(1..3) else { - return Err(Error::new(EINVAL)); - }; - files - .write(token.token()) - .resize(which as usize, size as usize)?; - Ok(size as usize) - } - _ => Err(Error::new(EOPNOTSUPP)), - } - } - _ => Err(Error::new(EBADF)), - } - } } fn write_env_regs( diff --git a/src/scheme/sys/context.rs b/src/scheme/sys/context.rs index 5f2ed0b946..313927a08e 100644 --- a/src/scheme/sys/context.rs +++ b/src/scheme/sys/context.rs @@ -19,7 +19,7 @@ pub fn resource(token: &mut CleanLockToken) -> Result> { let (contexts, mut token) = contexts.token_split(); for context_ref in contexts.iter() { let context = context_ref.read(token.token()); - let addr_space = context.addr_space().cloned(); + let addr_space = context.addr_space().map(|a| a.clone()); let affinity = context.sched_affinity.to_string(); let cpu_time_s = context.cpu_time / crate::time::NANOS_PER_SEC; diff --git a/src/scheme/sys/cpu_stat.rs b/src/scheme/sys/cpu_stat.rs new file mode 100644 index 0000000000..3a5dd85bd2 --- /dev/null +++ b/src/scheme/sys/cpu_stat.rs @@ -0,0 +1,28 @@ +use alloc::format; +use alloc::vec::Vec; + +use crate::{ + percpu::get_all_stats, + sync::CleanLockToken, + syscall::error::{Error, Result, ENOENT}, +}; + +/// Format one per-CPU stat line compatible with Linux `/proc/stat`: +/// `user nice system idle iowait irq softirq steal`. +/// +/// Redox's scheduler tracks user, nice, kernel, idle, and irq. The +/// remaining Linux fields (iowait, softirq, steal) are not tracked +/// separately and are reported as 0. +pub fn resource_for(cpu: u32, _token: &mut CleanLockToken) -> Result> { + let stats = get_all_stats(); + for (id, stat) in stats { + if id.get() == cpu { + let line = format!( + "{} {} {} {} {} 0 0 0\n", + stat.user, stat.nice, stat.kernel, stat.idle, stat.irq + ); + return Ok(line.into_bytes()); + } + } + Err(Error::new(ENOENT)) +} diff --git a/src/scheme/sys/iostat.rs b/src/scheme/sys/iostat.rs index e3edcb66d0..d3bdd81dba 100644 --- a/src/scheme/sys/iostat.rs +++ b/src/scheme/sys/iostat.rs @@ -1,7 +1,7 @@ use crate::{ context::{ self, - memory::{handle_notify_files, Grant, PageSpan}, + memory::{Grant, PageSpan}, }, memory::PAGE_SIZE, scheme, @@ -120,7 +120,9 @@ pub fn resource(token: &mut CleanLockToken) -> Result> { { let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?); let res = addr_space.munmap(PageSpan::new(fpath_page, page_count.get()), false, token)?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } res diff --git a/src/scheme/sys/mem.rs b/src/scheme/sys/mem.rs new file mode 100644 index 0000000000..408c95b585 --- /dev/null +++ b/src/scheme/sys/mem.rs @@ -0,0 +1,20 @@ +use crate::{memory, sync::CleanLockToken, syscall::error::Result}; +use alloc::vec::Vec; + +pub fn resource(_token: &mut CleanLockToken) -> Result> { + let total_kib = memory::total_frames() as u64 * memory::PAGE_SIZE as u64 / 1024; + let free_kib = memory::free_frames() as u64 * memory::PAGE_SIZE as u64 / 1024; + + Ok(format!( + "MemTotal: {} kB\n\ +MemFree: {} kB\n\ +MemAvailable: {} kB\n\ +Buffers: 0 kB\n\ +Cached: 0 kB\n\ +SwapTotal: 0 kB\n\ +SwapFree: 0 kB\n\ +SwapCached: 0 kB\n", + total_kib, free_kib, free_kib + ) + .into_bytes()) +} diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index 5222be9907..c47e199f39 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -5,7 +5,7 @@ use ::syscall::{ dirent::{DirEntry, DirentBuf, DirentKind}, - EINVAL, EIO, EISDIR, ENOTDIR, EPERM, + EACCES, EINVAL, EIO, EISDIR, ENOTDIR, EPERM, }; use alloc::{sync::Arc, vec::Vec}; use core::str; @@ -28,15 +28,50 @@ use super::{CallerCtx, HandleMap, KernelScheme, OpenResult, StrOrBytes}; mod block; mod context; mod cpu; +mod cpu_stat; mod exe; mod fdstat; mod iostat; mod irq; mod log; +mod mem; +mod msr; mod stat; mod syscall; mod uname; +/// Extract the (cpu<<32 | msr) u64 handle stored in an MSR fd's +/// data buffer. Returns None if the fd is not an MSR fd. We clone the +/// data Arc to drop the HANDLES read lock before calling data.read() +/// (which needs &mut token). +fn decode_msr_handle(id: usize, token: &mut CleanLockToken) -> Option { + type MsrData = Arc>>>; + // Wrap the lookup in a closure so the inner `return` doesn't exit + // decode_msr_handle itself; instead it returns a value from the + // closure, which the outer let-block receives as Option. + let mut lookup = || -> Option { + let _handles = HANDLES.read(token.token()); + let h_opt = _handles.get(id).ok(); + let h = h_opt?; + if let Handle::Resource { data, path, .. } = h { + if *path == "msr" { + return Some(Arc::clone(data)); + } + } + None + }; + let data_arc: Option = lookup(); + let data: MsrData = data_arc?; + let b = data.read(token.token()); + b.as_ref().and_then(|b| { + if b.len() >= 8 { + Some(u64::from_le_bytes(b[..8].try_into().ok()?)) + } else { + None + } + }) +} + enum Handle { TopLevel, Resource { @@ -76,6 +111,7 @@ const FILES: &[(&str, Kind)] = &[ ("iostat", Rd(iostat::resource)), ("irq", Rd(irq::resource)), ("log", Rd(log::resource)), + ("mem", Rd(mem::resource)), ("numa", Rd(crate::numa::get_numa_info)), ("numa_dist", Rd(crate::numa::get_numa_distance_info)), ("numa_dom", Rd(crate::numa::get_numa_dom_info)), @@ -103,6 +139,54 @@ const FILES: &[(&str, Kind)] = &[ b"shutdown" => crate::stop::kstop(token), b"reset" => crate::stop::kreset(), b"emergency_reset" => crate::stop::emergency_reset(), + // Phase I.5: hardware-agnostic s2idle / Modern + // Standby. acpid writes "s2idle" to + // /scheme/sys/kstop. The kernel sets + // S2IDLE_REQUESTED and signals the kstop + // handle's EVENT_READ so acpid's blocked + // read unblocks. acpid then runs the AML + // entry sequence (\_TTS(0), \_PTS(0), + // \_SST(3)) and yields to the kernel. The + // kernel's idle path sees S2IDLE_REQUESTED + // and calls mwait_loop(). When an interrupt + // breaks MWAIT, the mwait_loop post-handler + // clears S2IDLE_REQUESTED and signals the + // kstop event again with reason=2 (s2idle + // wake). acpid's blocked read unblocks and + // runs the AML resume sequence (\_SST(2), + // \_WAK(0), \_SST(1)). + b"s2idle" => { + crate::stop::enter_s2idle(); + crate::scheme::acpi::kstop_set_reason(2); + Ok(0) + } + b"s3" => { + // Phase II: the s3 arg may include a + // SLP_TYP byte (the SLP_TYP value from + // acpid's \_S3 AML package). Format: arg is + // "s3X" where X is the SLP_TYP byte (0-7). + // If absent (arg.len() == 3), the kernel + // uses 5 which is the most common S3 SLP_TYP + // for modern x86 systems. + if arg.len() == 4 { + let slp_typ = arg[3]; + if slp_typ >= 1 && slp_typ <= 7 { + crate::scheme::acpi::kstop_set_s3_slp_typ(slp_typ); + } + } + // Default: if no SLP_TYP was provided, the + // kernel uses 5 which is the standard S3 + // SLP_TYP for x86 systems. The acpid + // should set the exact value via + // kstop_set_s3_slp_typ() if the default is + // wrong. + if crate::scheme::acpi::S3_SLP_TYP + .load(core::sync::atomic::Ordering::Acquire) == 0 + { + crate::scheme::acpi::kstop_set_s3_slp_typ(5); + } + crate::stop::enter_s3(token) + } _ => Err(Error::new(EINVAL)), } }), @@ -116,13 +200,17 @@ impl KernelScheme for SysScheme { } fn kopenat( &self, - _id: usize, + id: usize, user_buf: StrOrBytes, _flags: usize, _fcntl_flags: u32, ctx: CallerCtx, token: &mut CleanLockToken, ) -> Result { + if !matches!(HANDLES.read(token.token()).get(id)?, Handle::SchemeRoot) { + return Err(Error::new(EACCES)); + } + let path = user_buf .as_str() .or(Err(Error::new(EINVAL)))? @@ -131,6 +219,45 @@ impl KernelScheme for SysScheme { if path.is_empty() { let id = HANDLES.write(token.token()).insert(Handle::TopLevel); + Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) + } else if let Some(rest) = path.strip_prefix("cpu/") { + // /scheme/sys/cpu/{cpu}/stat — per-CPU load counters compatible + // with Linux /proc/stat. Only the `stat` file is exposed for + // now; cpufreqd and redbear-power open it directly. + let mut parts = rest.split('/').filter(|p| !p.is_empty()); + let cpu_str = parts.next().ok_or(Error::new(EINVAL))?; + let file = parts.next().ok_or(Error::new(EINVAL))?; + if parts.next().is_some() || file != "stat" { + return Err(Error::new(ENOENT)); + } + let cpu: u32 = cpu_str.parse().map_err(|_| Error::new(EINVAL))?; + let data = cpu_stat::resource_for(cpu, token)?; + let id = HANDLES.write(token.token()).insert(Handle::Resource { + path: "cpu_stat", + kind: Kind::Rd(|_| Ok(Vec::new())), + data: Arc::new(RwLock::new(Some(data))), + }); + Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) + } else if path == "msr" || path.starts_with("msr/") { + // /scheme/sys/msr/{cpu}/0x{msr} — Phase G.1: MSR R/W scheme + // for cpufreqd and redbear-power. Open is parse-only; reads + // and writes happen via the handle's read/write paths below. + // Pass the full "msr/{cpu}/0x{msr}" path to msr::open so it + // can do its own strip_prefix("msr") and parse the remainder. + // Stripping here (the previous behavior) left "0/0x199" + // which msr::open's strip_prefix("msr") rejected with ENOENT, + // causing every MSR open to fail and cpufreqd to oscillate. + let handle = msr::open(path, _flags, _fcntl_flags, &ctx, token)?; + // Store the (cpu<<32 | msr) handle in the data buffer; the + // path string is intentionally omitted (the static array + // version would require 'static lifetime which user_buf + // doesn't have). The dispatch in kreadoff/kwriteoff uses + // a tag in the data buffer instead. + let id = HANDLES.write(token.token()).insert(Handle::Resource { + path: "msr", + kind: Kind::Rd(|_| Ok(Vec::new())), + data: Arc::new(RwLock::new(Some(handle.to_le_bytes().to_vec()))), + }); Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) } else { //Have to iterate to get the path without allocation @@ -164,7 +291,7 @@ impl KernelScheme for SysScheme { if matches!(kind, Kind::Wr(_)) { return Ok(0); } - let is_data_none = data_lock.read(token.token()).is_none(); + let is_data_none = data_lock.write(token.token()).is_none(); if is_data_none { let new_data = kind.generate_data(token)?; let mut data_guard = data_lock.write(token.token()); @@ -189,7 +316,7 @@ impl KernelScheme for SysScheme { Handle::SchemeRoot => return Err(Error::new(EBADF)), }; - const FIRST: &[u8] = b"/scheme/sys/"; + const FIRST: &[u8] = b"sys:"; let mut bytes_read = buf.copy_common_bytes_from_slice(FIRST)?; if let Some(remaining) = buf.advance(FIRST.len()) { @@ -211,13 +338,21 @@ impl KernelScheme for SysScheme { return Ok(0); }; + // MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} read. + // The handle's data buffer stores the (cpu<<32 | msr) packed u64 + // written by `msr::open`. Decode, dispatch to msr::read. + let msr_handle: Option = decode_msr_handle(id, token); + if let Some(handle_u64) = msr_handle { + return msr::read(handle_u64, buffer, token); + } + let (kind, data_lock) = { match HANDLES.read(token.token()).get(id)? { Handle::Resource { kind, data, .. } => (*kind, data.clone()), _ => return Err(Error::new(EBADF)), } }; - let is_data_none = data_lock.read(token.token()).is_none(); + let is_data_none = data_lock.write(token.token()).is_none(); if is_data_none { let new_data = kind.generate_data(token)?; let mut data_guard = data_lock.write(token.token()); @@ -239,6 +374,13 @@ impl KernelScheme for SysScheme { _stored_flags: u32, token: &mut CleanLockToken, ) -> Result { + // MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} write. + // Dispatch to msr::write if the path is an MSR path. + let msr_handle: Option = decode_msr_handle(id, token); + if let Some(handle_u64) = msr_handle { + return msr::write(handle_u64, buffer, token); + } + let (handler, intermediate, len) = match HANDLES.read(token.token()).get(id)? { Handle::TopLevel | Handle::Resource { @@ -295,7 +437,7 @@ impl KernelScheme for SysScheme { } }; let stat = if let Some((kind, data_lock)) = stat_base { - let is_data_none = data_lock.read(token.token()).is_none(); + let is_data_none = data_lock.write(token.token()).is_none(); if is_data_none { let new_data = kind.generate_data(token)?; let mut data_guard = data_lock.write(token.token()); diff --git a/src/scheme/sys/msr.rs b/src/scheme/sys/msr.rs new file mode 100644 index 0000000000..94e48316b4 --- /dev/null +++ b/src/scheme/sys/msr.rs @@ -0,0 +1,114 @@ +//! /scheme/sys/msr — Model-Specific Register R/W +//! +//! Path layout: `/scheme/sys/msr/{cpu}/0x{msr_hex}`. Open for write, +//! then write 8 bytes (little-endian u64). Read for read returns the +//! current register value. +//! +//! CPU enumeration comes from `crate::cpu::cpu_count()` (8 by default +//! on QEMU Arrow Lake sim, up to 16 on the real LG Gram). The exact +//! set of accessible MSRs depends on CPU features; the MSR read/write +//! uses `core::arch::x86_64::{rdmsr, wrmsr}` (in userspace we use the +//! `scheme:msr` interface for ring-3 access, but this kernel-side +//! helper is for the scheme to forward requests to the active CPU). +//! +use x86::msr::{rdmsr, wrmsr}; + +use crate::cpu_count; +use crate::ipi::{ipi_single, IpiKind}; +use crate::percpu::PercpuBlock; +use syscall::{ + error::{Error, Result, EBUSY, EBADF, EINVAL, ENOENT, EPERM, ETIMEDOUT}, +}; +use crate::scheme::CallerCtx; +use crate::sync::CleanLockToken; +use crate::syscall::usercopy::{UserSliceRo, UserSliceWo}; + +const MSR_TIMEOUT_SPINS: usize = 1_000_000; + +/// Open: `msr/{cpu}/0x{msr}` (read or write, root only). +pub fn open( + path: &str, + _flags: usize, + _fcntl_flags: u32, + caller: &CallerCtx, + _token: &mut CleanLockToken, +) -> Result { + if caller.uid != 0 { + return Err(Error::new(EPERM)); + } + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new(EBADF)); + } + let rest = trimmed.strip_prefix("msr").ok_or(Error::new(ENOENT))?; + let rest = rest.trim_matches('/'); + let mut parts = rest.split('/'); + let cpu_str = parts.next().ok_or(Error::new(EINVAL))?; + let msr_str = parts.next().ok_or(Error::new(EINVAL))?; + if parts.next().is_some() { + return Err(Error::new(EINVAL)); + } + let cpu: u32 = cpu_str.parse().map_err(|_| Error::new(EINVAL))?; + let msr_clean = msr_str.strip_prefix("0x").unwrap_or(msr_str); + let msr = u32::from_str_radix(msr_clean, 16).map_err(|_| Error::new(EINVAL))?; + if cpu >= cpu_count() { + return Err(Error::new(EINVAL)); + } + Ok(((cpu as u64) << 32) | (msr as u64)) +} + +pub fn read(handle: u64, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result { + let cpu = (handle >> 32) as u32; + let msr = (handle & 0xFFFFFFFF) as u32; + let value = access_msr(cpu, msr, None)?; + let bytes = value.to_le_bytes(); + let n = buf.copy_common_bytes_from_slice(&bytes)?; + Ok(n) +} + +pub fn write(handle: u64, buf: UserSliceRo, _token: &mut CleanLockToken) -> Result { + let cpu = (handle >> 32) as u32; + let msr = (handle & 0xFFFFFFFF) as u32; + let mut bytes = [0u8; 8]; + let n = buf.copy_common_bytes_to_slice(&mut bytes)?; + let value = u64::from_le_bytes(bytes); + let _ = access_msr(cpu, msr, Some(value))?; + Ok(n) +} + +fn access_msr(cpu: u32, msr: u32, write: Option) -> Result { + let current_cpu = PercpuBlock::current().cpu_id.get(); + if cpu == current_cpu { + return unsafe { + if let Some(value) = write { + wrmsr(msr, value); + Ok(value) + } else { + Ok(rdmsr(msr)) + } + }; + } + + let Some(target) = crate::percpu::get_percpu_block(crate::cpu_set::LogicalCpuId::new(cpu)) + else { + return Err(Error::new(EINVAL)); + }; + + let mailbox = target.msr_mailbox(); + if !mailbox.wait_done() { + return Err(Error::new(EBUSY)); + } + + mailbox.begin_request(cpu, msr, write.is_some(), write.unwrap_or(0)); + ipi_single(IpiKind::Msr, target); + + for _ in 0..MSR_TIMEOUT_SPINS { + if mailbox.wait_done() { + let value = mailbox.read_value(); + return Ok(value); + } + core::hint::spin_loop(); + } + + Err(Error::new(ETIMEDOUT)) +} diff --git a/src/scheme/sys/stat.rs b/src/scheme/sys/stat.rs index e3e3ce473d..877d252f4f 100644 --- a/src/scheme/sys/stat.rs +++ b/src/scheme/sys/stat.rs @@ -1,12 +1,11 @@ use core::fmt::Write as _; use crate::{ - context::get_contexts_stats, + context::{contexts, Status}, cpu_stats::{get_context_switch_count, get_contexts_count, irq_counts}, - event::get_event_stat, percpu::get_all_stats, sync::CleanLockToken, - syscall::{error::Result, futex::get_futex_stat}, + syscall::error::Result, time::START, }; use alloc::{string::String, vec::Vec}; @@ -15,25 +14,14 @@ use alloc::{string::String, vec::Vec}; pub fn resource(token: &mut CleanLockToken) -> Result> { let start_time_sec = *START.lock(token.token()) / 1_000_000_000; - let (contexts_alive, contexts_running, contexts_blocked) = get_contexts_stats(token); - let (event_keys, event_subs) = get_event_stat(token); - let (futex_keys, futex_subs) = get_futex_stat(token); - let pipe_subs = crate::scheme::pipe::get_pipe_stat(token); - let timeout_subs = crate::context::timeout::get_timeout_stat(token); + let (contexts_running, contexts_blocked) = get_contexts_stats(token); let res = format!( "{}{}\n\ boot_time: {start_time_sec}\n\ context_switches: {}\n\ contexts_created: {}\n\ - contexts_alive: {contexts_alive}\n\ contexts_running: {contexts_running}\n\ - contexts_blocked: {contexts_blocked}\n\ - event_registries: {event_keys}\n\ - event_subcribers: {event_subs}\n\ - futex_registries: {futex_keys}\n\ - futex_subcribers: {futex_subs}\n\ - pipe_subcribers: {pipe_subs}\n\ - timeout_subscribers: {timeout_subs}\n", + contexts_blocked: {contexts_blocked}", get_cpu_stats(), get_irq_stats(), get_context_switch_count(), @@ -82,3 +70,27 @@ fn get_irq_stats() -> String { output } + +/// Format contexts stats. +fn get_contexts_stats(token: &mut CleanLockToken) -> (u64, u64) { + let mut running = 0; + let mut blocked = 0; + + let statuses = { + let mut contexts = contexts(token.downgrade()); + let (contexts, mut token) = contexts.token_split(); + contexts + .iter() + .map(|context| context.read(token.token()).status.clone()) + .collect::>() + }; + + for status in statuses { + if matches!(status, Status::Runnable) { + running += 1; + } else if !matches!(status, Status::Dead { .. }) { + blocked += 1; + } + } + (running, blocked) +} diff --git a/src/scheme/time.rs b/src/scheme/time.rs index 2a76a6d843..4ec3aed320 100644 --- a/src/scheme/time.rs +++ b/src/scheme/time.rs @@ -157,7 +157,10 @@ impl KernelScheme for TimeScheme { (CLOCK_MONOTONIC, TimeSchemeKind::ClockGetres) => time::monotonic_resolution(), _ => return Err(Error::new(EINVAL)), }; - let time = TimeSpec::from_nanos(arch_time); + let time = TimeSpec { + tv_sec: (arch_time / time::NANOS_PER_SEC) as i64, + tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32, + }; current_chunk.copy_exactly(&time)?; bytes_read += size_of::(); diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 2141f83d2c..f09b21665a 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -16,11 +16,11 @@ use syscall::{ use crate::{ context::{ self, - context::{bulk_insert_fds, HardBlockedReason}, + context::{bulk_add_fds, bulk_insert_fds, HardBlockedReason}, file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription}, memory::{ - handle_notify_files, AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, - GrantFileRef, MmapMode, PageSpan, UnmapVec, DANGLING, + AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode, + PageSpan, DANGLING, }, BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status, }, @@ -167,7 +167,8 @@ impl UserInner { fn next_id(&self, token: &mut CleanLockToken) -> Result { let idx = { let mut states = self.states.lock(token.token()); - states.insert(State::Placeholder) + let idx = states.insert(State::Placeholder); + idx }; // TODO: implement blocking? @@ -723,7 +724,8 @@ impl UserInner { pub fn write(&self, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { let mut bytes_read = 0; for chunk in buf.in_exact_chunks(size_of::()) { - match ParsedCqe::parse_cqe(&unsafe { chunk.read_exact::()? }) + let cqe = unsafe { chunk.read_exact::()? }; + match ParsedCqe::parse_cqe(&cqe) .and_then(|p| self.handle_parsed(&p, token)) { Ok(()) => bytes_read += size_of::(), @@ -741,8 +743,6 @@ impl UserInner { flags: MapFlags, token: &mut CleanLockToken, ) -> Result<()> { - info!("REQUEST FMAP"); - let tag = self.next_id(token)?; { let mut states = self.states.lock(token.token()); @@ -779,20 +779,27 @@ impl UserInner { ParsedCqe::RespondAndNotifyOnDetach { tag, res, extra0 } => { self.respond(tag, Response::Regular(res, extra0, true), token)? } - ParsedCqe::ResponseWithFd { tag, fd } => self.respond( + ParsedCqe::ResponseWithFd { tag, fd } => { + self.respond( tag, Response::Fd({ - { + let removed = { let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context.remove_file(FileHandle::from(fd), &mut token) + }; + match removed { + Some(desc) => desc.description, + None => { + error!("EINVAL ResponseWithFd: tag={} fd={} not in procmgr fd table", tag, fd); + return Err(Error::new(EINVAL)); + } } - .ok_or(Error::new(EINVAL))? - .description }), token, - )?, + )?; + }, ParsedCqe::ResponseWithMultipleFds { tag, num_fds: _ } => { self.respond(tag, Response::MultipleFds(None), token)?; } @@ -819,14 +826,33 @@ impl UserInner { }; // FIXME: Description can leak if there is no additional file table space. - let current_lock = context::current(); - let mut current = current_lock.read(token.token()); - let (context, mut token) = current.token_split(); - context.insert_file( - FileHandle::from(dst_fd_or_ptr), - FileDescriptor { description }, - &mut token, - ); + if flags.contains(FobtainFdFlags::MANUAL_FD) { + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + context.insert_file( + FileHandle::from(dst_fd_or_ptr), + FileDescriptor { + description, + cloexec: true, + }, + &mut token, + ); + } else { + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + let fd = context + .add_file( + FileDescriptor { + description, + cloexec: true, + }, + &mut token, + ) + .ok_or(Error::new(EMFILE))?; + UserSlice::wo(dst_fd_or_ptr, size_of::())?.write_usize(fd.get())?; + } } ParsedCqe::ProvideMmap { tag, @@ -834,11 +860,6 @@ impl UserInner { base_addr, page_count, } => { - info!( - "PROVIDE_MAP {:x} {:x} {:?} {:x}", - tag, offset, base_addr, page_count - ); - if offset % PAGE_SIZE as u64 != 0 { return Err(Error::new(EINVAL)); } @@ -912,6 +933,11 @@ impl UserInner { State::Placeholder => return Err(Error::new(EBADFD)), // invalid scheme to kernel call old_state @ (State::Responded(_) | State::Fmap(_)) => { + error!("EINVAL respond: tag={} state={:?}", tag, match &old_state { + State::Responded(_) => "Responded", + State::Fmap(_) => "Fmap", + _ => unreachable!(), + }); *o = old_state; return Err(Error::new(EINVAL)); } @@ -967,7 +993,9 @@ impl UserInner { let unpin = true; let res = AddrSpace::current()?.munmap(callee_responsible, unpin, token)?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } }, // invalid state @@ -1096,7 +1124,7 @@ impl UserInner { }; let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0"); - let mut notify_files = UnmapVec::new(); + let mut notify_files = Vec::new(); // TODO: Not a Lock ordering violation // we've checked Arc::ptr_eq(&src_address_space, &dst_addr_space) before, // but it's difficult to apply src.arquire_rewrite @@ -1122,7 +1150,9 @@ impl UserInner { )? }; - handle_notify_files(notify_files, token); + for map in notify_files { + let _ = map.unmap(token); + } Ok(dst_base.start_address().data()) } @@ -1131,6 +1161,7 @@ impl UserInner { &self, descs: Vec>, flags: CallFlags, + _arg: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { @@ -1212,6 +1243,9 @@ impl UserInner { if flags.contains(CallFlags::FD_EXCLUSIVE) { obtainfd_flags |= FobtainFdFlags::EXCLUSIVE; } + if flags.contains(CallFlags::FD_CLOEXEC) { + obtainfd_flags |= FobtainFdFlags::CLOEXEC; + } self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags, token) } _ => Err(Error::new(EINVAL)), @@ -1236,7 +1270,21 @@ impl UserInner { }; let mut token = token.downgrade(); - let num_fds = bulk_insert_fds(descriptions, payload, &mut token.token())?; + let num_fds = if flags.contains(FobtainFdFlags::UPPER_TBL) { + bulk_insert_fds( + descriptions, + payload, + flags.contains(FobtainFdFlags::CLOEXEC), + &mut token.token(), + )? + } else { + bulk_add_fds( + descriptions, + payload, + flags.contains(FobtainFdFlags::CLOEXEC), + &mut token.token(), + )? + }; Ok(num_fds) } @@ -1299,13 +1347,12 @@ impl CaptureGuard { dst.copy_from_slice(&src.buf()[..dst.len()])?; } let unpin = true; - if let Some(addrsp) = self.addrsp.take() { - if !self.span.is_empty() { - let res = addrsp.munmap(self.span, unpin, token)?; - handle_notify_files(res, token); - } - if let Some(addrsp) = Arc::into_inner(addrsp) { - addrsp.into_drop(token); + if let Some(ref addrsp) = self.addrsp + && !self.span.is_empty() + { + let res = addrsp.munmap(self.span, unpin, token)?; + for r in res { + let _ = r.unmap(token); } } @@ -1313,6 +1360,11 @@ impl CaptureGuard { } pub fn release(mut self, token: &mut CleanLockToken) -> Result<()> { self.release_inner(token)?; + if let Some(addrsp) = self.addrsp.take() + && let Some(addrsp) = Arc::into_inner(addrsp) + { + addrsp.into_drop(token); + } if let Some(src) = self.head.src.take() { src.into_drop(token); } @@ -1432,6 +1484,21 @@ impl KernelScheme for UserScheme { .map(|o| o as u64) } + fn fchmod(&self, file: usize, mode: u16, token: &mut CleanLockToken) -> Result<()> { + let ctx = { context::current().read(token.token()).caller_ctx() }; + self.inner + .call( + ctx, + Vec::new(), + Opcode::Fchmod, + [file, mode as usize], + &mut PageSpan::empty(), + token, + )? + .into_regular()?; + Ok(()) + } + fn fchown(&self, file: usize, uid: u32, gid: u32, token: &mut CleanLockToken) -> Result<()> { { let ctx = context::current(); @@ -1566,6 +1633,21 @@ impl KernelScheme for UserScheme { Ok(()) } + fn ftruncate(&self, file: usize, len: usize, token: &mut CleanLockToken) -> Result<()> { + let ctx = { context::current().read(token.token()).caller_ctx() }; + self.inner + .call( + ctx, + Vec::new(), + Opcode::Ftruncate, + [file, len], + &mut PageSpan::empty(), + token, + )? + .into_regular()?; + Ok(()) + } + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { self.inner.todo.send( Sqe { @@ -1728,18 +1810,61 @@ impl KernelScheme for UserScheme { result } + fn kfutimens( + &self, + file: usize, + buf: UserSliceRo, + token: &mut CleanLockToken, + ) -> Result { + let ctx = { context::current().read(token.token()).caller_ctx() }; + let mut address = self.inner.capture_user(buf, token)?; + let result = self + .inner + .call( + ctx, + Vec::new(), + Opcode::Futimens, + [file, address.base(), address.len()], + address.span(), + token, + )? + .into_regular(); + address.release(token)?; + result + } fn getdents( &self, - _file: usize, - _buf: UserSliceWo, - _header_size: u16, - _opaque_id_start: u64, - _token: &mut CleanLockToken, + file: usize, + buf: UserSliceWo, + header_size: u16, + opaque_id_start: u64, + token: &mut CleanLockToken, ) -> Result { - // SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which - // overrides translate_stdfscall - error!("getdents should be unreachable for UserScheme"); - Err(Error::new(EBADFD)) + let ctx = { context::current().read(token.token()).caller_ctx() }; + let mut address = self.inner.capture_user(buf, token)?; + // TODO: Support passing the 16-byte record_len of the last dent, to make it possible to + // iterate backwards without first interating forward? The last entry will contain the + // opaque id to pass to the next getdents. Since this field is small, this would fit in the + // extra_raw field of `Cqe`s. + let result = self + .inner + .call( + ctx, + Vec::new(), + Opcode::Getdents, + [ + file, + address.base(), + address.len(), + header_size.into(), + opaque_id_start as usize, + ], + address.span(), + token, + )? + .into_regular(); + address.release(token)?; + result } fn kfstat(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { let ctx = { context::current().read(token.token()).caller_ctx() }; @@ -1758,16 +1883,22 @@ impl KernelScheme for UserScheme { address.release(token)?; result.map(|_| ()) } - fn kfstatvfs( - &self, - _file: usize, - _stat: UserSliceWo, - _token: &mut CleanLockToken, - ) -> Result<()> { - // SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which - // overrides translate_stdfscall - error!("kfstatvfs should be unreachable for UserScheme"); - Err(Error::new(EBADFD)) + fn kfstatvfs(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let ctx = { context::current().read(token.token()).caller_ctx() }; + let mut address = self.inner.capture_user(stat, token)?; + let result = self + .inner + .call( + ctx, + Vec::new(), + Opcode::Fstatvfs, + [file, address.base(), address.len()], + address.span(), + token, + )? + .into_regular(); + address.release(token)?; + result.map(|_| ()) } fn kfmap( &self, @@ -1811,41 +1942,28 @@ impl KernelScheme for UserScheme { metadata: &[u64], token: &mut CleanLockToken, ) -> Result { + let id = fds.first().copied().ok_or(Error::new(EINVAL))?; let inner = self.inner.clone(); let mut address = inner.capture_user(payload, token)?; let ctx = { context::current().read(token.token()).caller_ctx() }; - let mut sqe_flags = SqeFlags::empty(); - let mut last_arg = 0; - if fds.len() == 2 { - sqe_flags |= SqeFlags::MULTIPLE_IDS; - last_arg = fds[1] as u64; - } else if fds.len() > 2 || fds.is_empty() { - return Err(Error::new(EINVAL)); - } - let mut sqe = Sqe { opcode: Opcode::Call as u8, - sqe_flags, + sqe_flags: SqeFlags::empty(), _rsvd: 0, tag: inner.next_id(token)?, caller: ctx.pid as u64, args: [ - fds[0] as u64, + id as u64, address.base() as u64, address.len() as u64, 0, 0, - last_arg, + 0, ], }; - - if fds.len() == 2 { - let dst = &mut sqe.args[3..5]; - let len = dst.len().min(metadata.len()); - dst[..len].copy_from_slice(&metadata[..len]); - } else { + { let dst = &mut sqe.args[3..]; let len = dst.len().min(metadata.len()); dst[..len].copy_from_slice(&metadata[..len]); @@ -1863,7 +1981,7 @@ impl KernelScheme for UserScheme { } fn kstdfscall( &self, - fds: &[usize], + id: usize, _kind: StdFsCallKind, desc: Arc, payload: UserSliceRw, @@ -1876,35 +1994,22 @@ impl KernelScheme for UserScheme { let mut address = inner.capture_user(payload, token)?; let ctx = { context::current().read(token.token()).caller_ctx() }; - let mut sqe_flags = SqeFlags::empty(); - let mut last_arg = 0; - if fds.len() == 2 { - sqe_flags |= SqeFlags::MULTIPLE_IDS; - last_arg = fds[1] as u64; - } else if fds.len() > 2 || fds.is_empty() { - return Err(Error::new(EINVAL)); - } - let mut sqe = Sqe { opcode: Opcode::StdFsCall as u8, - sqe_flags, + sqe_flags: SqeFlags::empty(), _rsvd: 0, tag: inner.next_id(token)?, caller: ctx.pid as u64, args: [ - fds[0] as u64, + id as u64, address.base() as u64, address.len() as u64, 0, 0, - last_arg, + 0, ], }; - if fds.len() == 2 { - let dst = &mut sqe.args[3..5]; - let len = dst.len().min(metadata.len()); - dst[..len].copy_from_slice(&metadata[..len]); - } else { + { let dst = &mut sqe.args[3..]; let len = dst.len().min(metadata.len()); dst[..len].copy_from_slice(&metadata[..len]); @@ -1929,7 +2034,8 @@ impl KernelScheme for UserScheme { number: usize, descs: Vec>, flags: CallFlags, - metadata: &[u64], + arg: u64, + _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { let inner = self.inner.clone(); @@ -1946,12 +2052,7 @@ impl KernelScheme for UserScheme { ctx, descs, Opcode::Sendfd, - [ - number, - sendfd_flags.bits(), - *metadata.first().unwrap_or(&0) as usize, - len, - ], + [number, sendfd_flags.bits(), arg as usize, len], &mut PageSpan::empty(), token, )? @@ -1962,7 +2063,7 @@ impl KernelScheme for UserScheme { id: usize, payload: UserSliceRw, flags: CallFlags, - metadata: &[u64], + _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { let inner = self.inner.clone(); @@ -1974,19 +2075,17 @@ impl KernelScheme for UserScheme { if flags.contains(CallFlags::FD_UPPER) { recvfd_flags |= RecvFdFlags::UPPER_TBL; } + if flags.contains(CallFlags::FD_CLOEXEC) { + recvfd_flags |= RecvFdFlags::CLOEXEC; + } + let ctx = { context::current().read(token.token()).caller_ctx() }; let len = payload.len() / size_of::(); let res = inner.call( ctx, Vec::new(), Opcode::Recvfd, - [ - id, - recvfd_flags.bits(), - len, - *metadata.first().unwrap_or(&0) as usize, - *metadata.get(1).unwrap_or(&0) as usize, - ], + [id, recvfd_flags.bits(), len], &mut PageSpan::empty(), token, )?; @@ -2004,17 +2103,30 @@ impl KernelScheme for UserScheme { let mut token = token.downgrade(); let num_fds = if let Some(descriptions) = descriptions_opt { - bulk_insert_fds(descriptions, payload, &mut token)? + if recvfd_flags.contains(RecvFdFlags::UPPER_TBL) { + bulk_insert_fds( + descriptions, + payload, + recvfd_flags.contains(RecvFdFlags::CLOEXEC), + &mut token, + )? + } else { + bulk_add_fds( + descriptions, + payload, + recvfd_flags.contains(RecvFdFlags::CLOEXEC), + &mut token, + )? + } } else { 0 }; Ok(num_fds) } - fn translate_std_fs_call( &self, - fds: &[usize], + id: usize, desc: Arc, payload: UserSliceRw, flags: CallFlags, @@ -2028,7 +2140,7 @@ impl KernelScheme for UserScheme { return Err(Error::new(EOPNOTSUPP)); }; let metadata = StdFsCallMeta::new(kind, arg1, arg2); - self.kstdfscall(fds, kind, desc, payload, flags, metadata, token) + self.kstdfscall(id, kind, desc, payload, flags, metadata, token) } } diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 2a9fa4b589..90f6801fde 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,7 +1,6 @@ use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, - numa, startup::{memory::BootloaderMemoryKind::Null, KernelArgs}, }; use core::{ @@ -75,14 +74,16 @@ impl MemoryEntry { } struct MemoryMap { - entries: [MemoryEntry; 512], + entries: [MemoryEntry; 1024], size: usize, } impl MemoryMap { fn register(&mut self, base: usize, size: usize, kind: BootloaderMemoryKind) { if self.size >= self.entries.len() { - panic!("Early memory map overflow!"); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + unsafe { core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'!', options(nostack, preserves_flags)); } + panic!("Early memory map overflow at entry {} (max {})", self.size, self.entries.len()); } let start = if kind == BootloaderMemoryKind::Free { align_up(base) @@ -135,7 +136,7 @@ static MEMORY_MAP: SyncUnsafeCell = SyncUnsafeCell::new(MemoryMap { start: 0, end: 0, kind: BootloaderMemoryKind::Null, - }; 512], + }; 1024], size: 0, }); diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 7b3f532fa4..42cd38760e 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -233,7 +233,6 @@ fn run_userspace(token: &mut CleanLockToken) -> ! { interrupt::enable_and_nop(); } SwitchResult::AllContextsIdle => { - // Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired. interrupt::enable_and_halt(); } } diff --git a/src/sync/ordered.rs b/src/sync/ordered.rs index 1bcfa4926f..0b8ebda894 100644 --- a/src/sync/ordered.rs +++ b/src/sync/ordered.rs @@ -271,6 +271,38 @@ impl Mutex { }) } + /// Arcquires the lock_token to replace older MutexGuard. + /// SAFETY: Caller must guarantee lock_token is coming from MutexWriteGuard::into_token() from the same lock. + /// OR Caller must guarantee lock_token is coming from different lock, which can happen when two lock need to copy data each other. + pub unsafe fn relock<'a>(&'a self, lock_token: LockToken<'a, L>) -> MutexGuard<'a, L, T> { + let inner = { + #[cfg(feature = "busy_panic")] + let mut i = DEADLOCK_SPIN_CAP; + let my_percpu = PercpuBlock::current(); + + loop { + match self.inner.try_lock() { + Some(inner) => break inner, + None => { + my_percpu.maybe_handle_tlb_shootdown(); + core::hint::spin_loop(); + #[cfg(feature = "busy_panic")] + { + i -= 1; + if i == 0 { + panic!("Deadlock at mutex may have triggered") + } + } + } + } + } + }; + MutexGuard { + inner, + lock_token: lock_token, + } + } + /// Consumes this Mutex, returning the underlying data. pub fn into_inner(self) -> T { self.inner.into_inner() @@ -299,6 +331,14 @@ impl<'a, L: Level, T: ?Sized + 'a> MutexGuard<'a, L, T> { pub fn into_split(self) -> (spin::MutexGuard<'a, T>, LockToken<'a, L>) { (self.inner, self.lock_token) } + + /// Merge the guard from `into_split` + pub fn from_split(lock: spin::MutexGuard<'a, T>, token: LockToken<'a, L>) -> Self { + Self { + inner: lock, + lock_token: token, + } + } } impl<'a, L: Level, T: ?Sized + 'a> core::ops::Deref for MutexGuard<'a, L, T> { @@ -553,14 +593,6 @@ impl RwLock { RwLockUpgradableGuard { inner, lock_token } } - // Unsafe due to not using token, currently required by context::switch - pub unsafe fn write_arc(self: &Arc) -> ArcRwLockWriteGuard { - core::mem::forget(self.inner.write()); - ArcRwLockWriteGuard { - rwlock: self.clone(), - } - } - // Unsafe due to not using token, currently required by context::switch pub unsafe fn try_write_arc(self: &Arc) -> Option> { let Some(guard) = self.inner.try_write() else { @@ -571,6 +603,14 @@ impl RwLock { rwlock: self.clone(), }) } + + // Unsafe due to not using token, currently required by context::switch + pub unsafe fn write_arc(self: &Arc) -> ArcRwLockWriteGuard { + core::mem::forget(self.inner.write()); + ArcRwLockWriteGuard { + rwlock: self.clone(), + } + } } /// RAII structure used to release the exclusive write access of a lock when dropped @@ -591,14 +631,6 @@ impl<'a, L: Level, T> RwLockWriteGuard<'a, L, T> { drop(self.inner); self.lock_token } - - /// Upgrade to RO lock - pub fn downgrade(self) -> RwLockReadGuard<'a, L, T> { - RwLockReadGuard { - inner: self.inner.downgrade(), - lock_token: self.lock_token, - } - } } impl core::ops::Deref for RwLockWriteGuard<'_, L, T> { @@ -673,41 +705,6 @@ impl core::ops::Deref for RwLockUpgradableGuard<'_, L, T> { } } -pub struct ArcRwLockReadGuard { - rwlock: Arc>, -} - -impl ArcRwLockReadGuard { - pub fn rwlock(s: &Self) -> &Arc> { - &s.rwlock - } -} - -impl core::ops::Deref for ArcRwLockReadGuard { - type Target = T; - - #[inline] - fn deref(&self) -> &Self::Target { - unsafe { &*self.rwlock.inner.as_mut_ptr() } - } -} - -impl core::ops::DerefMut for ArcRwLockReadGuard { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { &mut *self.rwlock.inner.as_mut_ptr() } - } -} - -impl Drop for ArcRwLockReadGuard { - #[inline] - fn drop(&mut self) { - unsafe { - self.rwlock.inner.force_read_decrement(); - } - } -} - pub struct ArcRwLockWriteGuard { rwlock: Arc>, } diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index e521c64189..c508eb64fe 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -80,13 +80,13 @@ impl WaitCondition { let mut preempt = PreemptGuardL2::new(¤t_context_ref, &mut token); let token = preempt.token(); { - let context = current_context_ref.upgradeable_read(token.token()); + let mut context = current_context_ref.write(token.token()); if let Some((control, pctl, _)) = context.sigcontrol() && control.currently_pending_unblocked(pctl) != 0 { return false; } - context.upgrade().block(reason); + context.block(reason); } self.contexts @@ -111,7 +111,7 @@ impl WaitCondition { .iter() .position(|c| Weak::as_ptr(c) == Arc::as_ptr(¤t_context_ref)) { - contexts.swap_remove(index); + contexts.remove(index); waited = false; } } diff --git a/src/sync/wait_queue.rs b/src/sync/wait_queue.rs index b049ef7849..7e2c21e2df 100644 --- a/src/sync/wait_queue.rs +++ b/src/sync/wait_queue.rs @@ -2,7 +2,7 @@ use alloc::collections::VecDeque; use syscall::{EAGAIN, EINTR}; use crate::{ - sync::{CleanLockToken, LockToken, Mutex, WaitCondition, L1, L2}, + sync::{CleanLockToken, LockToken, Mutex, MutexGuard, WaitCondition, L1, L2, L3}, syscall::{ error::{Error, Result, EINVAL}, usercopy::UserSliceWo, @@ -11,20 +11,24 @@ use crate::{ #[derive(Debug)] pub struct WaitQueue { - inner: Mutex>, + incoming: Mutex>, + outgoing: Mutex>, pub condition: WaitCondition, } impl WaitQueue { pub const fn new() -> WaitQueue { WaitQueue { - inner: Mutex::new(VecDeque::new()), + incoming: Mutex::new(VecDeque::new()), + outgoing: Mutex::new(VecDeque::new()), condition: WaitCondition::new(), } } + pub fn is_currently_empty(&self, token: &mut CleanLockToken) -> bool { - self.inner.lock(token.token()).is_empty() + self.incoming.lock(token.token()).is_empty() && self.outgoing.lock(token.token()).is_empty() } + pub fn receive_into_user( &self, buf: UserSliceWo, @@ -32,17 +36,42 @@ impl WaitQueue { reason: &'static str, token: &mut CleanLockToken, ) -> Result { + let mut out_guard = self.outgoing.lock(token.token()); loop { - let inner = self.inner.lock(token.token()); - let (mut inner, mut token) = inner.into_split(); + let (mut outgoing, mut token) = out_guard.into_split(); + if !outgoing.is_empty() { + let (s1, s2) = outgoing.as_slices(); + let s1_bytes = unsafe { + core::slice::from_raw_parts(s1.as_ptr().cast::(), size_of_val(s1)) + }; + let s2_bytes = unsafe { + core::slice::from_raw_parts(s2.as_ptr().cast::(), size_of_val(s2)) + }; - if inner.is_empty() { + let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?; + + if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) { + bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?; + } + + let _ = outgoing.drain(..bytes_copied / size_of::()); + return Ok(bytes_copied); + } + + let mut incoming = self.incoming.lock(token.token()); + + if incoming.is_empty() { if block { + drop(incoming); // SAFETY: Uses wait_inner because this inner is L2. It's guaranteed there's no other // lock held at this point because clean token is provided from caller. - if !self.condition.wait_inner(inner, reason, &mut token) { + if !self + .condition + .wait_inner(outgoing, reason, &mut token.token()) + { return Err(Error::new(EINTR)); } + out_guard = unsafe { self.outgoing.relock(token) }; continue; } else if buf.is_empty() { return Ok(0); @@ -54,21 +83,10 @@ impl WaitQueue { } } - let (s1, s2) = inner.as_slices(); - let s1_bytes = - unsafe { core::slice::from_raw_parts(s1.as_ptr().cast::(), size_of_val(s1)) }; - let s2_bytes = - unsafe { core::slice::from_raw_parts(s2.as_ptr().cast::(), size_of_val(s2)) }; + core::mem::swap(&mut *incoming, &mut outgoing); + drop(incoming); - let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?; - - if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) { - bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?; - } - - let _ = inner.drain(..bytes_copied / size_of::()); - - return Ok(bytes_copied); + out_guard = MutexGuard::from_split(outgoing, token); } } @@ -78,7 +96,7 @@ impl WaitQueue { pub fn send_locked(&self, value: T, mut token: LockToken<'_, L1>) -> usize { let len = { - let mut inner = self.inner.lock(token.token()); + let mut inner = self.incoming.lock(token.token()); inner.push_back(value); inner.len() }; diff --git a/src/syscall/debug.rs b/src/syscall/debug.rs index edf2821364..853974d95b 100644 --- a/src/syscall/debug.rs +++ b/src/syscall/debug.rs @@ -44,8 +44,8 @@ unsafe fn read_struct(ptr: usize) -> Result { //TODO: calling format_call with arguments from another process space will not work pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize) -> String { match a { - SYS_OPENAT_INTO => format!( - "openat_into({} {:?}, {:#0x}, {}, out: {})", + SYS_OPENAT => format!( + "openat({} {:?}, {:#0x}, {}, {})", b, debug_path(c, d).as_ref().map(|p| ByteStr(p.as_bytes())), e, @@ -61,11 +61,10 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g g, ), SYS_CLOSE => format!("close({})", b), - SYS_DUP_INTO => format!( - "dup_into({}, {:?}, out: {})", + SYS_DUP => format!( + "dup({}, {:?})", b, debug_buf(c, d).as_ref().map(|b| ByteStr(b)), - e, ), SYS_DUP2 => format!( "dup2({}, {}, {:?})", @@ -73,6 +72,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g c, debug_buf(d, e).as_ref().map(|b| ByteStr(b)), ), + SYS_SENDFD => format!("sendfd({}, {}, {:#0x} {:#0x} {:#0x})", b, c, d, e, f,), SYS_READ => format!("read({}, {:#X}, {})", b, c, d), SYS_READ2 => format!( "read2({}, {:#X}, {}, {}, {:?})", @@ -103,6 +103,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g }, d ), + SYS_FCHMOD => format!("fchmod({}, {:#o})", b, c), SYS_FCHOWN => format!("fchown({}, {}, {})", b, c, d), SYS_FCNTL => format!( "fcntl({}, {} ({}), {:#X})", @@ -113,6 +114,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g F_SETFD => "F_SETFD", F_SETFL => "F_SETFL", F_GETFL => "F_GETFL", + F_DUPFD_CLOEXEC => "F_DUPFD_CLOEXEC", _ => "UNKNOWN", }, c, @@ -132,7 +134,22 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g b, UserSlice::ro(c, d).and_then(|buf| unsafe { buf.read_exact::() }), ), + SYS_FSTATVFS => format!("fstatvfs({}, {:#X}, {})", b, c, d), SYS_FSYNC => format!("fsync({})", b), + SYS_FTRUNCATE => format!("ftruncate({}, {})", b, c), + SYS_FUTIMENS => format!( + "futimens({}, {:?})", + b, + UserSlice::ro(c, d).and_then(|buf| { + let mut times = vec![unsafe { buf.read_exact::()? }]; + + // One or two timespecs + if let Some(second) = buf.advance(size_of::()) { + times.push(unsafe { second.read_exact::()? }); + } + Ok(times) + }), + ), SYS_CALL => format!( "call({b}, {c:x}+{d}, {:?}, {:0x?}", CallFlags::from_bits_retain(e & !0xff), @@ -153,6 +170,15 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g e, f ), + SYS_MKNS => format!( + "mkns({:p} len: {})", + // TODO: Print out all scheme names? + + // Simply printing out simply the pointers and lengths may not provide that much useful + // debugging information, so only print the raw args. + b as *const u8, + c, + ), SYS_MPROTECT => format!("mprotect({:#X}, {}, {:?})", b, c, MapFlags::from_bits(d)), SYS_MREMAP => format!("mremap({:#X}, {:#X}, {:#X}, {:#X}, {:#X})", b, c, d, e, f), SYS_NANOSLEEP => format!( @@ -203,11 +229,12 @@ pub fn debug_start([a, b, c, d, e, f, g]: [usize; 7], token: &mut CleanLockToken #[expect(clippy::overly_complex_bool_expr)] #[expect(clippy::needless_bool)] - let do_debug = if true && { - let ctx = crate::context::current(); - let guard = ctx.read(token.token()); - guard.name.contains("init") || guard.name.contains("bootstrap") - } { + let do_debug = if false + && crate::context::current() + .read(token.token()) + .name + .contains("init") + { if a == SYS_CLOCK_GETTIME || a == SYS_YIELD || a == SYS_FUTEX { false } else if (a == SYS_WRITE || a == SYS_FSYNC) && (b == 1 || b == 2) { diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index f796517c2e..450a25a184 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -2,16 +2,14 @@ use core::num::NonZeroUsize; -use alloc::{string::String, sync::Arc, vec::Vec}; +use alloc::{string::{String, ToString}, sync::Arc, vec::Vec}; use redox_path::RedoxPath; use crate::{ context::{ self, file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription}, - memory::{ - handle_notify_files, AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions, - }, + memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions}, }, memory::{Page, VirtualAddress, PAGE_SIZE}, scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes}, @@ -64,12 +62,13 @@ pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result // TODO: Define elsewhere const PATH_MAX: usize = PAGE_SIZE; -pub fn openat_into( +pub fn openat( fh: FileHandle, raw_path: UserSliceRo, flags: usize, fcntl_flags: u32, - new_fd: FileHandle, + euid: u32, + egid: u32, token: &mut CleanLockToken, ) -> Result { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; @@ -79,11 +78,14 @@ pub fn openat_into( let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?; - let desc = pipe.description.read(token.token()); - (desc.scheme, desc.number) + let desc = pipe.description.read(token.token()); + (desc.scheme, desc.number) }; - let caller_ctx = context::current().read(token.token()).caller_ctx(); + let caller_ctx = context::current() + .read(token.token()) + .caller_ctx() + .filter_uid_gid(euid, egid); let new_description = { let scheme = scheme::get_scheme(token.token(), scheme_id)?; @@ -99,15 +101,21 @@ pub fn openat_into( match res? { OpenResult::SchemeLocal(number, internal_flags) => { - Arc::new(RwLock::new(FileDescription { - offset: 0, - internal_flags, - scheme: scheme_id, - number, - flags: flags as u32, - })) + FileDescriptor { + description: Arc::new(RwLock::new(FileDescription { + offset: 0, + internal_flags, + scheme: scheme_id, + number, + flags: (flags & !O_CLOEXEC) as u32, + })), + cloexec: flags & O_CLOEXEC == O_CLOEXEC, + } } - OpenResult::External(desc) => desc, + OpenResult::External(desc) => FileDescriptor { + description: desc, + cloexec: flags & O_CLOEXEC == O_CLOEXEC, + }, } }; @@ -115,20 +123,65 @@ pub fn openat_into( let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context - .insert_file( - new_fd, - FileDescriptor { - description: new_description, - }, + .add_file( + new_description, &mut token, ) - .ok_or(Error::new(EEXIST)) + .ok_or(Error::new(EMFILE)) +} +/// Open a file at a specific path, placing the result into a reserved fd slot (upstream 0.9.0). +pub fn openat_into( + fh: FileHandle, + raw_path: UserSliceRo, + flags: usize, + fcntl_flags: u32, + new_fd: FileHandle, + token: &mut CleanLockToken, +) -> Result { + let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; + let (scheme_id, number) = { + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?; + let desc = pipe.description.read(token.token()); + (desc.scheme, desc.number) + }; + let caller_ctx = context::current().read(token.token()).caller_ctx(); + let new_description = { + let scheme = scheme::get_scheme(token.token(), scheme_id)?; + let res = scheme.kopenat(number, StrOrBytes::from_str(&path_buf), flags, fcntl_flags, caller_ctx, token)?; + match res { + OpenResult::SchemeLocal(number, internal_flags) => { + FileDescriptor { + description: Arc::new(RwLock::new(FileDescription { + offset: 0, + internal_flags, + scheme: scheme_id, + number, + flags: (flags & !O_CLOEXEC) as u32, + })), + cloexec: flags & O_CLOEXEC == O_CLOEXEC, + } + } + OpenResult::External(desc) => FileDescriptor { + description: desc, + cloexec: flags & O_CLOEXEC == O_CLOEXEC, + }, + } + }; + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + context.insert_file(new_fd, new_description, &mut token).ok_or(Error::new(EEXIST)) } /// Unlinkat syscall pub fn unlinkat( fh: FileHandle, raw_path: UserSliceRo, flags: usize, + euid: u32, + egid: u32, token: &mut CleanLockToken, ) -> Result<()> { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; @@ -144,7 +197,10 @@ pub fn unlinkat( let scheme = scheme::get_scheme(token.token(), scheme_id)?; - let caller_ctx = context::current().read(token.token()).caller_ctx(); + let caller_ctx = context::current() + .read(token.token()) + .caller_ctx() + .filter_uid_gid(euid, egid); /* let mut path_buf = BorrowedHtBuf::head()?; @@ -170,6 +226,7 @@ pub fn close(fd: FileHandle, token: &mut CleanLockToken) -> Result<()> { fn duplicate_file( fd: FileHandle, user_buf: UserSliceRo, + cloexec: bool, token: &mut CleanLockToken, ) -> Result { let (caller_ctx, file) = { @@ -185,6 +242,7 @@ fn duplicate_file( if user_buf.is_empty() { Ok(FileDescriptor { description: Arc::clone(&file.description), + cloexec, }) } else { let description = { *file.description.read(token.token()) }; @@ -208,24 +266,45 @@ fn duplicate_file( Ok(FileDescriptor { description: new_description, + cloexec, }) } } /// Duplicate file descriptor -pub fn dup_into( - fd: FileHandle, - new_fd: FileHandle, - buf: UserSliceRo, - token: &mut CleanLockToken, -) -> Result { - let new_file = duplicate_file(fd, buf, token)?; +pub fn dup(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + let new_file = duplicate_file(fd, buf, false, token)?; let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context - .insert_file(new_fd, new_file, &mut token) - .ok_or(Error::new(EEXIST)) + .add_file(new_file, &mut token) + .ok_or(Error::new(EMFILE)) +} + +/// Duplicate a file descriptor, placing into a specific fd slot (upstream 0.9.0). +/// If the target slot is occupied, the existing descriptor is closed first +/// (POSIX `dup3` semantics, consistent with `dup2` below). +pub fn dup_into( + fd: FileHandle, + new_cd: FileHandle, + buf: UserSliceRo, + token: &mut CleanLockToken, +) -> Result { + if fd == new_cd { + return Ok(new_cd); + } + + let _ = close(new_cd, token); + + let new_file = duplicate_file(fd, buf, false, token)?; + + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + context + .insert_file(new_cd, new_file, &mut token) + .ok_or(Error::new(EMFILE)) } /// Duplicate file descriptor, replacing another @@ -239,7 +318,7 @@ pub fn dup2( Ok(new_fd) } else { let _ = close(new_fd, token); - let new_file = duplicate_file(fd, buf, token)?; + let new_file = duplicate_file(fd, buf, false, token)?; let current_lock = context::current(); let mut current = current_lock.read(token.token()); @@ -250,7 +329,7 @@ pub fn dup2( } } pub fn call( - fds: &[usize], + fd: FileHandle, payload: UserSliceRw, flags: CallFlags, metadata: UserSliceRo, @@ -265,103 +344,42 @@ pub fn call( match flags { f if f.contains(CallFlags::WRITE | CallFlags::FD) => { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - call_fdwrite( - FileHandle::from(fds[0]), - payload, - flags, - &meta[..copied / 8], - token, - ) + call_fdwrite(fd, payload, flags, &meta[..copied / 8], token) } f if f.contains(CallFlags::READ | CallFlags::FD) => { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - call_fdread( - FileHandle::from(fds[0]), - payload, - flags, - &meta[..copied / 8], - token, - ) + call_fdread(fd, payload, flags, &meta[..copied / 8], token) } - _ => call_normal(fds, payload, flags, &meta[..copied / 8], token), + _ => call_normal(fd, payload, flags, &meta[..copied / 8], token), } } fn call_normal( - fds: &[usize], + fd: FileHandle, payload: UserSliceRw, flags: CallFlags, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { - if fds.len() > 2 || fds.is_empty() { - return Err(Error::new(EINVAL)); - } - - let mut nums = arrayvec::ArrayVec::<_, 2>::new(); - - let current_lock = context::current(); - let consume = flags.contains(CallFlags::CONSUME); - let mut fds = fds.iter(); - - let (target_file, scheme_id) = { - let fd = FileHandle::from(fds.next().copied().unwrap()); + let file = { + let current_lock = context::current(); let mut current = current_lock.read(token.token()); - - let (file, mut split_token) = match (current.token_split(), consume) { - ((ctxt, mut split_token), true) => { - (ctxt.remove_file(fd, &mut split_token), split_token) - } - ((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token), - }; - let file = file.ok_or(Error::new(EBADF))?; - - let desc = file.description.read(split_token.token()); - - let scheme_id = desc.scheme; - nums.push(desc.number); - drop(desc); - let target_file = file; - (target_file, scheme_id) - }; - - for &fd in fds { - let fd = FileHandle::from(fd); - let mut current = current_lock.read(token.token()); - - let (file, mut split_token) = match (current.token_split(), consume) { - ((ctxt, mut split_token), true) => { - (ctxt.remove_file(fd, &mut split_token), split_token) - } - ((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token), - }; - let file = file.ok_or(Error::new(EBADF))?; - - let desc = file.description.read(split_token.token()); - if desc.scheme != scheme_id { - return Err(Error::new(EXDEV)); + match (current.token_split(), flags.contains(CallFlags::CONSUME)) { + ((ctxt, mut token), true) => ctxt.remove_file(fd, &mut token), + ((ctxt, mut token), false) => ctxt.get_file(fd, &mut token), } - nums.push(desc.number); } + .ok_or(Error::new(EBADF))?; + let (scheme_id, number) = { + let desc = file.description.read(token.token()); + (desc.scheme, desc.number) + }; let scheme = scheme::get_scheme(token.token(), scheme_id)?; if flags.contains(CallFlags::STD_FS) { - scheme.translate_std_fs_call( - &nums, - target_file.description, - payload, - flags, - metadata, - token, - ) + scheme.translate_std_fs_call(number, file.description, payload, flags, metadata, token) } else { - scheme.kcall(&nums, payload, flags, metadata, token) + scheme.kcall(&[number], payload, flags, metadata, token) } } @@ -382,13 +400,28 @@ fn call_fdwrite( let len = fds.len(); + fdwrite_inner(fd, fds, flags, 0, metadata, token)?; + + Ok(len) +} + +fn fdwrite_inner( + socket: FileHandle, + target_fds: Vec, + flags: CallFlags, + arg: u64, + metadata: &[u64], + token: &mut CleanLockToken, +) -> Result { // TODO: Ensure deadlocks can't happen let (scheme, number, descs_to_send) = { let (scheme, number) = { let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); - let file_descriptor = context.get_file(fd, &mut token).ok_or(Error::new(EBADF))?; + let file_descriptor = context + .get_file(socket, &mut token) + .ok_or(Error::new(EBADF))?; let desc = &file_descriptor.description.read(token.token()); (desc.scheme, desc.number) }; @@ -401,9 +434,9 @@ fn call_fdwrite( scheme, number, if flags.contains(CallFlags::FD_CLONE) { - context.bulk_get_files(&fds, &mut token) + context.bulk_get_files(&target_fds, &mut token) } else { - context.bulk_remove_files(&fds, &mut token) + context.bulk_remove_files(&target_fds, &mut token) }? .into_iter() .map(|f| f.description) @@ -426,9 +459,7 @@ fn call_fdwrite( CallFlags::empty() }; - scheme.kfdwrite(number, descs_to_send, flags_to_scheme, metadata, token)?; - - Ok(len) + scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata, token) } fn call_fdread( @@ -455,6 +486,24 @@ fn call_fdread( scheme.kfdread(number, payload, flags, metadata, token) } +pub fn sendfd( + socket: FileHandle, + fd: FileHandle, + flags_raw: usize, + arg: u64, + token: &mut CleanLockToken, +) -> Result { + let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?; + let mut call_flags = CallFlags::FD | CallFlags::WRITE; + if sendfd_flags.contains(SendFdFlags::CLONE) { + call_flags |= CallFlags::FD_CLONE; + } + if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) { + call_flags |= CallFlags::FD_EXCLUSIVE; + } + fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[], token) +} + /// File descriptor controls pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) -> Result { let file = { @@ -466,19 +515,19 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) .ok_or(Error::new(EBADF))?; let (scheme_id, number, flags) = { - let desc = file.description.read(token.token()); + let desc = file.description.write(token.token()); (desc.scheme, desc.number, desc.flags) }; - if cmd == F_DUPFD { + if cmd == F_DUPFD || cmd == F_DUPFD_CLOEXEC { // Not in match because 'files' cannot be locked - let new_file = duplicate_file(fd, UserSlice::empty(), token)?; + let new_file = duplicate_file(fd, UserSlice::empty(), cmd == F_DUPFD_CLOEXEC, token)?; let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); return context - .insert_file(FileHandle::from(arg), new_file, &mut token) + .add_file_min(new_file, arg, &mut token) .ok_or(Error::new(EMFILE)) .map(FileHandle::into); } @@ -500,8 +549,17 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) let (files, mut token) = files.token_split(); match *files.get_mut(fd.get()).ok_or(Error::new(EBADF))? { Some(ref mut file) => match cmd { - F_GETFD => Ok(0), - F_SETFD => Ok(0), + F_GETFD => { + if file.cloexec { + Ok(O_CLOEXEC) + } else { + Ok(0) + } + } + F_SETFD => { + file.cloexec = arg & O_CLOEXEC == O_CLOEXEC; + Ok(0) + } F_GETFL => Ok(flags as usize), F_SETFL => { let new_flags = (flags & O_ACCMODE as u32) | (arg as u32 & !O_ACCMODE as u32); @@ -630,7 +688,10 @@ pub fn funmap(virtual_address: usize, length: usize, token: &mut CleanLockToken) .ok_or(Error::new(EINVAL))?; let unpin = false; let notify = addr_space.munmap(span, unpin, token)?; - handle_notify_files(notify, token); + + for map in notify { + let _ = map.unmap(token); + } Ok(0) } @@ -756,7 +817,7 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) -> let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) { desc.offset } else { - u64::MAX + 0 }; Ok(( scheme.kreadoff(desc.number, buf, offset, desc.flags, desc.flags, token)?, @@ -768,18 +829,32 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) -> let offset = &mut desc_arc.write(token.token()).offset; *offset = offset.saturating_add(bytes_read as u64) } + // I/O accounting: increment rchar and syscr on current context. + // Cross-referenced with Linux 7.1 mm/filemap.c:filemap_get_pages + // and task_io_account_read(). + let current = context::current(); + let mut guard = current.write(token.token()); + guard.io_rchar = guard.io_rchar.saturating_add(bytes_read as u64); + guard.io_syscr = guard.io_syscr.saturating_add(1); + guard.io_read_bytes = guard.io_read_bytes.saturating_add(bytes_read as u64); Ok(bytes_read) } pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + let result = (|| { let (bytes_written, desc_arc, desc) = file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| { let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) { desc.offset } else { - u64::MAX + 0 }; + let res = scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags, token); + if res.is_err() { + let pid = context::current().read(token.token()).pid; + error!("SYS_WRITE_ERR: pid={} fd={} scheme={} number={} offset={} err={:?}", pid, fd.get(), desc.scheme.get(), desc.number, offset, res.as_ref().err()); + } Ok(( - scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags, token)?, + res?, desc_arc, desc, )) @@ -788,5 +863,13 @@ pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) - let offset = &mut desc_arc.write(token.token()).offset; *offset = offset.saturating_add(bytes_written as u64) } + // I/O accounting: increment wchar and syscw on current context. + let current = context::current(); + let mut guard = current.write(token.token()); + guard.io_wchar = guard.io_wchar.saturating_add(bytes_written as u64); + guard.io_syscw = guard.io_syscw.saturating_add(1); + guard.io_write_bytes = guard.io_write_bytes.saturating_add(bytes_written as u64); Ok(bytes_written) + })(); + result } diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index fe8a4bf0bd..65216532e6 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -19,12 +19,13 @@ use crate::{ }, memory::{Page, PhysicalAddress, VirtualAddress}, sync::{CleanLockToken, Mutex, L1}, + time, }; use crate::syscall::{ - data::TimeSpec, + data::{FutexWaitv, TimeSpec}, error::{Error, Result, EAGAIN, EFAULT, EINVAL, ETIMEDOUT}, - flag::{FUTEX_WAIT, FUTEX_WAIT64, FUTEX_WAKE}, + flag::{FUTEX_REQUEUE, FUTEX_WAIT, FUTEX_WAIT64, FUTEX_WAIT_MULTIPLE, FUTEX_WAKE}, }; use super::usercopy::UserSlice; @@ -36,12 +37,12 @@ type FutexList = HashMap>; pub struct FutexEntry { // Virtual address, required if synchronizing across the same address space, if the memory is // CoW. - // TODO: FUTEX_REQUEUE target_virtaddr: VirtualAddress, // Context to wake up, and compare address spaces. - context_lock: Weak, + context_lock: Arc, // address space to check against if virt matches but not phys addr_space: Weak, + waitv_index: Option, } // TODO: Process-private futexes? In that case, put the futex table in each AddrSpace, or just @@ -51,17 +52,6 @@ pub struct FutexEntry { static FUTEXES: Mutex = Mutex::new(FutexList::with_hasher(DefaultHashBuilder::new())); -pub fn get_futex_stat(token: &mut CleanLockToken) -> (usize, usize) { - let mut regc = 0; - let mut regl = 0; - let registry = FUTEXES.lock(token.token()); - for (_, v) in registry.iter() { - regl += v.len(); - regc += 1; - } - (regc, regl) -} - fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option { // TODO: Move this elsewhere! if addr.data().saturating_add(size_of::()) >= crate::USER_END_OFFSET { @@ -81,7 +71,7 @@ pub fn futex( op: usize, val: usize, val2: usize, - _addr2: usize, + addr2: usize, token: &mut CleanLockToken, ) -> Result { let current_addrsp = AddrSpace::current()?; @@ -95,7 +85,6 @@ pub fn futex( .ok_or(Error::new(EFAULT))?; match op { - // TODO: FUTEX_WAIT_MULTIPLE? FUTEX_WAIT | FUTEX_WAIT64 => { let timeout_opt = UserSlice::ro(val2, size_of::())? .none_if_null() @@ -153,7 +142,9 @@ pub fn futex( { let mut context = context_lock.write(token.token()); - context.wake = timeout_opt.map(|time| time.to_nanos()); + context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| { + tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128 + }); if let Some((tctl, pctl, _)) = context.sigcontrol() && tctl.currently_pending_unblocked(pctl) != 0 { @@ -168,8 +159,9 @@ pub fn futex( .or_insert_with(Vec::new) .push(FutexEntry { target_virtaddr, - context_lock: Arc::downgrade(&context_lock), + context_lock: context_lock.clone(), addr_space: Arc::downgrade(¤t_addrsp), + waitv_index: None, }); } @@ -206,18 +198,25 @@ pub fn futex( if futex.target_virtaddr != target_virtaddr || !current_addrsp_weak.ptr_eq(&futex.addr_space) { - if futex.addr_space.strong_count() == 0 { - futexes.swap_remove(i); - } else { - i += 1; - } + i += 1; continue; } - if let Some(ctx) = futex.context_lock.upgrade() { - ctx.write(token.token()).unblock(); + let should_wake = { + let mut context = futex.context_lock.write(token.token()); + if let Some(index) = futex.waitv_index { + if context.futex_wake_index.is_none() { + context.futex_wake_index = Some(index); + } + } + context.unblock(); + true + }; + if should_wake { + futexes.swap_remove(i); + woken += 1; + } else { + i += 1; } - futexes.swap_remove(i); - woken += 1; } futexes.is_empty() @@ -231,6 +230,157 @@ pub fn futex( Ok(woken) } + FUTEX_REQUEUE => { + // Linux 7.1 kernel/futex/requeue.c futex_requeue(). + // Wake up to val waiters on primary futex, requeue up to + // val2 waiters to secondary futex (addr2). + let mut woken = 0; + let mut requeued = 0; + + let target_virtaddr2 = VirtualAddress::new(addr2); + let target_physaddr2 = validate_and_translate_virt(&addr_space_guard, target_virtaddr2) + .ok_or(Error::new(EFAULT))?; + + { + drop(addr_space_guard); + let mut futexes_map = FUTEXES.lock(token.token()); + let (futexes_map, mut token) = futexes_map.token_split(); + + if let Some(futexes) = futexes_map.get_mut(&target_physaddr) { + let mut i = 0; + let current_addrsp_weak = Arc::downgrade(¤t_addrsp); + let mut to_requeue = Vec::new(); + + while i < futexes.len() { + let futex = unsafe { futexes.get_unchecked_mut(i) }; + if futex.target_virtaddr != target_virtaddr + || !current_addrsp_weak.ptr_eq(&futex.addr_space) + { + i += 1; + continue; + } + if woken < val { + futex.context_lock.write(token.token()).unblock(); + futexes.remove(i); + woken += 1; + } else if requeued < val2 { + let entry = futexes.remove(i); + to_requeue.push(entry); + requeued += 1; + } else { + i += 1; + } + } + + if futexes.is_empty() { + futexes_map.remove(&target_physaddr); + } + + if !to_requeue.is_empty() { + futexes_map + .entry(target_physaddr2) + .or_insert_with(Vec::new) + .extend(to_requeue); + } + } + } + + Ok(woken + requeued) + } + FUTEX_WAIT_MULTIPLE => { + // Wait on multiple futexes atomically; return the index of the first + // one that is woken. Semantics: val = number of FutexWaitv entries, + // addr2 = pointer to the array, val2 = optional TimeSpec timeout. + let count = val; + if count == 0 || count > 128 { + return Err(Error::new(EINVAL)); + } + let timeout_opt = UserSlice::ro(val2, size_of::())? + .none_if_null() + .map(|buf| unsafe { buf.read_exact::() }) + .transpose()?; + + let array_size = size_of::() + .checked_mul(count) + .ok_or(Error::new(EINVAL))?; + let waitv_slice = UserSlice::ro(addr2, array_size)?; + let mut waitv = Vec::with_capacity(count); + waitv.resize(count, FutexWaitv::default()); + let waitv_bytes = unsafe { + core::slice::from_raw_parts_mut( + waitv.as_mut_ptr() as *mut u8, + array_size, + ) + }; + waitv_slice.copy_to_slice(waitv_bytes)?; + + let context_lock = context::current(); + + let mut translated = Vec::with_capacity(count); + for entry in &waitv { + if entry.flags != 0 { + return Err(Error::new(EINVAL)); + } + if !entry.uaddr.is_multiple_of(4) { + return Err(Error::new(EINVAL)); + } + let virt = VirtualAddress::new(entry.uaddr); + let phys = validate_and_translate_virt(&addr_space_guard, virt) + .ok_or(Error::new(EFAULT))?; + let accessible_addr = crate::memory::RmmA::phys_to_virt(phys).data(); + let fetched = u64::from(unsafe { + (*(accessible_addr as *const AtomicU32)).load(Ordering::SeqCst) + }); + if fetched != entry.val as u64 { + return Err(Error::new(EAGAIN)); + } + translated.push((virt, phys)); + } + + { + let mut token = unsafe { CleanLockToken::new() }; + let mut futexes = FUTEXES.lock(token.token()); + let (futexes, mut token) = futexes.token_split(); + + { + let mut context = context_lock.write(token.token()); + context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| { + tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128 + }); + context.futex_wake_index = None; + if let Some((tctl, pctl, _)) = context.sigcontrol() + && tctl.currently_pending_unblocked(pctl) != 0 + { + return Err(Error::new(EINTR)); + } + context.block("futex_wait_multiple"); + } + + for (i, (virt, phys)) in translated.into_iter().enumerate() { + futexes + .entry(phys) + .or_insert_with(Vec::new) + .push(FutexEntry { + target_virtaddr: virt, + context_lock: context_lock.clone(), + addr_space: Arc::downgrade(¤t_addrsp), + waitv_index: Some(i), + }); + } + } + + drop(addr_space_guard); + + context::switch(token); + + let context = context_lock.read(token.token()); + + if context.wake.is_none() && timeout_opt.is_some() { + Err(Error::new(ETIMEDOUT)) + } else { + Ok(context.futex_wake_index.unwrap_or(0)) + } + } _ => Err(Error::new(EINVAL)), } } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 550d260f86..bfd638354b 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,6 +4,8 @@ extern crate syscall; +use syscall::{dirent::DirentHeader, CallFlags, RwFlags, EINVAL}; + pub use self::syscall::{ data, error, flag, io, number, ptrace_event, EnvRegisters, FloatRegisters, IntRegisters, }; @@ -13,14 +15,14 @@ pub use self::{fs::*, futex::futex, process::*, time::*, usercopy::validate_regi use self::{ data::{Map, TimeSpec}, debug::{debug_end, debug_start}, - error::{Error, Result, EINVAL, ENOSYS}, - flag::{CallFlags, EventFlags, MapFlags, RwFlags}, + error::{Error, Result, ENOSYS}, + flag::{EventFlags, MapFlags}, number::*, usercopy::UserSlice, }; use crate::{ - context::memory::AddrSpace, + context::{self, memory::AddrSpace}, percpu::PercpuBlock, scheme::{memory::MemoryScheme, FileHandle}, sync::CleanLockToken, @@ -71,26 +73,34 @@ pub fn syscall( let fd = FileHandle::from(b); //SYS_* is declared in kernel/syscall/src/number.rs match a { - SYS_WRITE2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| { - let flags = if f == usize::MAX { - None - } else { - Some( - u32::try_from(f) - .ok() - .and_then(RwFlags::from_bits) - .ok_or(Error::new(EINVAL))?, + SYS_WRITE2 => { + let bytes = file_op_generic_ext(fd, token, |scheme, _, desc, token| { + let flags = if f == usize::MAX { + None + } else { + Some( + u32::try_from(f) + .ok() + .and_then(RwFlags::from_bits) + .ok_or(Error::new(EINVAL))?, + ) + }; + scheme.kwriteoff( + desc.number, + UserSlice::ro(c, d)?, + e as u64, + flags.map_or(desc.flags, |f| desc.rw_flags(f)), + desc.flags, + token, ) - }; - scheme.kwriteoff( - desc.number, - UserSlice::ro(c, d)?, - e as u64, - flags.map_or(desc.flags, |f| desc.rw_flags(f)), - desc.flags, - token, - ) - }), + })?; + let current = context::current(); + let mut guard = current.write(token.token()); + guard.io_wchar = guard.io_wchar.saturating_add(bytes as u64); + guard.io_syscw = guard.io_syscw.saturating_add(1); + guard.io_write_bytes = guard.io_write_bytes.saturating_add(bytes as u64); + Ok(bytes) + } SYS_WRITE => sys_write(fd, UserSlice::ro(c, d)?, token), SYS_FMAP => { let addrspace = AddrSpace::current()?; @@ -103,43 +113,88 @@ pub fn syscall( }) } } - SYS_READ2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| { - let flags = if f == usize::MAX { - None - } else { - Some( - u32::try_from(f) - .ok() - .and_then(RwFlags::from_bits) - .ok_or(Error::new(EINVAL))?, - ) - }; - scheme.kreadoff( - desc.number, - UserSlice::wo(c, d)?, - e as u64, - flags.map_or(desc.flags, |f| desc.rw_flags(f)), - desc.flags, - token, - ) + SYS_GETDENTS => { + let header_size = u16::try_from(e).map_err(|_| Error::new(EINVAL))?; + + if usize::from(header_size) != size_of::() { + // TODO: allow? If so, zero_out must be implemented for UserSlice + return Err(Error::new(EINVAL)); + } + + file_op_generic(fd, token, |scheme, number, token| { + scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64, token) + }) + } + SYS_FUTIMENS => file_op_generic(fd, token, |scheme, number, token| { + scheme.kfutimens(number, UserSlice::ro(c, d)?, token) }), + + SYS_READ2 => { + let bytes = file_op_generic_ext(fd, token, |scheme, _, desc, token| { + let flags = if f == usize::MAX { + None + } else { + Some( + u32::try_from(f) + .ok() + .and_then(RwFlags::from_bits) + .ok_or(Error::new(EINVAL))?, + ) + }; + scheme.kreadoff( + desc.number, + UserSlice::wo(c, d)?, + e as u64, + flags.map_or(desc.flags, |f| desc.rw_flags(f)), + desc.flags, + token, + ) + })?; + let current = context::current(); + let mut guard = current.write(token.token()); + guard.io_rchar = guard.io_rchar.saturating_add(bytes as u64); + guard.io_syscr = guard.io_syscr.saturating_add(1); + guard.io_read_bytes = guard.io_read_bytes.saturating_add(bytes as u64); + Ok(bytes) + } SYS_READ => sys_read(fd, UserSlice::wo(c, d)?, token), SYS_FPATH => file_op_generic(fd, token, |scheme, number, token| { scheme.kfpath(number, UserSlice::wo(c, d)?, token) }), - - // TODO: Can't replace yet with std_fs_call, as fstat overrides device ID, but that can - // be moved to UserScheme. SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?, token).map(|()| 0), + SYS_FSTATVFS => file_op_generic(fd, token, |scheme, number, token| { + scheme + .kfstatvfs(number, UserSlice::wo(c, d)?, token) + .map(|()| 0) + }), - SYS_DUP_INTO => { - dup_into(fd, FileHandle::from(e), UserSlice::ro(c, d)?, token).map(FileHandle::into) - } + SYS_DUP => dup(fd, UserSlice::ro(c, d)?, token).map(FileHandle::into), + SYS_DUP_INTO => dup_into( + fd, + FileHandle::from(e), + UserSlice::ro(c, d)?, + token, + ).map(FileHandle::into), SYS_DUP2 => { dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?, token).map(FileHandle::into) } + #[cfg(target_pointer_width = "32")] + SYS_SENDFD => sendfd( + fd, + FileHandle::from(c), + d, + e as u64 | ((f as u64) << 32), + token, + ), + + #[cfg(target_pointer_width = "64")] + SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64, token), + SYS_LSEEK => lseek(fd, c as i64, d, token), + SYS_FCHMOD => file_op_generic(fd, token, |scheme, number, token| { + scheme.fchmod(number, c as u16, token).map(|()| 0) + }), SYS_FCHOWN => file_op_generic(fd, token, |scheme, number, token| { scheme.fchown(number, c as u32, d as u32, token).map(|()| 0) }), @@ -153,60 +208,59 @@ pub fn syscall( SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?, token).map(|()| 0), SYS_FUNMAP => funmap(b, c, token), - // TODO: This can't be removed yet, since the pre-libredox softbuffer crate is a blocker. - SYS_FSYNC => { - //let ctxt_name = crate::context::current().read(token.token()).name; - //warn!("Context `{ctxt_name}` is using deprecated SYS_FSYNC"); - - file_op_generic(fd, token, |scheme, number, token| { - scheme.fsync(number, token).map(|()| 0) - }) + SYS_FSYNC => file_op_generic(fd, token, |scheme, number, token| { + scheme.fsync(number, token).map(|()| 0) + }), + SYS_SYNC => { + // Red Bear filesystem I/O is synchronous — all data is + // already on disk. sync() is a no-op per POSIX semantics. + // Cross-referenced with Linux 7.1 fs/sync.c ksys_sync(). + Ok(0) } + SYS_SYNCFS => file_op_generic(fd, token, |scheme, number, token| { + // syncfs(fd) syncs the filesystem containing fd. + // On Red Bear, I/O is synchronous so this is a no-op. + scheme.fsync(number, token).map(|()| 0) + }), + // TODO: 64-bit lengths on 32-bit platforms + SYS_FTRUNCATE => file_op_generic(fd, token, |scheme, number, token| { + scheme.ftruncate(number, c, token).map(|()| 0) + }), SYS_CLOSE => close(fd, token).map(|()| 0), - SYS_CALL => { - let flags = CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?; - if flags.contains(CallFlags::MULTIPLE_FDS) { - if g / core::mem::size_of::() > 16 { - return Err(Error::new(EINVAL)); - }; - let mut fds = [0_usize; 16]; - let fds_slice = UserSlice::ro(b, g)?; - - // TODO: bytemuck/plain - let copied = fds_slice.copy_common_bytes_to_slice(unsafe { - core::slice::from_raw_parts_mut( - fds.as_mut_ptr().cast(), - fds.len() * core::mem::size_of::(), - ) - })?; - call( - &fds[..copied / core::mem::size_of::()], - UserSlice::rw(c, d)?, - flags, - UserSlice::ro(f, (e & 0xff) * 8)?, - token, - ) - } else { - call( - &[b], - UserSlice::rw(c, d)?, - flags, - UserSlice::ro(f, (e & 0xff) * 8)?, - token, - ) - } + SYS_CALL => call( + fd, + UserSlice::rw(c, d)?, + CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?, + UserSlice::ro(f, (e & 0xff) * 8)?, + token, + ), + SYS_OPENAT => { + openat(fd, UserSlice::ro(c, d)?, e, f as _, 0, 0, token).map(FileHandle::into) } SYS_OPENAT_INTO => openat_into( fd, UserSlice::ro(c, d)?, e, - f as _, - FileHandle::from(g), + f as u32, + FileHandle::from(g as usize), token, ) .map(FileHandle::into), - SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, token).map(|()| 0), + SYS_OPENAT_WITH_FILTER => openat( + fd, + UserSlice::ro(c, d)?, + e, + (e & syscall::O_FCNTL_MASK) as _, + f as _, + g as _, + token, + ) + .map(FileHandle::into), + SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, 0, 0, token).map(|()| 0), + SYS_UNLINKAT_WITH_FILTER => { + unlinkat(fd, UserSlice::ro(c, d)?, e, f as _, g as _, token).map(|()| 0) + } SYS_YIELD => sched_yield(token).map(|()| 0), SYS_NANOSLEEP => nanosleep( UserSlice::ro(b, size_of::())?, @@ -221,7 +275,13 @@ pub fn syscall( SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d), token).map(|()| 0), SYS_MREMAP => mremap(b, c, d, e, f, token), - _ => Err(Error::new(ENOSYS)), + SYS_MKNS => sys_mkns(UserSlice::ro(b, c)?, token).map(FileHandle::into), + SYS_SETNS => sys_setns(fd, token).map(|()| 0), + + _ => { + println!("KERNEL: unimplemented syscall a={:#x} b={:#x} c={:#x} d={:#x}", a, b, c, d); + Err(Error::new(ENOSYS)) + } } } diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 18b0f5ffe7..6e773c5d90 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -20,41 +20,45 @@ use crate::{ context::{self, context::FdTbl}, memory::{Page, VirtualAddress, PAGE_SIZE}, scheme::{ - FileHandle, KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES, - KERNEL_SCHEMES_COUNT, + KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES, KERNEL_SCHEMES_COUNT, }, startup::Bootstrap, syscall::{error::*, flag::MapFlags}, CurrentRmmArch, }; -use super::usercopy::UserSliceWo; +use super::usercopy::{UserSliceRo, UserSliceWo}; + +use crate::scheme::FileHandle; pub fn exit_this_context(excp: Option, token: &mut CleanLockToken) -> ! { + let mut close_files; + let addrspace_opt; + let context_lock = context::current(); - let (addrspace_opt, mut close_files) = { + { let mut context = context_lock.write(token.token()); - let (context, token) = context.token_split(); - let close_files = Arc::try_unwrap(mem::take(&mut context.files)) + // let (context, mut token) = context.token_split(); + close_files = Arc::try_unwrap(mem::take(&mut context.files)) .map_or_else(|_| FdTbl::new(), RwLock::into_inner); - let addrspace_opt = context.set_addr_space(None, token); + // TODO: Lock ordering violation + let mut token = unsafe { CleanLockToken::new() }; + addrspace_opt = context + .set_addr_space(None, token.downgrade()) + .and_then(|a| Arc::try_unwrap(a).ok()); drop(mem::replace(&mut context.syscall_head, SyscallFrame::Dummy)); drop(mem::replace(&mut context.syscall_tail, SyscallFrame::Dummy)); - (addrspace_opt, close_files) - }; + } // Files must be closed while context is valid so that messages can be passed close_files.force_close_all(token); if let Some(addrspace) = addrspace_opt { - if let Ok(addrspace) = Arc::try_unwrap(addrspace) { - addrspace.into_drop(token); - } + addrspace.into_drop(token); } // TODO: Should status == Status::HardBlocked be handled differently? let owner = { let mut guard = context_lock.write(token.token()); guard.status = context::Status::Dead { excp }; - guard.owner_proc_id }; if let Some(owner) = owner { @@ -73,7 +77,6 @@ pub fn exit_this_context(excp: Option, token: &mut CleanLock } } } - drop(close_files); context::switch(token); unreachable!(); } @@ -92,6 +95,49 @@ pub fn mprotect( AddrSpace::current()?.mprotect(span, flags, token) } +/// Create a new namespace containing the named schemes (space-separated in `names`). +/// +/// Returns a file handle that refers to the new namespace; the caller is expected to +/// pass it to `sys_setns` to switch into it, or to `redox-rt::sys::setns` for a +/// pure-userspace swap of the per-thread dynamic proc info `ns_fd`. +/// +/// Implementation: dups the calling context's namespace fd with the caller-supplied +/// buffer (which contains `NsDup::ForkNs` followed by scheme names). The dup is +/// handled by the bootstrap's userspace namespace manager daemon (`initnsmgr`), +/// which creates the new namespace and returns a new fd. +pub fn sys_mkns( + names: UserSliceRo, + token: &mut CleanLockToken, +) -> Result { + let ns_fd_opt = { + let ctx_lock = context::current(); + let ctx = ctx_lock.read(token.token()); + ctx.ns_fd + }; + let ns_fd = ns_fd_opt.ok_or(Error::new(EBADF))?; + crate::syscall::fs::dup(FileHandle::from(ns_fd), names, token) +} + +/// Switch the calling context to the namespace referenced by `fd`. +/// +/// The fd must have been produced by `sys_mkns` (or by the bootstrap's namespace +/// manager). After this syscall returns, all scheme lookups in the calling +/// context resolve against the new namespace. +pub fn sys_setns( + fd: FileHandle, + token: &mut CleanLockToken, +) -> Result<()> { + let ns_fd = fd.get(); + if ns_fd == 0 || ns_fd == usize::MAX { + return Err(Error::new(EBADF)); + } + + let ctx_lock = context::current(); + let mut ctx = ctx_lock.write(token.token()); + ctx.ns_fd = Some(ns_fd); + Ok(()) +} + const KERNEL_METADATA_BASE: usize = crate::USER_END_OFFSET - syscall::KERNEL_METADATA_SIZE; const KERNEL_METADATA_PAGE_COUNT: usize = syscall::KERNEL_METADATA_SIZE / PAGE_SIZE + { if syscall::KERNEL_METADATA_SIZE.is_multiple_of(PAGE_SIZE) { @@ -155,9 +201,15 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok inner.fd = { let cap_fd = match scheme.as_scheme().scheme_root(token) { Ok(fd) => fd, - Err(_) => usize::MAX, + Err(_) => usize::MAX }; - insert_fd(scheme.scheme_id(), cap_fd, token) + let inserted = insert_fd( + scheme.scheme_id(), + cap_fd, + matches!(scheme, GlobalSchemes::Proc), + token, + ); + inserted }; } } @@ -170,7 +222,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok }; // Second, retrieve the scheme ID. let scheme_id = &SchemeList.id(); - insert_fd(*scheme_id, cap_fd, token) + insert_fd(*scheme_id, cap_fd, false, token) }; let mut lock_token = token.token(); @@ -239,10 +291,57 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok .expect("failed to copy memory to bootstrap"); let bootstrap_entry = u64::from_le_bytes(bootstrap_slice[0x1a..0x22].try_into().unwrap()); - debug!("Bootstrap entry point: {:X}", bootstrap_entry); assert_ne!(bootstrap_entry, 0); - // Start in a minimal environment without any stack. + // Map a minimal user stack for the bootstrap process. Without this, + // RSP defaults to 0 from InterruptStack::init() and the first `push` or + // `call` in the binary triggers a page fault at address 0, which the + // C startup turns into `ud2` (Rust panic handler) — visible as + // "Invalid opcode fault" at a low RIP before main() ever runs. + // + // The stack is mapped just above the initfs image in user virtual + // memory. The top-of-stack (RSP initial value) points to the END of + // the mapped region, since x86-64 stacks grow downward. + const BOOTSTRAP_STACK_PAGES: usize = 8; // 32 KiB — enough for Rust _start + let stack_base_vaddr = PAGE_SIZE + bootstrap.page_count * PAGE_SIZE; + let stack_top_vaddr = stack_base_vaddr + BOOTSTRAP_STACK_PAGES * PAGE_SIZE; + { + let addr_space = Arc::clone( + context::current() + .read(token.token()) + .addr_space() + .expect("expected bootstrap context to have an address space"), + ); + let page_count = + NonZeroUsize::new(BOOTSTRAP_STACK_PAGES).expect("BOOTSTRAP_STACK_PAGES must be non-zero"); + let stack_base = Page::containing_address(VirtualAddress::new(stack_base_vaddr)); + let _base_page = { + let mut lock_token = token.token(); + let mut addr_space_lock = addr_space.acquire_write(lock_token.downgrade()); + let _ = addr_space_lock + .mmap( + &addr_space, + Some(stack_base), + page_count, + MapFlags::MAP_FIXED_NOREPLACE | MapFlags::PROT_READ | MapFlags::PROT_WRITE, + None, + |page, flags, mapper, flusher| { + let span = PageSpan::new(page, BOOTSTRAP_STACK_PAGES); + Ok(Grant::zeroed( + span, + flags, + mapper, + flusher, + false, // private mapping for bootstrap stack + )?) + }, + ); + }; + debug!( + "Bootstrap stack mapped at {:#x}..{:#x} ({} pages)", + stack_base_vaddr, stack_top_vaddr, BOOTSTRAP_STACK_PAGES + ); + } let ctx = context::current(); let mut lock = ctx.write(token.token()); @@ -252,8 +351,9 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok { regs.init(); regs.set_instr_pointer(bootstrap_entry.try_into().unwrap()); + regs.set_stack_pointer(stack_top_vaddr); + } } -} unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] { unsafe { @@ -264,23 +364,12 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] } } -fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usize { +fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize { let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context - .files - .write(token.token()) - .resize(0, 64) - .expect("failed to resize lower fdtbl"); - context - .files - .write(token.token()) - .resize(syscall::flag::UPPER_FDTBL_TAG, 64) - .expect("failed to resize upper fdtbl"); - context - .insert_file( - FileHandle::from(syscall::flag::UPPER_FDTBL_TAG | scheme.get()), + .add_file_min( FileDescriptor { description: Arc::new(RwLock::new(FileDescription { scheme, @@ -289,7 +378,9 @@ fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usi flags: (O_CREAT | O_RDWR) as u32, internal_flags: InternalFlags::empty(), })), + cloexec, }, + syscall::flag::UPPER_FDTBL_TAG + scheme.get(), &mut token, ) .expect("failed to insert fd to current context") diff --git a/src/syscall/time.rs b/src/syscall/time.rs index e5b64fa0a5..aa6763f5b0 100644 --- a/src/syscall/time.rs +++ b/src/syscall/time.rs @@ -18,7 +18,10 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo, token: &mut CleanLockToken) _ => return Err(Error::new(EINVAL)), }; - buf.copy_exactly(&TimeSpec::from_nanos(arch_time)) + buf.copy_exactly(&TimeSpec { + tv_sec: (arch_time / time::NANOS_PER_SEC) as i64, + tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32, + }) } /// Nanosleep will sleep by switching the current context @@ -34,18 +37,18 @@ pub fn nanosleep( } let start = time::monotonic(token); - let end = start + req.to_nanos(); + let end = start + (req.tv_sec as u128 * time::NANOS_PER_SEC) + (req.tv_nsec as u128); let current_context = context::current(); { - let context = current_context.upgradeable_read(token.token()); + let mut context = current_context.write(token.token()); if let Some((tctl, pctl, _)) = context.sigcontrol() && tctl.currently_pending_unblocked(pctl) != 0 { return Err(Error::new(EINTR)); } - let mut context = context.upgrade(); + context.wake = Some(end); context.block("nanosleep"); } @@ -61,9 +64,15 @@ pub fn nanosleep( rem_buf.copy_exactly(&if current < end { let diff = end - current; - TimeSpec::from_nanos(diff) + TimeSpec { + tv_sec: (diff / time::NANOS_PER_SEC) as i64, + tv_nsec: (diff % time::NANOS_PER_SEC) as i32, + } } else { - TimeSpec::default() + TimeSpec { + tv_sec: 0, + tv_nsec: 0, + } })?; } diff --git a/targets/aarch64-unknown-kernel.json b/targets/aarch64-unknown-kernel.json index f5a777497f..58b72ffac3 100644 --- a/targets/aarch64-unknown-kernel.json +++ b/targets/aarch64-unknown-kernel.json @@ -17,7 +17,6 @@ "os": "none", "position-independent-executables": false, "relocation-model": "pic", - "rustc-abi": "softfloat", "target-c-int-width": 32, "target-endian": "little", "target-pointer-width": 64,