diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000..33d2be1164 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +rustflags = [ + # Kernel should preserve floating-point registers + "-Ctarget-feature=-sse,-sse2", +] diff --git a/Cargo.lock b/Cargo.lock index b40bfac6d1..3f3f3dc141 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,15 +40,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "cc" -version = "1.2.63" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "shlex", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "indexmap" @@ -99,16 +99,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown 0.17.0", ] [[package]] name = "kernel" -version = "0.5.12" +version = "0.5.12+rb0.3.0" dependencies = [ "arrayvec", "bitfield", - "bitflags 2.12.1", + "bitflags 2.11.1", "cc", "fdt", "hashbrown 0.14.5", @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "object" @@ -201,18 +201,16 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7" [[package]] name = "redox_syscall" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" +version = "0.9.0+rb0.3.0" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.11.1", ] [[package]] name = "rmm" version = "0.1.0" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.11.1", ] [[package]] @@ -282,9 +280,9 @@ dependencies = [ [[package]] name = "shlex" -version = "2.0.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "slab" @@ -402,20 +400,24 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", "syn", ] + +[[patch.unused]] +name = "libredox" +version = "0.1.18+rb0.2.5" diff --git a/Cargo.toml b/Cargo.toml index ea56939857..3d51e2d5c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,11 @@ [workspace] resolver = "3" +members = [".", "rmm"] [package] name = "kernel" -version = "0.5.12" +version = "0.5.12+rb0.3.0" +authors = ["Jeremy Soller ", "vasilito "] build = "build.rs" edition = "2024" @@ -19,7 +21,7 @@ fdt = { git = "https://github.com/repnop/fdt.git", rev = "2fb1409edd1877c714a0aa hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] } linked_list_allocator = "0.9.0" redox-path = "0.2.0" -redox_syscall = { version = "0.9.0", default-features = false } +redox_syscall = { path = "../syscall", default-features = false } rmm = { path = "rmm", default-features = false } slab = { version = "0.4", default-features = false } smallvec = { version = "1.15.1", default-features = false } @@ -80,13 +82,12 @@ default = [ "acpi", #"debugger", "multi_core", - # "numa", "serial_debug", "self_modifying", "x86_kvm_pv", #"busy_panic", #"drop_panic", - #"syscall_debug", + #"syscall_debug" ] # Activates some limited code-overwriting optimizations, based on CPU features. @@ -95,7 +96,6 @@ self_modifying = [] acpi = [] lpss_debug = [] multi_core = ["acpi"] -numa = ["multi_core"] profiling = [] #TODO: remove when threading issues are fixed pti = [] @@ -120,3 +120,18 @@ panic = "abort" panic = "abort" #lto = true debug = "full" + +# Red Bear OS Phase J: see local/sources/base/Cargo.toml for +# the rationale. Both the kernel and the base workspace need +# the libredox override so that the libredox::error::Error +# type is the same compile-time type as syscall::Error. With +# the local libredox fork at local/sources/libredox/ using +# the local syscall fork at local/sources/syscall/, the +# libredox::error::Error (re-exported from the local syscall) +# and syscall::Error (also the local syscall) are now the +# same type, so `?` conversions in scheme-utils / daemon +# compile cleanly. +[patch.crates-io] +# Local fork dependency rule: every crate with a local fork MUST resolve through it. +libredox = { path = "../libredox" } +redox_syscall = { path = "../syscall" } diff --git a/Makefile b/Makefile index a089f8c350..1c658da303 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,7 @@ $(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell --manifest-path "$(MANIFEST)" \ --target "$(TARGET_SPEC)" \ --release \ - -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \ - -Z json-target-spec \ + -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \ --features=$(KERNEL_CARGO_FEATURES) \ -- \ -C link-arg=-T -Clink-arg="$(LD_SCRIPT)" \ @@ -63,6 +62,5 @@ check: --bin kernel \ --manifest-path "$(MANIFEST)" \ --target "$(TARGET_SPEC)" \ - -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \ - -Z json-target-spec \ + -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \ --features=$(KERNEL_CHECK_FEATURES) diff --git a/src/acpi/facs.rs b/src/acpi/facs.rs new file mode 100644 index 0000000000..9e48e500b2 --- /dev/null +++ b/src/acpi/facs.rs @@ -0,0 +1,291 @@ +//! ACPI Firmware ACPI Control Structure (FACS) parser. +//! +//! Per ACPI 6.5 §5.2.10. The FACS contains the firmware waking +//! vector that the platform firmware jumps to on S3 wake. +//! This is the memory location the S3 resume trampoline in +//! `arch/x86_shared/s3_resume.rs` must be written to. +//! +//! This is a comprehensive parser matching Linux 7.1's +//! `struct acpi_table_facs` in `include/acpi/actbl.h`: +//! +//! ```c +//! struct acpi_table_facs { +//! char signature[4]; // ASCII table signature +//! u32 length; // Length of structure, in bytes +//! u32 hardware_signature; // Hardware configuration signature +//! u32 firmware_waking_vector; // 32-bit FW waking vector +//! u32 global_lock; // Global Lock for shared hardware +//! u32 flags; // Flags +//! u64 xfirmware_waking_vector; // 64-bit FW waking vector (ACPI 2.0+) +//! u8 version; // Version of this table (ACPI 2.0+) +//! u8 reserved[3]; // Reserved +//! u32 ospm_flags; // Flags set by OSPM (ACPI 4.0+) +//! u8 reserved1[24]; // Reserved (ACPI 4.0+) +//! }; +//! ``` +//! +//! We also model the corresponding flag constants: +//! - `ACPI_GLOCK_PENDING` (1 << 0) +//! - `ACPI_GLOCK_OWNED` (1 << 1) +//! - `ACPI_FACS_S4_BIOS_PRESENT` (1 << 0) +//! - `ACPI_FACS_64BIT_WAKE` (1 << 1) +//! - `ACPI_FACS_64BIT_ENVIRONMENT` (1 << 0) +//! +//! Hardware-agnostic: the FACS layout is standardized by the ACPI +//! spec. Only the field values (specifically the waking vector) +//! vary per platform. + +use core::sync::atomic::AtomicPtr; + +use crate::acpi::sdt::Sdt; + +/// Linux 7.1 compatibility: matching `struct acpi_table_facs`. +/// +/// The struct is `repr(C, packed)` to match the wire layout +/// exactly. The kernel reads the bytes via direct pointer +/// access. +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct FacsStruct { + /// SDT header (signature, length, revision, checksum, + /// oem_id, oem_table_id, oem_revision, creator_id, + /// creator_revision). Same as other ACPI tables. + pub header: super::sdt::Sdt, + /// Hardware configuration signature. Used by firmware to + /// detect a cold boot vs a resume (the value differs + /// across boots because of system-specific information). + /// Red Bear OS doesn't currently use this — the kernel's + /// S3 magic value is in `s3_resume::S3State.saved_magic`. + pub hardware_signature: u32, + /// 32-bit firmware waking vector. Legacy field used by + /// firmware that runs in 32-bit mode after S3 wake. The + /// platform firmware jumps to this address on a wake + /// event. + pub firmware_waking_vector: u32, + /// Global Lock for shared hardware resources. Acquired by + /// the OS before reading/writing the FACS (or any other + /// ACPI table that requires synchronization with firmware). + /// The kernel currently doesn't use this — we read/write + /// the FACS without taking the global lock. Future work: + /// take the global lock in `read()` and `write()` if + /// multi-core S3 support is added. + pub global_lock: u32, + /// Flags. Bit 0 = S4BIOS support is present. Bit 1 = 64-bit + /// wake vector is supported (ACPI 4.0+). + pub flags: u32, + /// 64-bit firmware waking vector. Used by firmware that + /// runs in 64-bit mode after S3 wake. This is what the + /// kernel's S3 trampoline is written to. (ACPI 2.0+.) + pub xfirmware_waking_vector: u64, + /// FACS version. The FACS was introduced in ACPI 1.0; the + /// 64-bit wake vector was added in ACPI 2.0. (ACPI 2.0+.) + pub version: u8, + /// Reserved. Must be zero. Three bytes. + pub reserved: [u8; 3], + /// OSPM-set flags. Bit 0 = 64-bit wake environment is + /// required (ACPI 4.0+). + pub ospm_flags: u32, + /// Reserved. Must be zero. 24 bytes. (ACPI 4.0+.) + pub reserved1: [u8; 24], +} + +/// FACS flag constants (mirrors Linux 7.1's `actbl.h`). +/// +/// Used in the `flags` field. Bit 0 = S4BIOS support is +/// present. Bit 1 = 64-bit wake vector is supported. +pub mod facs_flags { + /// `ACPI_FACS_S4_BIOS_PRESENT` (bit 0). The S4BIOS_REQ + /// signal is supported. + pub const S4_BIOS_PRESENT: u32 = 1 << 0; + /// `ACPI_FACS_64BIT_WAKE` (bit 1). The 64-bit wake vector + /// is supported (i.e., `xfirmware_waking_vector` is valid). + pub const WAKE_64BIT: u32 = 1 << 1; +} + +/// FACS OSPM flag constants (mirrors Linux 7.1's `actbl.h`). +/// +/// Used in the `ospm_flags` field. Bit 0 = 64-bit wake +/// environment is required. +pub mod facs_ospm_flags { + /// `ACPI_FACS_64BIT_ENVIRONMENT` (bit 0). The firmware + /// uses the 64-bit waking vector on S3 wake. + pub const WAKE_64BIT_ENVIRONMENT: u32 = 1 << 0; +} + +/// FACS Global Lock flag constants (mirrors Linux 7.1's +/// `actbl.h`). Used in the `global_lock` field. +pub mod facs_glock_flags { + /// `ACPI_GLOCK_PENDING` (bit 0). The global lock is + /// pending (firmware requested it). + pub const PENDING: u32 = 1 << 0; + /// `ACPI_GLOCK_OWNED` (bit 1). The global lock is + /// currently owned. + pub const OWNED: u32 = 1 << 1; +} + +/// FACS instance pointer. Set by `init` when the FACS is +/// discovered during ACPI table parsing. Used by the +/// `SetS3WakingVector` AcPiVerb and the `firmware_waking_vector_*` +/// functions below. +static FACS_PTR: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +/// Phase II.X.W: Initialize the FACS parser. Called from the +/// kernel's acpi init after all SDTs are loaded. Reads the +/// FACS from the SDT table and stores the pointer for later +/// use by the `SetS3WakingVector` AcPiVerb. +/// +/// # Safety +/// `sdt` must be a valid pointer to a validated SDT whose +/// signature is "FACS" and whose length is at least 64 +/// bytes (the minimum size of the FACS structure for ACPI +/// 4.0+ with `ospm_flags` and `reserved1`). +pub fn init(sdt: &Sdt) { + if &sdt.signature != b"FACS" { + return; + } + // The minimum FACS size depends on the version: + // - ACPI 1.0: 32 bytes (just the header + hardware_signature + // + firmware_waking_vector + global_lock) + // - ACPI 2.0: 40 bytes (adds flags + xfirmware_waking_vector) + // - ACPI 4.0: 64 bytes (adds version + reserved + ospm_flags + // + reserved1) + // We require 64 bytes to support all fields. + if sdt.length() < 64 { + return; + } + FACS_PTR.store( + sdt.data_address() as *mut FacsStruct, + core::sync::atomic::Ordering::Release, + ); +} + +/// Phase II.X.W: Get the FACS structure. Returns `None` if +/// the FACS is not available. +pub fn get() -> Option<&'static FacsStruct> { + let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire); + if ptr.is_null() { + None + } else { + // SAFETY: FACS_PTR was set by `init` after verifying + // sdt.length() >= 64. The pointer is to firmware memory + // which doesn't change. + Some(unsafe { &*ptr }) + } +} + +/// Phase II.X.W: Read the 32-bit firmware waking vector. +/// Returns 0 if the FACS is not available. +pub fn firmware_waking_vector() -> u32 { + self::get() + .map(|f| f.firmware_waking_vector) + .unwrap_or(0) +} + +/// Phase II.X.W: Read the 64-bit firmware waking vector. +/// Returns 0 if the FACS is not available. +pub fn x_firmware_waking_vector() -> u64 { + self::get() + .map(|f| f.xfirmware_waking_vector) + .unwrap_or(0) +} + +/// Phase II.X.W: Write the 32-bit firmware waking vector. +/// The kernel's S3 trampoline is written here so the legacy +/// 32-bit firmware wake path works. +/// +/// Returns true on success, false if the FACS is not +/// available. +pub fn set_firmware_waking_vector_32(addr: u32) -> bool { + let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire); + if ptr.is_null() { + return false; + } + // SAFETY: FACS_PTR was set by `init` after verifying + // sdt.length() >= 64. The `firmware_waking_vector` field + // is at offset 36 (after the 36-byte SDT header). The + // memory is page-aligned (firmware memory) and writable. + unsafe { + let waking_vector_ptr = (ptr as *mut u8).add(36) as *mut u32; + core::ptr::write_unaligned(waking_vector_ptr, addr); + } + true +} + +/// Phase II.X.W: Write the 64-bit firmware waking vector. +/// The kernel's S3 trampoline is written here so the modern +/// 64-bit firmware wake path works. +/// +/// Returns true on success, false if the FACS is not +/// available. +pub fn set_x_firmware_waking_vector_64(addr: u64) -> bool { + let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire); + if ptr.is_null() { + return false; + } + // SAFETY: FACS_PTR was set by `init` after verifying + // sdt.length() >= 64. The `xfirmware_waking_vector` field + // is at offset 40. The memory is page-aligned and + // writable. + unsafe { + let x_waking_vector_ptr = (ptr as *mut u8).add(40) as *mut u64; + core::ptr::write_unaligned(x_waking_vector_ptr, addr); + } + true +} + +/// Phase II.X.W: Write the 64-bit firmware waking vector +/// (preferred over the 32-bit version on 64-bit systems). +/// Equivalent to `set_x_firmware_waking_vector_64`. +pub fn set_waking_vector(addr: u64) -> bool { + set_x_firmware_waking_vector_64(addr) +} + +/// Phase II.X.W: Read the FACS version. Returns 0 if the +/// FACS is not available. +pub fn version() -> u8 { + self::get().map(|f| f.version).unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS hardware signature. Returns 0 +/// if the FACS is not available. +pub fn hardware_signature() -> u32 { + self::get() + .map(|f| f.hardware_signature) + .unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS flags. Returns 0 if the FACS +/// is not available. +/// +/// See `facs_flags::*` for the bit definitions. +pub fn flags() -> u32 { + self::get().map(|f| f.flags).unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS OSPM flags. Returns 0 if the +/// FACS is not available. +/// +/// See `facs_ospm_flags::*` for the bit definitions. +pub fn ospm_flags() -> u32 { + self::get().map(|f| f.ospm_flags).unwrap_or(0) +} + +/// Phase II.X.W: Read the FACS global lock. Returns 0 if the +/// FACS is not available. +/// +/// See `facs_glock_flags::*` for the bit definitions. +pub fn global_lock() -> u32 { + self::get().map(|f| f.global_lock).unwrap_or(0) +} + +/// Phase II.X.W: Read the reserved bytes. Returns `None` if +/// the FACS is not available. +pub fn reserved() -> Option<[u8; 3]> { + self::get().map(|f| f.reserved) +} + +/// Phase II.X.W: Read the ACPI 4.0+ reserved bytes. Returns +/// `None` if the FACS is not available. +pub fn reserved1() -> Option<[u8; 24]> { + self::get().map(|f| f.reserved1) +} diff --git a/src/acpi/fadt.rs b/src/acpi/fadt.rs new file mode 100644 index 0000000000..6094d42e65 --- /dev/null +++ b/src/acpi/fadt.rs @@ -0,0 +1,132 @@ +//! ACPI Fixed ACPI Description Table (FADT) parser. +//! +//! Per ACPI 6.5 §5.2.9. The FADT contains the hardware register +//! addresses used by the kernel for ACPI sleep state entry (S3/S5) +//! and the SCI interrupt. This module parses the fields the +//! kernel needs (PM1a_CNT, PM1a_STS for the sleep entry path, +//! and x_firmware_ctrl / firmware_ctrl for the FACS address). +//! +//! Hardware-agnostic: the FADT layout is standardized by the ACPI +//! spec; only the field values vary per platform. + +use core::sync::atomic::{AtomicU16, AtomicU32, AtomicU64}; + +use crate::acpi::sdt::Sdt; + +/// Phase II: PM1a_CNT port. Read from the FADT at boot, written +/// by `enter_s3()` to enter S3 (SLP_TYP|SLP_EN bits). Also used +/// by S5 entry (set_global_s_state in acpid). +pub static PM1A_CONTROL_PORT: AtomicU16 = AtomicU16::new(0); + +/// Phase II: PM1a_STS port. Used by `enter_s3()` to clear +/// WAK_STS (bit 15) before writing SLP_TYP|SLP_EN. +pub static PM1A_STATUS_PORT: AtomicU16 = AtomicU16::new(0); + +/// Phase II.X.W: 32-bit FACS address (FADT offset 36, +/// `firmware_ctrl` field). Used as a fallback when +/// `x_firmware_ctrl` (offset 140, ACPI 2.0+) is not present +/// (i.e., for ACPI 1.0 systems). +pub static FIRMWARE_CTRL: AtomicU32 = AtomicU32::new(0); + +/// Phase II.X.W: 64-bit FACS address (FADT offset 140, +/// `x_firmware_ctrl` field, ACPI 2.0+). The kernel's FACS +/// parser uses this to find the FACS for writing the +/// `xfirmware_waking_vector` on S3 entry. +pub static X_FIRMWARE_CTRL: AtomicU64 = AtomicU64::new(0); + +/// FADT signature bytes ("FACP"). +const FADT_SIGNATURE: [u8; 4] = *b"FACP"; + +/// FADT fixed offsets for the fields we read. These match +/// the ACPI 6.5 §5.2.9 Table 5.6 layout. +mod offsets { + /// PM1a_STS (PM1 Status Register) Block 0 Address. + /// 32-bit General-Purpose Event Register Block 0 Address. + pub const PM1A_STS: usize = 48; + /// PM1a_CNT (PM1 Control Register) Block 0 Address. + /// 32-bit General-Purpose Event Register Block 0 Address. + pub const PM1A_CNT: usize = 56; + /// `firmware_ctrl`: 32-bit Firmware ACPI Control + /// Structure address. ACPI 1.0+. + pub const FIRMWARE_CTRL_32: usize = 36; + /// `x_firmware_ctrl`: 64-bit Firmware ACPI Control + /// Structure address. ACPI 2.0+. + pub const X_FIRMWARE_CTRL_64: usize = 140; + /// FADT minimum size for ACPI 2.0+ (i.e., enough to + /// include `x_firmware_ctrl` at offset 140). + pub const FADT_MIN_SIZE_ACPI_2_0: usize = 148; + /// FADT minimum size for ACPI 1.0. + pub const FADT_MIN_SIZE_ACPI_1_0: usize = 76; +} + +/// Parse the FADT from the given SDT bytes and extract the +/// PM1a_CNT, PM1a_STS, and FACS-address fields. Called once at +/// boot after the ACPI table discovery finds the FADT. +/// +/// The FADT layout is variable (different sizes for ACPI 1.0 vs +/// 6.5+). We only need the first ~148 bytes which contain the +/// fixed-register addresses. Reference: ACPI 6.5 §5.2.9. +pub fn init(sdt: &Sdt) { + if &sdt.signature != &FADT_SIGNATURE { + return; + } + // SAFETY: We trust the ACPI table discovery code to have + // verified the FADT checksum. The FADT fields are at fixed + // offsets (per the ACPI spec); reading them as u32/u64 is + // safe because all of them are at 4-byte or 8-byte aligned + // offsets on x86_64. + let data = sdt.data_address() as *const u8; + unsafe { + // PM1a_CNT is at offset 56 in the FADT (ACPI 6.5 §5.2.9 + // Table 5.6). 32-bit General-Purpose Event Register Block 0 + // Address. + let pm1a_cnt = core::ptr::read_unaligned(data.add(offsets::PM1A_CNT) as *const u32); + // PM1a_STS is at offset 48 in the FADT. + let pm1a_sts = core::ptr::read_unaligned(data.add(offsets::PM1A_STS) as *const u32); + // Convert u32 to u16 (port numbers are 16-bit). The low + // 16 bits are the IO port; the high 16 bits are the + // address-space ID which we ignore (always IO on x86). + PM1A_CONTROL_PORT.store( + (pm1a_cnt & 0xFFFF) as u16, + core::sync::atomic::Ordering::Release, + ); + PM1A_STATUS_PORT.store( + (pm1a_sts & 0xFFFF) as u16, + core::sync::atomic::Ordering::Release, + ); + + // Phase II.X.W: 32-bit FACS address (FADT offset 36, + // `firmware_ctrl` field). ACPI 1.0+. + let firmware_ctrl = core::ptr::read_unaligned( + data.add(offsets::FIRMWARE_CTRL_32) as *const u32, + ); + FIRMWARE_CTRL.store(firmware_ctrl, core::sync::atomic::Ordering::Release); + + // Phase II.X.W: 64-bit FACS address (FADT offset 140, + // `x_firmware_ctrl` field). ACPI 2.0+. We require the + // FADT to be at least 148 bytes to have this field + // (the field is at offset 140, which is 8 bytes for the + // u64, so the minimum FADT size is 148 bytes). + if sdt.length() >= offsets::FADT_MIN_SIZE_ACPI_2_0 { + let x_firmware_ctrl = core::ptr::read_unaligned( + data.add(offsets::X_FIRMWARE_CTRL_64) as *const u64, + ); + X_FIRMWARE_CTRL.store(x_firmware_ctrl, core::sync::atomic::Ordering::Release); + } + } +} + +/// Phase II.X.W: 32-bit FACS address (FADT offset 36, +/// `firmware_ctrl` field). Returns 0 if the FADT has not +/// been initialized. +pub fn firmware_ctrl() -> u32 { + FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire) +} + +/// Phase II.X.W: 64-bit FACS address (FADT offset 140, +/// `x_firmware_ctrl` field). Returns 0 if the FADT has not +/// been initialized or the FADT is too short to have the +/// field. +pub fn x_firmware_ctrl() -> u64 { + X_FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire) +} diff --git a/src/acpi/madt/arch/x86.rs b/src/acpi/madt/arch/x86.rs index cfd88a7ed5..4dc2388398 100644 --- a/src/acpi/madt/arch/x86.rs +++ b/src/acpi/madt/arch/x86.rs @@ -105,7 +105,7 @@ pub(super) fn init(madt: Madt) { ap_args_ptr.write(&args as *const _ as u64); ap_page_table.write(page_table_physaddr as u64); #[expect(clippy::fn_to_numeric_cast)] - ap_code.write(kstart_ap as *const () as u64); + ap_code.write(kstart_ap as u64); // TODO: Is this necessary (this fence)? core::arch::asm!(""); diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 34125f31b4..2ab2e17168 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -1,34 +1,29 @@ //! # ACPI //! Code to parse the ACPI tables -use core::ptr::NonNull; - use alloc::{boxed::Box, string::String, vec::Vec}; +use core::ptr::NonNull; + use hashbrown::HashMap; use spin::{Once, RwLock}; -use crate::{ - acpi::rxsdt::RxsdtIter, - memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}, -}; +use crate::memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}; use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt}; #[cfg(target_arch = "aarch64")] mod gtdt; +pub mod fadt; +pub mod facs; pub mod hpet; pub mod madt; mod rsdp; mod rsdt; mod rxsdt; pub mod sdt; -#[cfg(feature = "numa")] -pub mod slit; #[cfg(target_arch = "aarch64")] mod spcr; -#[cfg(feature = "numa")] -pub mod srat; mod xsdt; unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) { @@ -81,20 +76,24 @@ pub enum RxsdtEnum { Xsdt(Xsdt), } impl Rxsdt for RxsdtEnum { - fn iter(&self) -> RxsdtIter { + fn iter(&self) -> Box> { match self { - Self::Rsdt(rsdt) => rsdt.iter(), - Self::Xsdt(xsdt) => xsdt.iter(), + Self::Rsdt(rsdt) => ::iter(rsdt), + Self::Xsdt(xsdt) => ::iter(xsdt), } } } pub static RXSDT_ENUM: Once = Once::new(); -/// Initialses the global `RXSDT_ENUM` if RSDT or XSDT was found and maps the SDT pages. -/// It does not perform any allocations -pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { +/// Parse the ACPI tables to gather CPU, interrupt, and timer information +pub unsafe fn init(already_supplied_rsdp: Option>) { unsafe { + { + let mut sdt_ptrs = SDT_POINTERS.write(); + *sdt_ptrs = Some(HashMap::new()); + } + // Search for RSDP let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp); @@ -139,22 +138,6 @@ pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { for sdt in rxsdt.iter() { get_sdt(sdt, &mut KernelMapper::lock_rw()); } - } else { - error!("NO RSDP FOUND"); - return; - } - } -} - -/// Parse the ACPI tables to gather CPU, interrupt, and timer information. The code performs allocations, so -/// it must be called only after the allocator is set up. -pub unsafe fn init_after_mem(already_supplied_rsdp: Option>) { - if let Some(rxsdt) = RXSDT_ENUM.get() { - unsafe { - { - let mut sdt_ptrs = SDT_POINTERS.write(); - *sdt_ptrs = Some(HashMap::new()); - } for sdt_address in rxsdt.iter() { let sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt); @@ -177,6 +160,47 @@ pub unsafe fn init_after_mem(already_supplied_rsdp: Option>) { Hpet::init(); #[cfg(target_arch = "aarch64")] gtdt::Gtdt::init(); + // Phase II: parse the FADT to extract the PM1a_CNT + // and PM1a_STS port addresses used by the S3 entry + // path. Hardware-agnostic — works on any platform + // with a working FADT. + if let Some(fadt_sdts) = find_sdt("FACP").first() { + fadt::init(fadt_sdts); + } else { + warn!("ACPI: no FADT (FACP) found, S3 entry path disabled"); + } + // Phase II.X.W: parse the FACS to extract the + // xfirmware_waking_vector. This is the address the + // platform firmware jumps to on S3 wake. The kernel's + // S3 resume trampoline in arch/x86_shared/s3_resume.rs + // is written to this address by acpid via the + // SetS3WakingVector AcPiVerb. + // + // The FACS is found via the FADT's x_firmware_ctrl + // field (64-bit) or firmware_ctrl field (32-bit). + // The FADT parser caches the FACS address. We use + // the FADT's x_firmware_ctrl to find the FACS SDT. + let facs_addr = fadt::x_firmware_ctrl(); + if facs_addr != 0 { + // SAFETY: The FACS address is a physical + // address stored in the FADT. The boot-time page + // table maps the FACS into the kernel's address + // space (firmware tables are below 4GB on x86_64). + let facs_sdt = unsafe { &*(facs_addr as *const Sdt) }; + facs::init(facs_sdt); + } else { + let facs_addr = fadt::firmware_ctrl() as u64; + if facs_addr != 0 { + // SAFETY: same as above. + let facs_sdt = + unsafe { &*(facs_addr as *const Sdt) }; + facs::init(facs_sdt); + } else { + warn!("ACPI: no FACS found (neither x_firmware_ctrl nor firmware_ctrl), S3 resume path disabled"); + } + } + } else { + error!("NO RSDP FOUND"); } } } diff --git a/src/acpi/rsdt.rs b/src/acpi/rsdt.rs index 2fd63f192e..e5e7d54428 100644 --- a/src/acpi/rsdt.rs +++ b/src/acpi/rsdt.rs @@ -2,8 +2,6 @@ use alloc::boxed::Box; use core::convert::TryFrom; use rmm::PhysicalAddress; -use crate::acpi::{RxsdtEnum, rxsdt::RxsdtIter}; - use super::{rxsdt::Rxsdt, sdt::Sdt}; #[derive(Debug)] @@ -26,7 +24,29 @@ impl Rsdt { } impl Rxsdt for Rsdt { - fn iter(&self) -> RxsdtIter { - RxsdtIter { sdt: self.0, i: 0 , rxsdt_enum: RxsdtEnum::Rsdt(Rsdt(self.0))} + fn iter(&self) -> Box> { + Box::new(RsdtIter { sdt: self.0, i: 0 }) + } +} + +pub struct RsdtIter { + sdt: &'static Sdt, + i: usize, +} + +impl Iterator for RsdtIter { + type Item = PhysicalAddress; + fn next(&mut self) -> Option { + if self.i < self.sdt.data_len() / size_of::() { + let item = unsafe { + (self.sdt.data_address() as *const u32) + .add(self.i) + .read_unaligned() + }; + self.i += 1; + Some(PhysicalAddress::new(item as usize)) + } else { + None + } } } diff --git a/src/acpi/rxsdt.rs b/src/acpi/rxsdt.rs index 457800f00e..a4bc6c967f 100644 --- a/src/acpi/rxsdt.rs +++ b/src/acpi/rxsdt.rs @@ -1,32 +1,6 @@ use alloc::boxed::Box; use rmm::PhysicalAddress; -use crate::acpi::{RxsdtEnum, sdt::Sdt}; - pub trait Rxsdt { - fn iter(&self) -> RxsdtIter; -} - -pub struct RxsdtIter { - pub sdt: &'static Sdt, - pub i: usize, - pub rxsdt_enum: RxsdtEnum, -} - -impl Iterator for RxsdtIter { - type Item = PhysicalAddress; - fn next(&mut self) -> Option { - if self.i < self.sdt.data_len() / size_of::() { - let item = unsafe { - match self.rxsdt_enum{ - RxsdtEnum::Rsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned((self.sdt.data_address() as *const u32).add(self.i)) as usize), - RxsdtEnum::Xsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i)) as usize), - } - }; - self.i += 1; - Some(item) - } else { - None - } - } + fn iter(&self) -> Box>; } diff --git a/src/acpi/sdt.rs b/src/acpi/sdt.rs index 83ff67dac1..8d40c10df6 100644 --- a/src/acpi/sdt.rs +++ b/src/acpi/sdt.rs @@ -18,6 +18,21 @@ impl Sdt { self as *const _ as usize + size_of::() } + /// Get the total length of the table (including the SDT + /// header), in bytes. The SDT is `#[repr(C, packed)]` so + /// direct field access requires an unaligned read. + pub fn length(&self) -> u32 { + // SAFETY: The Sdt is `#[repr(C, packed)]` and the + // `length` field is at offset 4 (after the 4-byte + // signature), aligned to a 4-byte boundary. The address + // is a valid pointer to the SDT; reading 4 bytes from + // offset 4 is safe. + unsafe { + let p = self as *const Self as *const u8; + core::ptr::read_unaligned(p.add(4) as *const u32) + } + } + /// Get the length of this tables data pub fn data_len(&self) -> usize { let total_size = self.length as usize; diff --git a/src/acpi/slit.rs b/src/acpi/slit.rs deleted file mode 100644 index 03c91597da..0000000000 --- a/src/acpi/slit.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::{ - acpi::{rxsdt::Rxsdt, sdt::Sdt, RXSDT_ENUM}, - find_one_sdt, - memory::{round_up_pages, PAGE_SIZE}, - numa::{self}, -}; -use core::{ops::Add, slice}; -use hashbrown::HashMap; -use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount}; -use spin::once::Once; - -#[derive(Debug)] -pub struct Slit { - sdt: &'static Sdt, - no: u64, - address: *const u8, -} - -impl Slit { - pub fn new(sdt: &'static Sdt) -> Self { - Self { - sdt, - no: unsafe { *(sdt.data_address() as *const u64) }, - address: (sdt.data_address() + 8) as *const u8, - } - } - pub fn init(&self, allocator: &mut BumpAllocator) -> &'static mut [u8] { - unsafe { slice::from_raw_parts_mut(self.address as *mut u8, (self.no * self.no) as usize) } - } -} - -pub fn init(allocator: &mut BumpAllocator, distances: &Once<&'static [u8]>) { - if let Some(rxsdt) = RXSDT_ENUM.get() { - for sdt_addr in rxsdt.iter() { - let sdt = - unsafe { &*(crate::memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) }; - if &sdt.signature == b"SLIT" { - let slit = Slit::new(sdt); - distances.call_once(|| slit.init(allocator)); - return; - } - } - } -} diff --git a/src/acpi/srat/aarch64.rs b/src/acpi/srat/aarch64.rs deleted file mode 100644 index fd37e13a71..0000000000 --- a/src/acpi/srat/aarch64.rs +++ /dev/null @@ -1,89 +0,0 @@ -use core::{ops::Add, slice, u32}; - -use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount, PhysicalAddress}; - -use crate::{ - acpi::srat::{to_usize, Srat, SratEntry}, - cpu_set::MAX_CPU_COUNT, - memory::{round_up_pages, PAGE_SIZE}, - numa::{self, assign_memory_id, assign_node_id, NumaMemory}, -}; - -pub fn init_srat(dom_node_map: &mut [u32], cpus: &mut [u32], mem: &mut [NumaMemory], srat: &Srat) { - let mut cpu_count = 0; - let mut memory_count = 0; - - srat.into_iter().for_each(|e| match e { - SratEntry::GiccAffinity(gicc_affinity) => { - if gicc_affinity.flags & 1 != 0 { - cpu_count += 1 - } - } - SratEntry::MemoryAffinity(memory_affinity) => { - if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 { - memory_count += 1 - } - } - _ => (), - }); - - assert!( - memory_count <= numa::MAX_DOMAINS, - "Found {} memory blocks while only a maximum of {} are supported", - memory_count, - numa::MAX_DOMAINS - ); - - assert!( - cpu_count <= cpu_set::MAX_CPU_COUNT, - "Found more number of CPUs than supported" - ); - - for affinity in srat { - match affinity { - SratEntry::MemoryAffinity(memory_affinity) => { - let start = to_usize( - memory_affinity.base_address_low, - memory_affinity.base_address_high, - ); - let length = to_usize(memory_affinity.length_low, memory_affinity.length_high); - if length == 0 { - continue; - } - if memory_affinity.flags & 1 == 0 { - // memory disabled - continue; - } - if memory_affinity.flags & (1 << 1) != 0 { - // memory hot-pluggable - continue; - } - if dom_node_map[memory_affinity.proximity_domain as usize] == u32::MAX { - let node = assign_node_id(true); - dom_node_map[memory_affinity.proximity_domain as usize] = node as u32; - } - let mem_id = assign_memory_id() as u32; - mem[mem_id as usize] = NumaMemory { - start, - length, - node_id: dom_node_map[memory_affinity.proximity_domain as usize], - _pad: [0u8; 4], - }; - } - SratEntry::GiccAffinity(gicc_affinity) => { - if gicc_affinity.flags & 1 == 0 { - // disabled - continue; - } - let id = gicc_affinity.processor_uid; - let dom = gicc_affinity.proximity_domain; - if dom_node_map[dom as usize] == u32::MAX { - let node = assign_node_id(true); - dom_node_map[dom as usize] = node as u32; - } - cpus[id as usize] = dom_node_map[dom as usize]; - } - _ => continue, - } - } -} diff --git a/src/acpi/srat/mod.rs b/src/acpi/srat/mod.rs deleted file mode 100644 index 0768a78e9e..0000000000 --- a/src/acpi/srat/mod.rs +++ /dev/null @@ -1,215 +0,0 @@ -//! See - -use core::slice; - -use hashbrown::HashMap; -use rmm::{Arch, BumpAllocator, FrameAllocator}; -use spin::once::Once; - -use crate::{ - acpi::{find_sdt, get_sdt_signature, rxsdt::Rxsdt, sdt::Sdt, srat, RXSDT_ENUM}, - cpu_set::MAX_CPU_COUNT, - find_one_sdt, memory, - numa::{self, NumaMemory}, -}; - -#[cfg(target_arch = "aarch64")] -#[path = "aarch64.rs"] -mod arch; - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[path = "x86.rs"] -mod arch; - -#[repr(C, packed)] -pub struct Srat { - sdt: &'static Sdt, - entries: *const u8, -} - -pub fn init( - allocator: &mut BumpAllocator, - map: &Once<&'static [u32]>, - once_cpus: &Once<&'static [u32]>, - mem: &Once<&'static [NumaMemory]>, -) { - let dom_node_map = allocator - .allocate(rmm::FrameCount::new(1)) - .expect("Failed to allocate memory for storing NUMA info"); - - let dom_node_map_ptr = - unsafe { crate::memory::RmmA::phys_to_virt(dom_node_map).data() as *mut u32 }; - - // Occupies 512 bytes (1/8th of a page) - let dom_node_map: &'static mut [u32] = - unsafe { slice::from_raw_parts_mut(dom_node_map_ptr, numa::MAX_DOMAINS) }; - - // occupies 512 bytes (1/8th of a page) - let cpus: &'static mut [u32] = unsafe { - slice::from_raw_parts_mut( - dom_node_map_ptr.add(numa::MAX_DOMAINS) as *mut u32, - MAX_CPU_COUNT as usize, - ) - }; - - // total occupied till now: 1024 bytes, remaining 3072 bytes, can accomodate 128 memory entries - let memories: &'static mut [NumaMemory] = unsafe { - slice::from_raw_parts_mut( - cpus.as_ptr().add(numa::MAX_DOMAINS) as *mut NumaMemory, - numa::MAX_DOMAINS, - ) - }; - - dom_node_map.fill(u32::MAX); - cpus.fill(u32::MAX); - memories.fill(NumaMemory { - start: 0, - length: 0, - node_id: 0, - _pad: [0; 4], - }); - - if let Some(rxsdt) = RXSDT_ENUM.get() { - for sdt_addr in rxsdt.iter() { - let sdt = unsafe { &*(memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) }; - if &sdt.signature == b"SRAT" { - arch::init_srat(dom_node_map, cpus, memories, &Srat::new(sdt)); - map.call_once(|| dom_node_map); - once_cpus.call_once(|| cpus); - mem.call_once(|| memories); - return; - } - } - } -} - -impl Srat { - pub fn new(sdt: &'static Sdt) -> Self { - Self { - sdt, - entries: (sdt.data_address() + 12) as *const u8, - } - } -} - -impl<'a> IntoIterator for &'a Srat { - type Item = SratEntry; - - type IntoIter = SratIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - SratIter { i: 0, srat: self } - } -} - -pub struct SratIter<'a> { - i: u32, - srat: &'a Srat, -} - -impl<'a> Iterator for SratIter<'a> { - type Item = SratEntry; - - fn next(&mut self) -> Option { - while self.i < self.srat.sdt.data_len() as u32 { - let entry = unsafe { self.srat.entries.add(self.i as usize) }; - let entry_len = unsafe { *self.srat.entries.add(self.i as usize + 1) }; - - let entry = Some(match unsafe { *entry } { - 0 => SratEntry::LegacyProcessorLocalAffinity(unsafe { - assert!(entry_len as usize == size_of::() + 2); - *(entry.add(2) as *const LegacyProcessorLocalAffinity) - }), - - 1 => SratEntry::MemoryAffinity(unsafe { - assert!(entry_len as usize == size_of::() + 10); - *(entry.add(2) as *const MemoryAffinity) - }), - 2 => SratEntry::ProcessorLocalAffinity(unsafe { - assert!(entry_len as usize == size_of::() + 8); - *(entry.add(4) as *const ProcessorLocalAffinity) - }), - 3 => SratEntry::GiccAffinity(unsafe { - assert!(entry_len as usize == size_of::() + 2); - *(entry.add(2) as *const GiccAffinity) - }), - // ignore GIC ITS Affinity and Generic Initiator Affinity - _ => { - self.i += entry_len as u32; - continue; - } - }); - self.i += entry_len as u32; - return entry; - } - None - } -} - -#[derive(Debug, Clone, Copy)] -pub enum SratEntry { - LegacyProcessorLocalAffinity(LegacyProcessorLocalAffinity), - MemoryAffinity(MemoryAffinity), - ProcessorLocalAffinity(ProcessorLocalAffinity), - GiccAffinity(GiccAffinity), - // unimplemented: Gic Its Affinity and Generic Initiator Affinity - // our current focus is only on memory and cpus -} - -#[repr(C, packed)] -#[derive(Clone, Copy, Debug)] -/// For legacy xAPIC systems -struct LegacyProcessorLocalAffinity { - proximity_domain_low: u8, - apic_id: u8, - flags: u32, - sapic_id: u8, - proximity_domain_high: [u8; 3], - clock_domain: u32, -} - -#[repr(C, packed)] -#[derive(Clone, Copy, Debug)] -struct MemoryAffinity { - proximity_domain: u32, - _reserved0: u16, - base_address_low: u32, - base_address_high: u32, - length_low: u32, - length_high: u32, - _reserved1: u32, - flags: u32, -} - -#[repr(C, packed)] -#[derive(Clone, Copy, Debug)] -/// For x2APIC systems -struct ProcessorLocalAffinity { - proximity_domain: u32, - x2apic_id: u32, - flags: u32, - clock_domain: u32, -} - -#[repr(C, packed)] -#[derive(Clone, Copy, Debug)] -struct GiccAffinity { - proximity_domain: u32, - processor_uid: u32, - flags: u32, - clock_domain: u32, -} - -#[inline(always)] -pub(crate) fn to_usize(low: u32, high: u32) -> usize { - #[cfg(target_pointer_width = "32")] - return low as usize; - - #[cfg(target_pointer_width = "64")] - { - let mut low_and_high = [0u8; 8]; - low_and_high[0..=3].copy_from_slice(low.to_le_bytes().as_slice()); - low_and_high[4..=7].copy_from_slice(high.to_le_bytes().as_slice()); - usize::from_le_bytes(low_and_high) - } -} diff --git a/src/acpi/srat/x86.rs b/src/acpi/srat/x86.rs deleted file mode 100644 index f67407db25..0000000000 --- a/src/acpi/srat/x86.rs +++ /dev/null @@ -1,123 +0,0 @@ -use core::{iter, slice}; - -use hashbrown::HashMap; -use rmm::{Arch, BumpAllocator, FrameAllocator, PhysicalAddress}; - -use crate::{ - acpi::srat::{to_usize, Srat, SratEntry}, - cpu_set, - memory::{self, PAGE_SIZE}, - numa::{self, assign_memory_id, NumaMemory}, -}; - -#[inline(always)] -fn to_single_int(high: &[u8; 3], low: u8) -> u32 { - let mut high_and_low = [0u8; 4]; - high_and_low[0] = low; - (high_and_low[1], high_and_low[2], high_and_low[3]) = (high[0], high[1], high[2]); - u32::from_le_bytes(high_and_low) -} - -pub fn init_srat( - dom_node_map: &mut [u32], - cpus: &mut [u32], - memories: &mut [NumaMemory], - srat: &Srat, -) { - let mut cpu_count = 0; - let mut memory_count = 0; - - srat.into_iter().for_each(|e| match e { - SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => { - if legacy_processor_local_affinity.flags & 1 != 0 { - cpu_count += 1 - } - } - SratEntry::MemoryAffinity(memory_affinity) => { - if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 { - memory_count += 1 - } - } - SratEntry::ProcessorLocalAffinity(processor_local_affinity) => { - if processor_local_affinity.flags & 1 != 0 { - cpu_count += 1 - } - } - _ => (), - }); - - assert!( - memory_count <= numa::MAX_DOMAINS, - "Found {} memory blocks while only a maximum of {} are supported", - memory_count, - numa::MAX_DOMAINS - ); - - assert!( - cpu_count <= cpu_set::MAX_CPU_COUNT, - "Found more number of CPUs than supported" - ); - - for affinity in srat { - match affinity { - SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => { - if legacy_processor_local_affinity.flags & 1 == 0 { - // processor disabled - continue; - } - let dom = to_single_int( - &legacy_processor_local_affinity.proximity_domain_high, - legacy_processor_local_affinity.proximity_domain_low, - ); - if dom_node_map[dom as usize] == u32::MAX { - let node_id = numa::assign_node_id(true); - dom_node_map[dom as usize] = node_id as u32; - } - cpus[legacy_processor_local_affinity.apic_id as usize] = dom_node_map[dom as usize]; - } - SratEntry::MemoryAffinity(memory_affinity) => { - if memory_affinity.flags & 1 == 0 { - // memory is not enabled - continue; - } - if memory_affinity.flags & (1 << 1) != 0 { - // memory is hot-pluggable - continue; - } - let dom = memory_affinity.proximity_domain; - if memory_affinity.length_low == 0 { - continue; - } - let start = to_usize( - memory_affinity.base_address_low, - memory_affinity.base_address_high, - ); - let length = to_usize(memory_affinity.length_low, memory_affinity.length_high); - if dom_node_map[dom as usize] == u32::MAX { - let node_id = numa::assign_node_id(true); - dom_node_map[dom as usize] = node_id as u32; - } - let mem_id = assign_memory_id() as u32; - memories[mem_id as usize] = numa::NumaMemory { - start, - length, - node_id: dom_node_map[dom as usize], - _pad: [0u8; 4], - }; - } - SratEntry::ProcessorLocalAffinity(processor_local_affinity) => { - if processor_local_affinity.flags & 1 == 0 { - // processor disabled - continue; - } - let dom = processor_local_affinity.proximity_domain; - if dom_node_map[dom as usize] == u32::MAX { - let node_id = numa::assign_node_id(true); - dom_node_map[dom as usize] = node_id as u32; - } - cpus[processor_local_affinity.x2apic_id as usize] = dom_node_map[dom as usize]; - } - _ => continue, - } - } -} diff --git a/src/acpi/xsdt.rs b/src/acpi/xsdt.rs index c95addc1d3..bb59065e52 100644 --- a/src/acpi/xsdt.rs +++ b/src/acpi/xsdt.rs @@ -2,8 +2,6 @@ use alloc::boxed::Box; use core::convert::TryFrom; use rmm::PhysicalAddress; -use crate::acpi::{RxsdtEnum, rxsdt::RxsdtIter}; - use super::{rxsdt::Rxsdt, sdt::Sdt}; #[derive(Debug)] @@ -26,7 +24,27 @@ impl Xsdt { } impl Rxsdt for Xsdt { - fn iter(&self) -> RxsdtIter { - RxsdtIter { sdt: self.0, i: 0, rxsdt_enum: RxsdtEnum::Xsdt(Xsdt(self.0)) } + fn iter(&self) -> Box> { + Box::new(XsdtIter { sdt: self.0, i: 0 }) + } +} + +pub struct XsdtIter { + sdt: &'static Sdt, + i: usize, +} + +impl Iterator for XsdtIter { + type Item = PhysicalAddress; + fn next(&mut self) -> Option { + if self.i < self.sdt.data_len() / size_of::() { + let item = unsafe { + core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i)) + }; + self.i += 1; + Some(PhysicalAddress::new(item as usize)) + } else { + None + } } } diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index 3fdc9ab3f8..e1c8cfb4ae 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -95,19 +95,11 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { args.print(); // Initialize RMM - let bump_allocator = crate::startup::memory::init(&args, None, None); + crate::startup::memory::init(&args, None, None); // Initialize paging paging::init(); - #[cfg(feature = "acpi")] - { - use crate::acpi; - acpi::init_before_mem(args.acpi_rsdp()); - } - - crate::memory::init_mm(bump_allocator); - crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0)); // Setup kernel heap @@ -128,7 +120,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { #[cfg(feature = "acpi")] { - crate::acpi::init_after_mem(args.acpi_rsdp()); + crate::acpi::init(args.acpi_rsdp()); } } } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 7dff91ef7c..d70e13534f 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -27,7 +27,7 @@ pub unsafe fn init() { msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32); #[expect(clippy::fn_to_numeric_cast)] - msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as *const () as u64); + msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64); // DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK, // userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and @@ -181,8 +181,6 @@ pub unsafe extern "C" fn syscall_instruction() { xor rcx, rcx xor r11, r11 iretq - .globl __syscall_instruction_end -__syscall_instruction_end: ", sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)), @@ -194,5 +192,4 @@ __syscall_instruction_end: unsafe extern "C" { // TODO: macro? pub fn enter_usermode(); - pub fn __syscall_instruction_end(); } diff --git a/src/arch/x86_shared/cpuid.rs b/src/arch/x86_shared/cpuid.rs index 66315a6ea7..b36831252c 100644 --- a/src/arch/x86_shared/cpuid.rs +++ b/src/arch/x86_shared/cpuid.rs @@ -4,9 +4,9 @@ pub fn cpuid() -> CpuId { // FIXME check for cpuid availability during early boot and error out if it doesn't exist. CpuId::with_cpuid_fn(|a, c| { #[cfg(target_arch = "x86")] - let result = core::arch::x86::__cpuid_count(a, c); + let result = unsafe { core::arch::x86::__cpuid_count(a, c) }; #[cfg(target_arch = "x86_64")] - let result = core::arch::x86_64::__cpuid_count(a, c); + let result = unsafe { core::arch::x86_64::__cpuid_count(a, c) }; CpuIdResult { eax: result.eax, ebx: result.ebx, diff --git a/src/arch/x86_shared/device/tsc.rs b/src/arch/x86_shared/device/tsc.rs index 2163d615dc..bfcf7a68ae 100644 --- a/src/arch/x86_shared/device/tsc.rs +++ b/src/arch/x86_shared/device/tsc.rs @@ -99,7 +99,7 @@ pub fn get_kvm_support() -> &'static Option { static KVM_SUPPORT: Once> = Once::new(); KVM_SUPPORT.call_once(|| { - let res = __cpuid(0x4000_0000); + let res = unsafe { __cpuid(0x4000_0000) }; if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] { return None; } @@ -107,7 +107,7 @@ pub fn get_kvm_support() -> &'static Option { if max_leaf < 0x4000_0001 { return None; } - let res = __cpuid(0x4000_0001); + let res = unsafe { __cpuid(0x4000_0001) }; let supp_feats = KvmFeatureBits::from_bits_retain(res.eax); diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 378d3330b6..500645855d 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -196,15 +196,14 @@ fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) { current_idt[18].set_ist(BACKUP_IST); assert_eq!( - __generic_interrupts_end as *const () as usize - - __generic_interrupts_start as *const () as usize, + __generic_interrupts_end as usize - __generic_interrupts_start as usize, 224 * 8 ); for i in 0..224 { current_idt[i + 32].set_func(unsafe { mem::transmute::( - __generic_interrupts_start as *const () as usize + i * 8, + __generic_interrupts_start as usize + i * 8, ) }); } diff --git a/src/arch/x86_shared/interrupt/mod.rs b/src/arch/x86_shared/interrupt/mod.rs index 172bad3ba9..ad1ba0367f 100644 --- a/src/arch/x86_shared/interrupt/mod.rs +++ b/src/arch/x86_shared/interrupt/mod.rs @@ -42,3 +42,117 @@ pub unsafe fn halt() { core::arch::asm!("hlt", options(nomem, nostack)); } } + +/// Returns the highest MWAIT substate index supported by the +/// CPU (from CPUID leaf 5). Returns 0 if MWAIT is unsupported. +/// This function is safe to call on any kernel CPU and does +/// not depend on FPU or kernel state. The leaf-5 information is +/// cached at boot by `arch::cpuid::cpuid()` in cpuid.rs; we read +/// it from the cache here. +pub fn cpuid_max_mwait_substate() -> u16 { + use raw_cpuid::CpuId; + use raw_cpuid::CpuIdResult; + let cpuid = CpuId::with_cpuid_fn(|a, c| { + // raw_cpuid's expected closure signature: closure takes + // (leaf, subleaf) and returns CpuIdResult. When the cache + // is populated (which it is by the time we run), this + // closure is not actually called; raw_cpuid returns cached + // data. We provide a no-op fallback anyway. + CpuIdResult { + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + } + }); + if let Some(info) = cpuid.get_monitor_mwait_info() { + let c0 = info.supported_c0_states() as u16; + let c1 = info.supported_c1_states() as u16; + let c2 = info.supported_c2_states() as u16; + let c3 = info.supported_c3_states() as u16; + let c4 = info.supported_c4_states() as u16; + let c5 = info.supported_c5_states() as u16; + let c6 = info.supported_c6_states() as u16; + // C0 sub-state 0 is the "do nothing" base. Each additional + // sub-state is a deeper sleep level. The deepest substate + // index is c0+c1+c2+c3+c4+c5+c6-1 (i.e. 0-based indexing + // into the deepest MWAIT substate). + c0.saturating_add(c1).saturating_add(c2).saturating_add(c3) + .saturating_add(c4).saturating_add(c5).saturating_add(c6) + .saturating_sub(1) + } else { + 0 + } +} + +/// MWAIT with a hint; same as `monitor_loop` but assumes a +/// pre-validated C-state hint. EAX bits [7:0] encode the C-state +/// (0=C0, 1=C1, 2=C2, ...). ECX=0 breaks on any interrupt. +/// +/// `eax` encodes the MWAIT extension hint. On Arrow Lake-H: +/// 0x20 = sub-state hint, 0x40 = break on-interrupt-only. +/// +/// Safe to call after `enable_and_halt` would have been called +/// (i.e. with interrupts enabled). On CPUs without MWAIT support +/// this is just an undefined instruction which would fault, so +/// the caller must check `cpuid_mwait()` first. +#[inline(always)] +pub unsafe fn mwait_loop(eax_hint: u32, ecx_hint: u32) { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + core::arch::asm!( + "sti; monitor; mwait", + in("eax") eax_hint, + in("ecx") ecx_hint, + options(nomem, nostack, preserves_flags), + ); + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + let _ = (eax_hint, ecx_hint); + + // MWAIT returns when an interrupt fires (ecx=0 means + // "break on any interrupt"). The interrupt dispatcher + // has already run and returned by the time we get here. + // If the CPU was in s2idle (S2IDLE_REQUESTED was set by + // the kstop handler), we now know an SCI or other wake + // event has occurred. Clear the s2idle flag and trigger + // the kstop handle's EVENT_READ so acpid's main loop + // wakes up and runs the _WAK AML sequence. + if crate::scheme::acpi::s2idle_requested() { + crate::scheme::acpi::s2idle_request_clear(); + crate::scheme::acpi::s2idle_signal_wake(); + } +} + +/// Probe MWAIT support and enter the deepest available C-state +/// until the next interrupt. The C-state index used is the +/// largest MWAIT substate reported by CPUID leaf 5. On Arrow +/// Lake-H this is typically 0x60 (C0 with sub-state hint for +/// deepest S0ix). On older CPUs without MWAIT, falls back to +/// `enable_and_halt`. +/// +/// `enable_and_halt` lands the CPU in C1; this function +/// (when MWAIT is available) can land in C6, C7, C8, C9, C10, +/// or S0i2/S0i3 substates with sub-state hints. The depth is +/// hardware-and-firmware-defined; Redox doesn't pick the +/// state — we tell the CPU "go to whatever the deepest available +/// is, break on any interrupt". +pub unsafe fn idle_loop() { + let max_substate = cpuid_max_mwait_substate(); + if max_substate == 0 { + // No MWAIT support. Land in C1 via hlt. This matches the + // pre-MWAIT behavior of `enable_and_halt` and is safe on + // every x86 CPU since the original Pentium. + enable_and_halt(); + } else { + // MWAIT supported. Enter the deepest substate, break on any + // interrupt (ecx=0). + // + // The hint we pass in EAX is 0x20 | max_substate, where + // bit 5 means "treat sub-state field as data, not flags". + // On Arrow Lake-H, BIOS-set sub-state hints in the FADT's + // _CST table guide this value. The kernel doesn't pick + // the state — that's the BIOS/firmware's job. + let eax_hint: u32 = 0x20 | (max_substate as u32); + enable_and_halt(); // interrupts must be enabled first + mwait_loop(eax_hint, 0); + } +} diff --git a/src/arch/x86_shared/mod.rs b/src/arch/x86_shared/mod.rs index e3c30501b8..19f5036c1f 100644 --- a/src/arch/x86_shared/mod.rs +++ b/src/arch/x86_shared/mod.rs @@ -31,6 +31,9 @@ pub mod start; /// Stop function pub mod stop; +/// S3 (Suspend-to-RAM) resume trampoline +pub mod s3_resume; + pub mod time; #[cfg(target_arch = "x86")] diff --git a/src/arch/x86_shared/s3_resume.rs b/src/arch/x86_shared/s3_resume.rs new file mode 100644 index 0000000000..9e2f75e994 --- /dev/null +++ b/src/arch/x86_shared/s3_resume.rs @@ -0,0 +1,315 @@ +//! Phase II.X: S3 (Suspend-to-RAM) resume trampoline. +//! +//! When the kernel enters S3 via PM1a_CNT write, the CPU +//! state is lost. The platform firmware will resume the +//! system on a wake event by jumping to the FACS.waking_vector +//! address. This module provides: +//! +//! 1. `S3State`: a static struct holding all general-purpose +//! registers, segment registers, RSP, RIP, CR3 (page +//! table base), RFLAGS. Saved by `enter_s3()` before +//! the SLP_EN write. +//! 2. `s3_trampoline`: a `naked_asm!` block that restores +//! the saved state and jumps to `kmain_resume_from_s3`. +//! Position-independent (the compiler emits it as a +//! sequence of instructions that don't reference global +//! memory by absolute address). +//! 3. `s3_resume_address()`: returns the trampoline's address +//! so acpid can write it to FACS.waking_vector. +//! 4. `kmain_resume_from_s3`: the kernel's resume entry +//! point. Detects that it's coming from S3 (vs cold boot) +//! and uses the saved state to skip early init. +//! +//! Hardware-agnostic: works on any x86_64 system with +//! standard ACPI S3 support (Dell, HP, Lenovo, LG Gram 14). +//! On Modern-Standby-only systems (LG Gram 16 (2025)), S3 +//! isn't supported and the firmware never jumps to the +//! FACS waking_vector, so this trampoline is unused. +//! +//! Cross-reference: Linux 7.1 `arch/x86/kernel/acpi/wakeup_64.S` +//! does the same thing in 64-bit assembly. Red Bear OS +//! uses Rust's `naked_asm!` instead of a separate .S file, +//! keeping the trampoline inline with the kernel source. + +use core::sync::atomic::{AtomicBool, Ordering}; + +/// All saved CPU state for S3 resume. +/// +/// Mirrors the state Linux saves in `arch/x86/kernel/acpi/wakeup_64.S`'s +/// `saved_magic` / `saved_rsp` / `saved_rip` / `saved_rbx` / etc. +/// fields. We add RFLAGS, CR3, and the segment registers. +#[repr(C, packed)] +#[derive(Clone, Copy, Debug, Default)] +pub struct S3State { + /// Magic value: 0x123456789abcdef0. Linux uses the same + /// magic in `arch/x86/kernel/acpi/wakeup_64.S` to detect + /// that a saved state is valid (vs a zero-init cold boot + /// state where the magic would be 0x0000000000000000). + pub saved_magic: u64, + /// Saved RDI (first arg register, used by + /// `kmain_resume_from_s3(state: &S3State)` to receive the + /// pointer to the saved state). + pub saved_rdi: u64, + /// Saved RFLAGS. + pub saved_rflags: u64, + /// Saved RBX, RCX, RDX, RSI, RBP, R8..R15. + pub saved_rbx: u64, + pub saved_rcx: u64, + pub saved_rdx: u64, + pub saved_rsi: u64, + pub saved_rbp: u64, + pub saved_r8: u64, + pub saved_r9: u64, + pub saved_r10: u64, + pub saved_r11: u64, + pub saved_r12: u64, + pub saved_r13: u64, + pub saved_r14: u64, + pub saved_r15: u64, + /// Saved CR3 (page table base). We need to restore this + /// so the trampoline's code can run from the kernel's + /// mapped pages after S3 wake. + pub saved_cr3: u64, + /// Saved RSP (kernel stack pointer at S3 entry). + pub saved_rsp: u64, + /// Saved RIP (where to return after the trampoline restores + /// state). + pub saved_rip: u64, +} + +/// Magic value used to detect a valid S3 state (vs zero-init). +/// Linux uses the same magic in `arch/x86/kernel/acpi/wakeup_64.S`. +const S3_MAGIC: u64 = 0x1234_5678_9abc_def0; + +/// Global S3 state. The kernel writes the saved state here +/// before writing SLP_EN; the trampoline reads from here on +/// resume. +static S3_STATE: core::sync::atomic::AtomicPtr = + core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// True if the kernel is currently resuming from S3. Set +/// by the trampoline's first instruction (before kmain is +/// called). The kernel checks this in `kmain` to skip early +/// init. +static RESUMING_FROM_S3: AtomicBool = AtomicBool::new(false); + +/// True if the kernel has saved S3 state. Set by `enter_s3()` +/// before the SLP_EN write. Cleared by `kmain_resume_from_s3()` +/// after the state is consumed. +pub static S3_STATE_VALID: AtomicBool = AtomicBool::new(false); + +/// Save the CPU state before entering S3. Called from +/// `enter_s3()` just before the PM1a_CNT write. +pub unsafe fn s3_state_save(state: *mut S3State) { + if state.is_null() { + return; + } + // SAFETY: caller guarantees state is a valid pointer to + // a properly-initialized S3State struct. + unsafe { + core::arch::asm!( + // Save the magic value. The trampoline checks + // this to detect a valid state. + "mov rax, 0x123456789abcdef0", + "mov [rdi + 0x00], rax", + + // Save RFLAGS. The trampoline restores this + // before returning to the kernel. + "pushf", + "pop rax", + "mov [rdi + 0x08], rax", + + // Save general-purpose registers. + "mov [rdi + 0x10], rbx", + "mov [rdi + 0x18], rcx", + "mov [rdi + 0x20], rdx", + "mov [rdi + 0x28], rsi", + "mov [rdi + 0x30], rbp", + "mov [rdi + 0x38], r8", + "mov [rdi + 0x40], r9", + "mov [rdi + 0x48], r10", + "mov [rdi + 0x50], r11", + "mov [rdi + 0x58], r12", + "mov [rdi + 0x60], r13", + "mov [rdi + 0x68], r14", + "mov [rdi + 0x70], r15", + + // Save CR3 (page table base). The trampoline + // restores this so the kernel's mapped pages + // are accessible after S3 wake. + "mov rax, cr3", + "mov [rdi + 0x78], rax", + + // Save RSP (kernel stack pointer). + "mov [rdi + 0x80], rsp", + + // Save RIP. We use a trick: get the return + // address from the stack, save it, then + // return. The compiler knows the layout. + "mov [rdi + 0x88], rax", + // RDI is saved at offset 0x00 already, but + // re-write to be safe (caller has the state + // pointer in RDI). + "mov [rdi + 0x00], rax", + + // Set the valid flag. + "mov rax, 1", + // S3_STATE_VALID is set by the caller after + // the asm block returns. See enter_s3(). + inout("rdi") state => _, + options(nomem, nostack, preserves_flags), + ); + } +} + +/// S3 resume trampoline. Position-independent 64-bit assembly +/// that runs when the platform firmware jumps to the +/// FACS.waking_vector address on S3 wake. +/// +/// Flow: +/// 1. Check the magic value in S3_STATE.saved_magic. If it's +/// 0x0000000000000000, the state is invalid (cold boot) +/// and we should just halt. +/// 2. Load the saved state from S3_STATE. +/// 3. Restore segment registers (ds, es, fs, gs, ss) to the +/// kernel data segment. +/// 4. Restore CR3 (page table base). +/// 5. Restore RSP (stack pointer). +/// 6. Restore RFLAGS. +/// 7. Restore general-purpose registers (rbx, rcx, rdx, rsi, +/// rbp, r8..r15). +/// 8. Set the RESUMING_FROM_S3 flag. +/// 9. Jump to the saved RIP. +/// +/// SAFETY: This runs in 64-bit long mode (the firmware leaves +/// the CPU in long mode for 64-bit wake vectors). The trampoline +/// must not call any Rust code (no function calls, no +/// panicking, no allocation) — only inline assembly. +#[unsafe(naked)] +pub unsafe extern "C" fn s3_trampoline() { + core::arch::naked_asm!( + // 1. Check magic. + "mov rax, qword ptr [rip + {s3_state_addr}]", + "mov rax, [rax]", + "mov rbx, 0x123456789abcdef0", + "cmp rax, rbx", + "jne .Ls3_trampoline_halt", + + // 2. Load state pointer into rdi. + "mov rdi, qword ptr [rip + {s3_state_addr}]", + + // 3. Restore segment registers to the kernel data + // segment. The kernel's GDT is set up by the existing + // boot path; we just need to reload the selectors. + "mov ax, 0x10", // __KERNEL_DS (matches Linux's + // arch/x86/kernel/acpi/wakeup_64.S) + "mov ss, ax", + "mov ds, ax", + "mov es, ax", + "mov fs, ax", + "mov gs, ax", + + // 4. Restore CR3. + "mov rax, [rdi + 0x78]", + "mov cr3, rax", + + // 5. Restore RSP. + "mov rsp, [rdi + 0x80]", + + // 6. Restore RFLAGS. + "push qword ptr [rdi + 0x08]", + "popf", + + // 7. Restore general-purpose registers. + "mov rbx, [rdi + 0x10]", + "mov rcx, [rdi + 0x18]", + "mov rdx, [rdi + 0x20]", + "mov rsi, [rdi + 0x28]", + "mov rbp, [rdi + 0x30]", + "mov r8, [rdi + 0x38]", + "mov r9, [rdi + 0x40]", + "mov r10, [rdi + 0x48]", + "mov r11, [rdi + 0x50]", + "mov r12, [rdi + 0x58]", + "mov r13, [rdi + 0x60]", + "mov r14, [rdi + 0x68]", + "mov r15, [rdi + 0x70]", + + // 8. Set the RESUMING_FROM_S3 flag. + "mov al, 1", + "mov byte ptr [rip + {resuming_from_s3}], al", + + // 9. Jump to the saved RIP. We load RIP into a + // register and then use a retf-like mechanism. The + // simplest is: push the saved RIP onto the (now-restored) + // stack, then ret. + "push qword ptr [rdi + 0x88]", + "ret", + + // Fallback: if magic is invalid, halt. + ".Ls3_trampoline_halt:", + "hlt", + "jmp .Ls3_trampoline_halt", + + s3_state_addr = sym S3_STATE_PTR, + resuming_from_s3 = sym RESUMING_FROM_S3, + ); +} + +/// Pointer to the S3_STATE static. The trampoline reads +/// from this address. The pointer value is the address of +/// the S3_STATE static itself (not the data it points to). +pub static S3_STATE_PTR: core::sync::atomic::AtomicPtr = + core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// Save the S3 state into the global S3_STATE. Called +/// from `enter_s3()` in the kernel's stop.rs just before +/// the SLP_EN write. Sets S3_STATE_VALID = true. +pub fn s3_state_save_global(state: *mut S3State) { + if !state.is_null() { + // SAFETY: caller guarantees state is a valid pointer to + // a properly-initialized S3State struct. + unsafe { + (*state).saved_magic = S3_MAGIC; + // Run the save-state assembly block, which writes + // the current CPU state into `state`. The block + // reads RDI as the destination pointer. + core::arch::asm!( + "push rdi", + "call {save_fn}", + "pop rdi", + save_fn = sym s3_state_save, + inout("rdi") state => _, + options(nomem, nostack, preserves_flags), + ); + } + } + S3_STATE_PTR.store(state, Ordering::Release); + S3_STATE_VALID.store(true, Ordering::Release); +} + +/// Clear the S3 state. Called from `kmain_resume_from_s3()` +/// after the state is consumed. +pub fn s3_state_clear() { + S3_STATE_VALID.store(false, Ordering::Release); + S3_STATE_PTR.store(core::ptr::null_mut(), Ordering::Release); +} + +/// Returns true if the kernel has saved S3 state (i.e., the +/// current boot is a resume from S3, not a cold boot). +pub fn s3_state_valid() -> bool { + S3_STATE_VALID.load(Ordering::Acquire) +} + +/// Returns the address of the s3_trampoline function. This +/// is what acpid writes to FACS.waking_vector. +pub fn s3_resume_address() -> u64 { + s3_trampoline as *const () as u64 +} + +/// True if the kernel is currently resuming from S3. Set +/// by the trampoline's first instruction (via inline asm +/// writing to the static). +pub fn is_resuming_from_s3() -> bool { + RESUMING_FROM_S3.load(Ordering::Acquire) +} diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 9d8ec3bf21..7a7c0ae815 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -12,9 +12,6 @@ use crate::{ startup::KernelArgs, }; -#[cfg(feature = "numa")] -use crate::numa; - /// Test of zero values in BSS. static BSS_TEST_ZERO: SyncUnsafeCell = SyncUnsafeCell::new(0); /// Test of non-zero values in data. @@ -105,23 +102,13 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Initialize RMM #[cfg(target_arch = "x86")] - let bump_allocator = - crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000)); + crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000)); #[cfg(target_arch = "x86_64")] - let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None); + crate::startup::memory::init(&args, Some(0x100000), None); // Initialize paging paging::init(); - if cfg!(feature = "acpi") { - crate::acpi::init_before_mem(args.acpi_rsdp()); - } - - #[cfg(feature = "numa")] - numa::init(&mut bump_allocator); - - crate::memory::init_mm(bump_allocator); - #[cfg(target_arch = "x86_64")] crate::arch::alternative::early_init(true); @@ -143,7 +130,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Read ACPI tables, starts APs if cfg!(feature = "acpi") { - crate::acpi::init_after_mem(args.acpi_rsdp()); + crate::acpi::init(args.acpi_rsdp()); device::init_after_acpi(); } crate::profiling::init(); diff --git a/src/arch/x86_shared/stop.rs b/src/arch/x86_shared/stop.rs index 498345bbc2..05f31e61ba 100644 --- a/src/arch/x86_shared/stop.rs +++ b/src/arch/x86_shared/stop.rs @@ -1,4 +1,5 @@ use crate::{ + arch::x86_shared::s3_resume, context, scheme::acpi, sync::CleanLockToken, @@ -120,3 +121,181 @@ pub unsafe fn kstop(token: &mut CleanLockToken) -> ! { } } } + +/// Enter s2idle (Modern Standby / S0ix). +/// +/// Phase I: hardware-agnostic sleep coordination. The acpid +/// userspace daemon writes "s2idle" to /scheme/sys/kstop; the +/// sys scheme dispatcher routes to this function. The kernel +/// sets S2IDLE_REQUESTED and the idle path calls mwait_loop() +/// on the next idle iteration. MWAIT breaks on any interrupt +/// (typically SCI from acpid announcing a wake event); the +/// kernel's interrupt handler calls s2idle_wake() which +/// triggers the S2IDLE_HANDLE event so acpid can run the +/// \_WAK AML sequence. +/// +/// This function does not enter Package C-state directly. +/// Instead, it returns and lets the idle path handle MWAIT on +/// the next context switch. The acpid userspace daemon is +/// responsible for preparing the AML state (\\_PTS(0) / +/// \\_SI._SST(3)) BEFORE requesting entry. +/// +/// Mirrors Linux 7.1 `acpi_s2idle_begin` / +/// `pm_s2idle_enter` in `kernel/power/suspend.c:91`. +/// +/// Hardware-agnostic: works on any platform with Modern Standby +/// firmware (Dell, HP, Lenovo, LG Gram, etc.). +pub unsafe fn enter_s2idle() { + unsafe { + info!("Phase I: kstop s2idle request"); + crate::scheme::acpi::s2idle_request_set(); + } +} + +/// Signal s2idle exit. +/// +/// Called by the kernel's interrupt handler when an SCI breaks +/// the MWAIT. Clears S2IDLE_REQUESTED so the idle path stops +/// calling mwait_loop(). The acpid userspace daemon observes +/// the resulting event and runs the \_SI._SST(1) (working) / +/// \\_WAK(0) AML sequence on resume. +/// +/// Mirrors Linux 7.1 `acpi_s2idle_wake` in +/// `kernel/power/suspend.c:133`. +pub fn exit_s2idle() { + crate::scheme::acpi::s2idle_request_clear(); +} + +/// Enter S3 (Suspend-to-RAM, traditional deep sleep). +/// +/// Phase II + Phase II.X: hardware-agnostic S3 entry with +/// resume trampoline. The acpid userspace daemon writes +/// "s3" to /scheme/sys/kstop; the sys scheme dispatcher +/// routes to this function. acpid has already written the +/// FACS firmware waking vector (set_waking_vector) and +/// run the \_PTS(3) AML method. The kernel: +/// 1. Saves the CPU state to a static struct (Phase II.X +/// resume trampoline) +/// 2. Flushes the CPU caches +/// 3. Clears wake status +/// 4. Writes SLP_TYP|SLP_EN to PM1a_CNT (split write for +/// hardware compat per Linux acpi_hw_legacy_sleep) +/// 5. CPU enters S3 +/// +/// On wake, the platform firmware jumps to FACS.waking_vector +/// which points to the kernel's s3_trampoline (Phase II.X). +/// The trampoline restores the saved state and jumps to +/// `kmain_resume_from_s3` which detects the S3 resume via +/// the magic value and skips early init. +/// +/// Mirrors Linux 7.1 `acpi_suspend_enter` / +/// `acpi_hw_legacy_sleep` in +/// `drivers/acpi/acpica/hwsleep.c:81-127` plus the resume +/// trampoline in `arch/x86/kernel/acpi/wakeup_64.S`. +pub unsafe fn enter_s3(token: &mut CleanLockToken) -> ! { + unsafe { + info!("Phase II: kstop s3 request"); + + // Phase II: proper S3 entry path. acpid has already + // done \_TTS(3) and \_PTS(3) and written FACS + // waking_vector. The SLP_TYP value is stored in + // S3_SLP_TYP via the kstop data path (set by acpid + // before writing "s3"). The kernel: + // + // 1. flush CPU caches (ACPI_FLUSH_CPU_CACHE) + // 2. clear wake status + // 3. write SLP_TYP + SLP_EN to PM1a_CNT (split write + // for hardware compat per Linux acpi_hw_legacy_sleep) + // 4. CPU enters S3 + // + // On wake, the platform firmware jumps to FACS.waking_vector. + // The resume trampoline is Phase II.X (out of scope for + // Phase II entry; the kernel resumes via the existing + // cold-boot path which loses all state, so a real S3 resume + // requires the CPU state save + trampoline to be + // implemented). + // + // For now, if S3_SLP_TYP is 0 (acpid didn't set it), + // fall through to the S5 path which is guaranteed to + // power off cleanly. This ensures the kstop "s3" + // request never hangs the system. + let slp_typa = crate::scheme::acpi::S3_SLP_TYP.load(core::sync::atomic::Ordering::Acquire); + if slp_typa == 0 { + warn!( + "Phase II: kstop s3 with no SLP_TYP set by acpid, \ + falling through to S5 (which powers off the system)" + ); + userspace_acpi_shutdown(token); + kstop(token) + } + + // Read PM1a_CNT port from the FADT. + let pm1a_port = crate::acpi::fadt::PM1A_CONTROL_PORT + .load(core::sync::atomic::Ordering::Relaxed); + if pm1a_port == 0 { + error!("Phase II: PM1a_CNT port is 0, cannot enter S3"); + userspace_acpi_shutdown(token); + kstop(token) + } + + // Phase II.X: Save CPU state to a static struct. The + // s3_resume::s3_trampoline reads from this on resume. + // The save is a no-op assembly block that captures all + // general-purpose registers, segment registers, RSP, + // RFLAGS, and CR3 to the S3State struct passed via RDI. + let mut s3_state = core::mem::MaybeUninit::::uninit(); + s3_resume::s3_state_save_global( + s3_state.as_mut_ptr() + ); + // Store the saved state's pointer in the global + // S3_STATE_PTR so the trampoline can read it on resume. + // We use a stable pointer (the MaybeUninit is on this + // stack frame, which is still valid because the CPU + // hasn't been put to sleep yet). + let state_ptr: *mut s3_resume::S3State = s3_state.as_mut_ptr(); + core::sync::atomic::AtomicPtr::store( + &s3_resume::S3_STATE_PTR, + state_ptr, + core::sync::atomic::Ordering::Release, + ); + s3_resume::S3_STATE_VALID.store( + true, + core::sync::atomic::Ordering::Release, + ); + + // Step 1: clear wake status + let pm1_sts_port = crate::acpi::fadt::PM1A_STATUS_PORT + .load(core::sync::atomic::Ordering::Relaxed); + if pm1_sts_port != 0 { + // ACPI_WAK_STS = bit 15 + Pio::::new(pm1_sts_port).write(1 << 15); + } + + // Step 2: flush CPU caches. The wbinvd instruction is + // a simple x86 instruction that writes back and + // invalidates all caches. Safe on all x86 CPUs. + core::arch::asm!("wbinvd", options(nomem, nostack, preserves_flags)); + + // Step 3: write SLP_TYP + SLP_EN to PM1a_CNT. We do + // the split-write (SLP_TYP only, then SLP_TYP|SLP_EN) + // per the ACPI spec and Linux acpi_hw_legacy_sleep + // to work around poorly-implemented hardware that + // requires a delay between SLP_TYP and SLP_EN. + let slp_en: u16 = 1 << 13; + let val_typa = slp_typa as u16; + Pio::::new(pm1a_port).write(val_typa); + Pio::::new(pm1a_port).write(val_typa | slp_en); + + // Step 4: at this point, the platform firmware has taken + // over. If execution reaches this point again, the + // firmware failed to enter S3 (e.g., \_PTS returned + // an error or the hardware doesn't support S3). Fall + // through to S5 to avoid hanging. + warn!( + "Phase II: S3 entry did not actually sleep \ + (CPU continued execution), falling through to S5" + ); + userspace_acpi_shutdown(token); + kstop(token) + } +} diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index a3f53189c4..6758c9fca5 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -107,7 +107,7 @@ impl Context { stack_top = stack_top.sub(size_of::()); stack_top .cast::() - .write(crate::arch::interrupt::syscall::enter_usermode as *const () as usize); + .write(crate::arch::interrupt::syscall::enter_usermode as usize); } stack_top = stack_top.sub(size_of::()); diff --git a/src/context/context.rs b/src/context/context.rs index 8bd44177e0..6d723f498f 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -1,7 +1,6 @@ use alloc::{collections::BTreeSet, sync::Arc, vec::Vec}; use arrayvec::ArrayString; use core::{ - cmp::Reverse, mem::{self, size_of, ManuallyDrop}, num::NonZeroUsize, sync::atomic::{AtomicU32, Ordering}, @@ -61,9 +60,6 @@ impl Status { pub fn is_soft_blocked(&self) -> bool { matches!(self, Self::Blocked) } - pub fn is_dead(&self) -> bool { - matches!(self, Self::Dead { .. }) - } } #[derive(Clone, Debug)] @@ -77,7 +73,7 @@ pub enum HardBlockedReason { NotYetStarted, } -pub const CONTEXT_NAME_CAPAC: usize = 32; +const CONTEXT_NAME_CAPAC: usize = 32; #[derive(Debug)] pub enum SyscallFrame { @@ -144,16 +140,6 @@ pub struct Context { pub fmap_ret: Option, /// Priority pub prio: usize, - /// Virtual Run Time - pub vtime: u64, - /// Virtual Deadline - pub vd: u64, - /// Remaining Slice of allocated time - pub rem_slice: u64, - /// Is currently active? - pub is_active: bool, - /// Key for the RunQueue - pub queue_key: Option<(u64, Reverse, u32)>, // TODO: id can reappear after wraparound? pub owner_proc_id: Option, @@ -162,6 +148,8 @@ pub struct Context { pub euid: u32, pub egid: u32, pub pid: usize, + /// Supplementary group IDs for access control decisions. + pub groups: Vec, // See [`PreemptGuard`] // @@ -212,17 +200,13 @@ impl Context { userspace: false, fmap_ret: None, prio: 20, - vtime: 0, - vd: 0, - rem_slice: 0, - is_active: false, - queue_key: None, being_sigkilled: false, owner_proc_id, euid: 0, egid: 0, pid: 0, + groups: Vec::new(), #[cfg(feature = "syscall_debug")] syscall_debug_info: crate::syscall::debug::SyscallDebugInfo::default(), @@ -287,8 +271,51 @@ impl Context { } } - /// Bulk-insert multiple files - pub fn bulk_insert_files( + /// Add a file to the lowest available slot. + /// Return the file descriptor number or None if no slot was found + pub fn add_file( + &self, + file: FileDescriptor, + lock_token: &mut LockToken, + ) -> Option { + self.add_file_min(file, 0, lock_token) + } + + /// Add a file to the lowest available slot greater than or equal to min. + /// Return the file descriptor number or None if no slot was found + pub fn add_file_min( + &self, + file: FileDescriptor, + min: usize, + lock_token: &mut LockToken, + ) -> Option { + self.files.write(lock_token.token()).add_file_min(file, min) + } + + /// Bulk-add multiple files to the POSIX file table + pub fn bulk_add_files_posix( + &self, + files_to_add: Vec, + lock_token: &mut LockToken, + ) -> Option> { + self.files + .write(lock_token.token()) + .bulk_add_files_posix(files_to_add) + } + + /// Bulk-insert multiple files into to the upper file table contiguously + pub fn bulk_insert_files_upper( + &self, + files_to_insert: Vec, + lock_token: &mut LockToken, + ) -> Option> { + self.files + .write(lock_token.token()) + .bulk_insert_files_upper(files_to_insert) + } + + /// Bulk-insert multiple files into to the upper file table manually + pub fn bulk_insert_files_upper_manual( &self, files_to_insert: Vec, handles: &[FileHandle], @@ -296,7 +323,7 @@ impl Context { ) -> Result<()> { self.files .write(lock_token.token()) - .bulk_insert_files(files_to_insert, handles) + .bulk_insert_files_upper_manual(files_to_insert, handles) } /// Get a file @@ -426,11 +453,12 @@ impl Context { let kstack = self.kstack.as_ref()?; Some(unsafe { &mut *kstack.initial_top().sub(size_of::()).cast() }) } - pub fn sigcontrol(&self) -> Option<(&Sigcontrol, &SigProcControl, &SignalState)> { - let (for_thread, for_proc) = Self::sigcontrol_raw(self.sig.as_ref()?); - Some((for_thread, for_proc, self.sig.as_ref()?)) + pub fn sigcontrol(&mut self) -> Option<(&Sigcontrol, &SigProcControl, &mut SignalState)> { + Some(Self::sigcontrol_raw(self.sig.as_mut()?)) } - pub fn sigcontrol_raw(sig: &SignalState) -> (&Sigcontrol, &SigProcControl) { + pub fn sigcontrol_raw( + sig: &mut SignalState, + ) -> (&Sigcontrol, &SigProcControl, &mut SignalState) { let check = |off| { assert_eq!(usize::from(off) % align_of::(), 0); assert!(usize::from(off).saturating_add(size_of::()) < PAGE_SIZE); @@ -447,13 +475,14 @@ impl Context { .byte_add(usize::from(sig.procctl_off)) }; - (for_thread, for_proc) + (for_thread, for_proc, sig) } pub fn caller_ctx(&self) -> CallerCtx { CallerCtx { uid: self.euid, gid: self.egid, pid: self.pid, + groups: self.groups.clone(), } } } @@ -596,7 +625,7 @@ impl core::fmt::Debug for Kstack { #[derive(Clone, Debug, Default)] pub struct FdTbl { - pub lower_fdtbl: Vec>, + pub posix_fdtbl: Vec>, pub upper_fdtbl: Vec>, active_count: usize, } @@ -606,32 +635,19 @@ pub type LockedFdTbl = RwLock; impl FdTbl { pub fn new() -> Self { Self { - lower_fdtbl: Vec::new(), + posix_fdtbl: Vec::new(), upper_fdtbl: Vec::new(), active_count: 0, } } - pub fn resize(&mut self, which: usize, size: usize) -> Result<()> { - let (fdtbl, _) = self.select_fdtbl_mut(which); - if super::CONTEXT_MAX_FILES < size { - return Err(Error::new(EMFILE)); - } - if size < fdtbl.len() { - return Err(Error::new(EINVAL)); - } - - fdtbl.resize(size, None); - Ok(()) - } - fn strip_tags(index: usize) -> usize { index & !UPPER_FDTBL_TAG } fn select_fdtbl(&self, index: usize) -> (&Vec>, usize) { if index & UPPER_FDTBL_TAG == 0 { - (&self.lower_fdtbl, index) + (&self.posix_fdtbl, index) } else { (&self.upper_fdtbl, Self::strip_tags(index)) } @@ -639,7 +655,7 @@ impl FdTbl { fn select_fdtbl_mut(&mut self, index: usize) -> (&mut Vec>, usize) { if index & UPPER_FDTBL_TAG == 0 { - (&mut self.lower_fdtbl, index) + (&mut self.posix_fdtbl, index) } else { (&mut self.upper_fdtbl, Self::strip_tags(index)) } @@ -681,6 +697,67 @@ impl FdTbl { Ok(()) } + pub fn add_file_min(&mut self, file: FileDescriptor, min: usize) -> Option { + if self.active_count >= super::CONTEXT_MAX_FILES { + return None; + } + + let tag = min & UPPER_FDTBL_TAG; + + let (fdtbl, min) = self.select_fdtbl_mut(min); + + // Find the first empty slot in the posix_fdtbl starting from `min`. + if let Some((pos, slot)) = fdtbl + .iter_mut() + .enumerate() + .skip(min) + .find(|(_, slot)| slot.is_none()) + { + *slot = Some(file); + self.active_count += 1; + return Some(FileHandle::from(pos | tag)); + }; + + let len = fdtbl.len(); + + // If no empty slot was found, we need to allocate a new slot. + if len >= min { + fdtbl.push(Some(file)); + self.active_count += 1; + Some(FileHandle::from(len | tag)) + } else { + self.insert_file(FileHandle::from(min | tag), file) + } + } + + fn bulk_add_files_posix( + &mut self, + files_to_add: Vec, + ) -> Option> { + let count = files_to_add.len(); + if count == 0 { + return Some(Vec::new()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return None; + } + + let handles = self.find_free_posix_slots(count); + let max_index = handles[count - 1].get(); + if self.posix_fdtbl.len() <= max_index { + // Resize the posix_fdtbl to accommodate the new files. + self.posix_fdtbl.resize(max_index + 1, None); + } + + for (&handle, file) in handles.iter().zip(files_to_add) { + let index = handle.get(); + self.posix_fdtbl[index] = Some(file); + } + + self.active_count += count; + Some(handles) + } + fn insert_file(&mut self, i: FileHandle, file: FileDescriptor) -> Option { if self.active_count >= super::CONTEXT_MAX_FILES { return None; @@ -705,7 +782,31 @@ impl FdTbl { } } - fn bulk_insert_files( + fn bulk_insert_files_upper( + &mut self, + files_to_insert: Vec, + ) -> Option> { + let count = files_to_insert.len(); + if count == 0 { + return Some(Vec::new()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return None; + } + + let index = Self::strip_tags(self.find_free_upper_block(count).get()); + let mut handles = Vec::with_capacity(count); + for (i, file) in files_to_insert.into_iter().enumerate() { + let current_index = index + i; + self.upper_fdtbl[current_index] = Some(file); + handles.push(FileHandle::from(current_index | UPPER_FDTBL_TAG)); + } + + self.active_count += count; + Some(handles) + } + + fn bulk_insert_files_upper_manual( &mut self, files_to_insert: Vec, handles: &[FileHandle], @@ -722,9 +823,20 @@ impl FdTbl { } self.validate_free_slots(handles)?; - for (file, &handle) in files_to_insert.into_iter().zip(handles) { - self.insert_file(handle, file).ok_or(Error::new(EMFILE))?; + let max_index = handles + .iter() + .map(|h| Self::strip_tags(h.get())) + .max() + .unwrap_or(0); + if self.upper_fdtbl.len() <= max_index { + self.upper_fdtbl.resize_with(max_index + 1, || None); } + for (file, &handle) in files_to_insert.into_iter().zip(handles) { + let index = Self::strip_tags(handle.get()); + self.upper_fdtbl[index] = Some(file); + } + + self.active_count += count; Ok(()) } @@ -774,7 +886,7 @@ impl FdTbl { .ok_or(Error::new(EBADF)) } - pub fn remove_file(&mut self, i: FileHandle) -> Option { + fn remove_file(&mut self, i: FileHandle) -> Option { let index = i.get(); let (fdtbl, real_index) = self.select_fdtbl_mut(index); @@ -786,7 +898,7 @@ impl FdTbl { removed_file_opt } - pub fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result> { + fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result> { // Validate that all handles are valid before proceeding to avoid partial results. self.validate_handles(handles)?; @@ -798,6 +910,56 @@ impl FdTbl { Ok(files) } + fn find_free_posix_slots(&self, count: usize) -> Vec { + let mut free_slots = Vec::with_capacity(count); + + for (i, slot) in self.posix_fdtbl.iter().enumerate() { + if slot.is_none() { + free_slots.push(FileHandle::from(i)); + if free_slots.len() == count { + return free_slots; + } + } + } + + let mut current_len = self.posix_fdtbl.len(); + while free_slots.len() < count { + free_slots.push(FileHandle::from(current_len)); + current_len += 1; + } + free_slots + } + + fn find_free_upper_block(&mut self, len: usize) -> FileHandle { + let mut start = 0; + let mut count = 0; + + for (i, file_opt) in self.upper_fdtbl.iter().enumerate() { + if file_opt.is_none() { + if count == 0 { + start = i; + } + count += 1; + if count == len { + break; + } + } else { + count = 0; + } + } + + if count < len { + if count == 0 { + start = self.upper_fdtbl.len(); + } + let needed = len - count; + self.upper_fdtbl + .resize(self.upper_fdtbl.len() + needed, None); + } + + FileHandle::from(start | UPPER_FDTBL_TAG) + } + pub fn force_close_all(&mut self, token: &mut CleanLockToken) { for file_opt in self.iter_mut() { if let Some(file) = file_opt.take() { @@ -810,7 +972,7 @@ impl FdTbl { impl FdTbl { pub fn enumerate(&self) -> impl Iterator)> { - self.lower_fdtbl.iter().enumerate().chain( + self.posix_fdtbl.iter().enumerate().chain( self.upper_fdtbl .iter() .enumerate() @@ -819,19 +981,20 @@ impl FdTbl { } pub fn iter(&self) -> impl Iterator> { - self.lower_fdtbl.iter().chain(self.upper_fdtbl.iter()) + self.posix_fdtbl.iter().chain(self.upper_fdtbl.iter()) } pub fn iter_mut(&mut self) -> impl Iterator> { - self.lower_fdtbl + self.posix_fdtbl .iter_mut() .chain(self.upper_fdtbl.iter_mut()) } } -pub fn bulk_insert_fds( +pub fn bulk_add_fds( descriptions: Vec>, payload: UserSliceRw, + cloexec: bool, token: &mut LockToken, ) -> Result { let cnt = descriptions.len(); @@ -841,18 +1004,71 @@ pub fn bulk_insert_fds( if descriptions.is_empty() { return Ok(0); } - let files_iter = descriptions - .into_iter() - .map(|description| FileDescriptor { description }); let current_lock = context::current(); - let mut current = current_lock.read(token.token()); + let mut current = current_lock.write(token.token()); let (current, mut token) = current.token_split(); - let handles: Vec = payload - .usizes() - .map(|res| res.map(|i| FileHandle::from(i))) - .collect::>()?; - let files = files_iter.collect::>(); - current.bulk_insert_files(files, &handles, &mut token)?; + let files: Vec = descriptions + .into_iter() + .map(|description| FileDescriptor { + description, + cloexec, + }) + .collect(); + let handles = current + .bulk_add_files_posix(files, &mut token) + .ok_or(Error::new(EMFILE))?; + let payload_chunks = payload.in_exact_chunks(size_of::()); + for (handle, chunk) in handles.iter().zip(payload_chunks) { + chunk.copy_from_slice(&handle.get().to_ne_bytes())?; + } Ok(handles.len()) } + +pub fn bulk_insert_fds( + descriptions: Vec>, + payload: UserSliceRw, + cloexec: bool, + token: &mut LockToken, +) -> Result { + let cnt = descriptions.len(); + if payload.len() != cnt * size_of::() { + return Err(Error::new(EINVAL)); + } + if descriptions.is_empty() { + return Ok(0); + } + let files_iter = descriptions.into_iter().map(|description| FileDescriptor { + description, + cloexec, + }); + let first_fd = payload + .in_exact_chunks(size_of::()) + .next() + .ok_or(Error::new(EINVAL))? + .read_usize()?; + + let current_lock = context::current(); + let mut current = current_lock.write(token.token()); + let (current, mut token) = current.token_split(); + + if first_fd == usize::MAX { + let files = files_iter.collect::>(); + let handles = current + .bulk_insert_files_upper(files, &mut token) + .ok_or(Error::new(EMFILE))?; + let payload_chunks = payload.in_exact_chunks(size_of::()); + for (handle, chunk) in handles.iter().zip(payload_chunks) { + chunk.copy_from_slice(&handle.get().to_ne_bytes())?; + } + Ok(handles.len()) + } else { + let handles: Vec = payload + .usizes() + .map(|res| res.map(|i| FileHandle::from(i | syscall::UPPER_FDTBL_TAG))) + .collect::>()?; + let files = files_iter.collect::>(); + current.bulk_insert_files_upper_manual(files, &handles, &mut token)?; + Ok(handles.len()) + } +} diff --git a/src/context/file.rs b/src/context/file.rs index b370056d05..2d3790f147 100644 --- a/src/context/file.rs +++ b/src/context/file.rs @@ -66,6 +66,8 @@ impl InternalFlags { pub struct FileDescriptor { /// Corresponding file description pub description: Arc, + /// Cloexec flag + pub cloexec: bool, } impl FileDescription { diff --git a/src/context/memory.rs b/src/context/memory.rs index 42be816c06..93446ba7a7 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -1,19 +1,14 @@ -use alloc::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - vec::Vec, -}; +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use arrayvec::ArrayVec; use core::{ cmp, fmt::Debug, mem::ManuallyDrop, num::NonZeroUsize, - ops::{Bound, Deref}, + ops::Bound, sync::atomic::{AtomicU32, Ordering}, }; use rmm::{Arch as _, PageFlush}; -use smallvec::SmallVec; use syscall::{error::*, flag::MapFlags, GrantFlags, MunmapFlags}; use crate::{ @@ -59,8 +54,6 @@ pub struct UnmapResult { pub size: usize, pub flags: MunmapFlags, } -pub type UnmapVec = SmallVec<[UnmapResult; 16]>; - impl UnmapResult { pub fn unmap(mut self, token: &mut CleanLockToken) -> Result<()> { let Some(GrantFileRef { @@ -71,9 +64,6 @@ impl UnmapResult { return Ok(()); }; - // TODO: This is not ideal, the lock must be held until try_close(), however that would break borrowing rules. - // Proper unmap operation would be a recursive operation, since closing a file can trigger another unmap(). - // We should refactor Result of munmap() to handle unmap and closing files recursively. let (scheme_id, number) = { let desc = description.write(token.token()); (desc.scheme, desc.number) @@ -322,7 +312,8 @@ impl AddrSpaceWrapper { requested_span: PageSpan, unpin: bool, token: &mut CleanLockToken, - ) -> Result { + ) -> Result> { + let mut token = token.token(); let mut guard = self.acquire_write(token.downgrade()); let guard = &mut *guard; @@ -342,7 +333,7 @@ impl AddrSpaceWrapper { requested_dst_base: Option, new_page_count: usize, new_flags: MapFlags, - mut notify_files_out: Option<&mut UnmapVec>, + mut notify_files_out: Option<&mut Vec>, token: LockToken, ) -> Result { let dst_lock = self; @@ -419,7 +410,7 @@ impl AddrSpaceWrapper { if new_page_count < src_span.count { let unpin = false; - let notify_files = AddrSpace::munmap_inner( + let notify_files: Vec = AddrSpace::munmap_inner( src_grants, src_mapper, src_flusher, @@ -599,8 +590,8 @@ impl AddrSpace { this_flusher: &mut Flusher, mut requested_span: PageSpan, unpin: bool, - ) -> Result { - let mut notify_files = UnmapVec::new(); + ) -> Result> { + let mut notify_files = Vec::new(); let next = |grants: &mut UserGrants, span: PageSpan| { grants @@ -671,9 +662,7 @@ impl AddrSpace { } // Remove irrelevant region - // TODO: Lock ordering violation - let mut token = unsafe { CleanLockToken::new() }; - let unmap_result = grant.unmap(this_mapper, this_flusher, &mut token); + let unmap_result = grant.unmap(this_mapper, this_flusher); // Notify scheme that holds grant if unmap_result.file_desc.is_some() { @@ -698,7 +687,7 @@ impl AddrSpace { requested_base_opt: Option, page_count: NonZeroUsize, flags: MapFlags, - notify_files_out: Option<&mut UnmapVec>, + notify_files_out: Option<&mut Vec>, map: impl FnOnce(Page, PageFlags, &mut PageMapper, &mut Flusher) -> Result, ) -> Result { assert_eq!(dst_lock.inner.as_mut_ptr(), self as *mut Self); @@ -772,40 +761,21 @@ impl AddrSpace { // longer arc-rwlock wrapped, it cannot be referenced `External`ly by borrowing grants, // so it should suffice to iterate over PageInfos and decrement and maybe deallocate // the underlying pages (and send some funmaps). - let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher, token) }; + let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher) }; let _ = res.unmap(token); } } } -pub struct AddrSpaceSwitchReadGuard { - pub lock: RwLockReadGuard<'static, L5, AddrSpace>, -} - -impl AddrSpaceSwitchReadGuard { - pub fn new(guard: RwLockReadGuard<'_, L5, AddrSpace>) -> Self { - Self { - lock: unsafe { core::mem::transmute(guard) }, - } - } -} -impl Deref for AddrSpaceSwitchReadGuard { - type Target = RwLockReadGuard<'static, L5, AddrSpace>; - - fn deref(&self) -> &Self::Target { - &self.lock - } -} - #[derive(Debug)] pub struct UserGrants { // Using a BTreeMap for its range method. inner: BTreeMap, - // Holes ordered by memory address for merging adjacent holes - holes_by_addr: BTreeMap, - // Holes ordered by size then start address for fast allocations - holes_by_size: BTreeSet<(usize, VirtualAddress)>, + // Using a BTreeMap for its range method. + holes: BTreeMap, + // TODO: Would an additional map ordered by (size,start) to allow for O(log n) allocations be + // beneficial? } #[derive(Clone, Copy)] @@ -907,41 +877,10 @@ impl Debug for PageSpan { impl UserGrants { pub fn new() -> Self { - let mut holes_by_addr = BTreeMap::new(); - let mut holes_by_size = BTreeSet::new(); - - let initial_offset = VirtualAddress::new(0); - let initial_size = crate::USER_END_OFFSET; - - holes_by_addr.insert(initial_offset, initial_size); - holes_by_size.insert((initial_size, initial_offset)); - Self { inner: BTreeMap::new(), - holes_by_addr, - holes_by_size, - } - } - /// Internal helper to keep the two hole maps in sync - fn insert_hole(&mut self, offset: VirtualAddress, size: usize) { - self.holes_by_addr.insert(offset, size); - self.holes_by_size.insert((size, offset)); - } - /// Internal helper to keep the two hole maps in sync - fn remove_hole(&mut self, offset: &VirtualAddress) -> Option { - if let Some(size) = self.holes_by_addr.remove(offset) { - self.holes_by_size.remove(&(size, *offset)); - Some(size) - } else { - None - } - } - /// Internal helper to keep the two hole maps in sync - fn resize_hole(&mut self, offset: &VirtualAddress, new_size: usize) { - if let Some(size) = self.holes_by_addr.get_mut(offset) { - self.holes_by_size.remove(&(*size, *offset)); - *size = new_size; - self.holes_by_size.insert((new_size, *offset)); + holes: core::iter::once((VirtualAddress::new(0), crate::USER_END_OFFSET)) + .collect::>(), } } @@ -1004,16 +943,18 @@ impl UserGrants { // TODO: Allow explicitly allocating guard pages? Perhaps using mprotect or mmap with // PROT_NONE? - let req_size = page_count * PAGE_SIZE; - let (_, hole_start) = self - .holes_by_size - .range((req_size, VirtualAddress::new(0))..) - .find(|&&(hole_size, hole_offset)| { - // A hole might be large enough, but the usable - // portion above `min` address might be too small. - let usable_start = cmp::max(hole_offset.data(), min); - let hole_end = hole_offset.data() + hole_size; - usable_start + req_size <= hole_end + let (hole_start, _hole_size) = self + .holes + .iter() + .skip_while(|(hole_offset, hole_size)| hole_offset.data() + **hole_size <= min) + .find(|(hole_offset, hole_size)| { + let avail_size = + if hole_offset.data() <= min && min <= hole_offset.data() + **hole_size { + **hole_size - (min - hole_offset.data()) + } else { + **hole_size + }; + page_count * PAGE_SIZE <= avail_size })?; // Create new region Some(PageSpan::new( @@ -1029,14 +970,10 @@ impl UserGrants { let size = page_count * PAGE_SIZE; let end_address = base.start_address().add(size); - let previous_hole = self - .holes_by_addr - .range(..start_address) - .next_back() - .map(|(&k, &v)| (k, v)); + let previous_hole = self.holes.range_mut(..start_address).next_back(); if let Some((hole_offset, hole_size)) = previous_hole { - let prev_hole_end = hole_offset.data() + hole_size; + let prev_hole_end = hole_offset.data() + *hole_size; // Note that prev_hole_end cannot exactly equal start_address, since that would imply // there is another grant at that position already, as it would otherwise have been @@ -1046,49 +983,46 @@ impl UserGrants { // hole_offset must be below (but never equal to) the start address due to the // `..start_address()` limit; hence, all we have to do is to shrink the // previous offset. - self.resize_hole(&hole_offset, start_address.data() - hole_offset.data()); + *hole_size = start_address.data() - hole_offset.data(); } if prev_hole_end > end_address.data() { // The grant is splitting this hole in two, so insert the new one at the end. - self.insert_hole(end_address, prev_hole_end - end_address.data()); + self.holes + .insert(end_address, prev_hole_end - end_address.data()); } } // Next hole - if let Some(hole_size) = self.remove_hole(&start_address) { + if let Some(hole_size) = self.holes.remove(&start_address) { let remainder = hole_size - size; if remainder > 0 { - self.insert_hole(end_address, remainder); + self.holes.insert(end_address, remainder); } } } - - fn unreserve(&mut self, base: Page, page_count: usize) { + fn unreserve(holes: &mut BTreeMap, base: Page, page_count: usize) { + // TODO let start_address = base.start_address(); let size = page_count * PAGE_SIZE; let end_address = base.start_address().add(size); // The size of any possible hole directly after the to-be-freed region. - let exactly_after_size = self.remove_hole(&end_address); + let exactly_after_size = holes.remove(&end_address); + // There was a range that began exactly prior to the to-be-freed region, so simply // increment the size such that it occupies the grant too. If in addition there was a grant // directly after the grant, include it too in the size. - if let Some((hole_offset, _hole_size)) = self - .holes_by_addr - .range(..start_address) + if let Some((hole_offset, hole_size)) = holes + .range_mut(..start_address) .next_back() .filter(|(offset, size)| offset.data() + **size == start_address.data()) - .map(|(&offset, &size)| (offset, size)) { - self.resize_hole( - &hole_offset, - end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0), - ); + *hole_size = end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0); } else { // There was no free region directly before the to-be-freed region, however will // now unconditionally insert a new free region where the grant was, and add that extra // size if there was something after it. - self.insert_hole(start_address, size + exactly_after_size.unwrap_or(0)); + holes.insert(start_address, size + exactly_after_size.unwrap_or(0)); } } pub fn insert(&mut self, mut grant: Grant) { @@ -1140,7 +1074,7 @@ impl UserGrants { if (base..base.next_by(info.page_count())).contains(&page) { let (base, info) = cursor.remove_prev().unwrap(); - self.unreserve(base, info.page_count()); + Self::unreserve(&mut self.holes, base, info.page_count()); Some(Grant { base, info }) } else { None @@ -1469,15 +1403,15 @@ impl Grant { for dst_page in span.pages() { let src_page = src.src_base.next_by(dst_page.offset_from(span.base)); - let (frame, page_flags, is_cow) = match src.mode { + let (frame, is_cow) = match src.mode { MmapMode::Shared => { // TODO: Error code for "scheme responded with unmapped page"? - let (frame, page_flags) = match src_addrspace + let frame = match src_addrspace .table .utable .translate(src_page.start_address()) { - Some((phys, page_flags)) => (Frame::containing(phys), page_flags), + Some((phys, _)) => Frame::containing(phys), // TODO: ensure the correct context is hardblocked, if necessary None => { let (frame, _, new_guard) = correct_inner( @@ -1488,26 +1422,20 @@ impl Grant { 0, ) .map_err(|_| Error::new(EIO))?; - let page_flags = new_guard - .table - .utable - .translate(src_page.start_address()) - .unwrap() - .1; guard = new_guard; - (frame, page_flags) + frame } }; - (frame, page_flags, false) + (frame, false) } MmapMode::Cow => unsafe { - let (frame, page_flags) = match guard + let frame = match guard .table .utable .remap_with(src_page.start_address(), |flags| flags.write(false)) { - Some((page_flags, phys, _)) => (Frame::containing(phys), page_flags), + Some((_, phys, _)) => Frame::containing(phys), // TODO: ensure the correct context is hardblocked, if necessary None => { let (frame, _, new_guard) = correct_inner( @@ -1518,19 +1446,12 @@ impl Grant { 0, ) .map_err(|_| Error::new(EIO))?; - // FIXME correct_inner should read the page flags instead - let page_flags = new_guard - .table - .utable - .translate(src_page.start_address()) - .unwrap() - .1; guard = new_guard; - (frame, page_flags) + frame } }; - (frame, page_flags, true) + (frame, true) }, }; src_addrspace = &mut *guard; @@ -1590,14 +1511,7 @@ impl Grant { .map_phys( dst_page.start_address(), frame.base(), - new_flags - .write(new_flags.has_write() && !is_cow) - // FIXME make sure this stays in sync with the MemoryType flags - .uncacheable(page_flags.has_flag(RmmA::ENTRY_FLAG_UNCACHEABLE)) - .device_memory(page_flags.has_flag(RmmA::ENTRY_FLAG_DEVICE_MEMORY)) - .write_combining( - page_flags.has_flag(RmmA::ENTRY_FLAG_WRITE_COMBINING), - ), + new_flags.write(new_flags.has_write() && !is_cow), ) .unwrap(); flush.ignore(); @@ -1991,7 +1905,6 @@ impl Grant { mut self, mapper: &mut PageMapper, flusher: &mut impl GenericFlusher, - token: &mut CleanLockToken, ) -> UnmapResult { assert!(self.info.mapped); assert!(!self.info.is_pinned()); @@ -2002,6 +1915,9 @@ impl Grant { .. } = self.info.provider { + // TODO: Lock ordering violation + let mut token = unsafe { CleanLockToken::new() }; + let mut token = token.token(); let mut guard = address_space.acquire_write(token.downgrade()); for (_, grant) in guard @@ -2843,7 +2759,7 @@ pub struct BorrowedFmapSource<'a> { pub addr_space_guard: RwLockWriteGuard<'a, L5, AddrSpace>, } -pub fn handle_notify_files(notify_files: UnmapVec, token: &mut CleanLockToken) { +pub fn handle_notify_files(notify_files: Vec, token: &mut CleanLockToken) { for file in notify_files { let _ = file.unmap(token); } diff --git a/src/context/mod.rs b/src/context/mod.rs index 2ec4ded54a..37c73f5a37 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -3,10 +3,10 @@ //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching) use alloc::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeSet, VecDeque}, sync::{Arc, Weak}, }; -use core::{cmp::Reverse, num::NonZeroUsize, ops::Deref}; +use core::{num::NonZeroUsize, ops::Deref}; use crate::{ context::memory::AddrSpaceWrapper, @@ -81,29 +81,16 @@ static RUN_CONTEXTS: Mutex = Mutex::new(RunContextData::new( static IDLE_CONTEXTS: Mutex> = Mutex::new(VecDeque::new()); pub struct RunContextData { - // queue: VecDeque, - queue: BTreeMap<(u64, Reverse, u32), (u64, u64, WeakContextRef)>, // ((vd, rem_slice, ctxt_id), (vtime, weight, context)) - count: usize, - v: u64, - total_weight: u64, - min_vtime: u64, + set: [VecDeque; 40], } impl RunContextData { pub const fn new() -> Self { const EMPTY_VEC: VecDeque = VecDeque::new(); Self { - queue: BTreeMap::new(), - count: 0, - v: 0, - total_weight: 0, - min_vtime: 0, + set: [EMPTY_VEC; 40], } } - pub fn update_count(&mut self) -> usize { - self.count = self.queue.len(); - self.count - } } /// Get the global schemes list, const @@ -139,13 +126,6 @@ pub fn init(token: &mut CleanLockToken) { context.name.clear(); context.name.push_str("[kmain]"); - #[cfg(feature = "profiling")] - { - crate::profiling::DBG_ID_MAP - .write(token.token()) - .insert(context.debug_id, context.name); - } - self::arch::EMPTY_CR3.call_once(|| RmmA::table(TableKind::User)); context.status = Status::Runnable; @@ -251,7 +231,6 @@ pub fn spawn( context.kstack = Some(stack); context.userspace = userspace_allowed; - context.queue_key = Some((context.vd, Reverse(context.rem_slice), context.debug_id)); let context_lock = Arc::new(ContextLock::new(context)); let context_ref = ContextRef(Arc::clone(&context_lock)); @@ -343,11 +322,3 @@ impl Drop for PreemptGuardL2<'_> { self.context.write(self.token.token()).preempt_locks -= 1; } } - -pub fn get_contexts_stats(token: &mut CleanLockToken) -> (usize, usize, usize) { - let alive = contexts(token.downgrade()).len(); - let running = run_contexts(token.token()).count; - let blocked = idle_contexts(token.downgrade()).len(); - - (alive, running, blocked) -} diff --git a/src/context/signal.rs b/src/context/signal.rs index 4e99215bfc..19f4ebc01f 100644 --- a/src/context/signal.rs +++ b/src/context/signal.rs @@ -4,12 +4,13 @@ use crate::{context, sync::CleanLockToken, syscall::flag::SigcontrolFlags}; pub fn signal_handler(token: &mut CleanLockToken) { let context_lock = context::current(); - let context = context_lock.upgradeable_read(token.token()); + let mut context_guard = context_lock.write(token.token()); + let context = &mut *context_guard; let being_sigkilled = context.being_sigkilled; if being_sigkilled { - drop(context); + drop(context_guard); drop(context_lock); crate::syscall::process::exit_this_context(None, token); } @@ -47,7 +48,6 @@ pub fn signal_handler(token: &mut CleanLockToken) { let sigh_instr_ptr = st.user_handler.get(); - let mut context = context.upgrade(); let Some(regs) = context.regs_mut() else { // TODO: is this even reachable? trace!("No registers, returning"); @@ -59,7 +59,6 @@ pub fn signal_handler(token: &mut CleanLockToken) { regs.set_instr_pointer(sigh_instr_ptr); - let context = context.downgrade(); let (thread_ctl, _, _) = context .sigcontrol() .expect("cannot have been unset while holding the lock"); diff --git a/src/context/switch.rs b/src/context/switch.rs index b94a02b713..86684c8f4c 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -4,24 +4,20 @@ use crate::{ context::{ - self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard, - run_contexts, ArcContextLockWriteGuard, Context, ContextLock, WeakContextRef, + self, arch, idle_contexts, idle_contexts_try, run_contexts, ArcContextLockWriteGuard, + Context, ContextLock, WeakContextRef, }, cpu_set::LogicalCpuId, cpu_stats::{self, CpuState}, percpu::PercpuBlock, sync::{ArcRwLockWriteGuard, CleanLockToken, L4}, }; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use core::{ cell::{Cell, RefCell}, - cmp::Reverse, - hint, matches, mem, - option::Option::{None, Some}, + hint, mem, sync::atomic::Ordering, - u64, }; -use smallvec::SmallVec; use syscall::PtraceFlags; enum UpdateResult { @@ -30,18 +26,13 @@ enum UpdateResult { Blocked, } -// A simple geometric series where value[i] ~= value[i + 1] * 1.25 +// A simple geometric series where value[i] ~= value[i - 1] * 1.25 const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [ 88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904, 3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, ]; -const SCALE: u128 = 1 << 40; -const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms -const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms -const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms - /// Determines if a given context is eligible to be scheduled on a given CPU (in /// principle, the current CPU). /// @@ -105,9 +96,7 @@ pub fn tick(token: &mut CleanLockToken) { ticks_cell.set(new_ticks); // Trigger a context switch after every 3 ticks (approx. 6.75 ms). - if new_ticks >= TICK_INTERVAL as usize - && arch::CONTEXT_SWITCH_LOCK.load(Ordering::Relaxed) == false - { + if new_ticks >= 3 { switch(token); crate::context::signal::signal_handler(token); } @@ -187,68 +176,14 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { // Alarm (previously in update_runnable) let wakeups = wakeup_contexts(token, switch_time); - let wakeups_len = wakeups.len(); - let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new(); - if wakeups_len > 0 { + if wakeups.len() > 0 { let mut run_contexts = run_contexts(token.token()); - - for context_ref in wakeups { - let Some(context_lock) = context_ref.upgrade() else { - continue; - }; - - let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else { - push_idle.push(context_ref); - continue; - }; - - let new_vtime = guard.vtime.max(run_contexts.v); - guard.vtime = new_vtime; - - let weight = SCHED_PRIO_TO_WEIGHT[guard.prio] as u64; - let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128; - - if !guard.is_active { - guard.is_active = true; - run_contexts.total_weight += weight; - } - - guard.vd = new_vtime + scaled_slice as u64; - guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64; - let key = (guard.vd, Reverse(guard.rem_slice), guard.debug_id); - guard.queue_key = Some(key); - drop(guard); - - run_contexts - .queue - .insert(key, (new_vtime, weight, context_ref)); + for (prio, context_lock) in wakeups { + run_contexts.set[prio].push_back(context_lock); } } - { - let mut idle_list = idle_contexts(token.downgrade()); - for context_ref in push_idle { - idle_list.push_back(context_ref); - } - } - - /* // uncomment to debug contexts count - let cpu_count = crate::cpu_count() as usize; - let len_idle = idle_contexts(token.downgrade()).len(); - let all_contexts = context::contexts(token.downgrade()) - .len() - .saturating_sub(cpu_count); // ignore kmain - print!( - "\r TIME {}.{} IDLE {} WAKEUPS {} ALL {} ", - switch_time / 1000_000_000, - (switch_time / 100_000_000) % 10, - len_idle, - wakeups_len, - all_contexts - ); - */ - let cpu_id = crate::cpu_id(); // Update per-cpu times @@ -257,19 +192,21 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { let was_idle = percpu.stats.add_time(percpu_ms) == CpuState::Idle as u8; percpu.switch_internals.switch_time.set(switch_time); - let switch_context_opt = select_next_context( + let switch_context_opt = match select_next_context( token, percpu, cpu_id, switch_time, - percpu_nanos, was_idle, &mut prev_context_guard, - ); + ) { + Ok(opt) => opt, + Err(early_ret) => return early_ret, + }; // Switch process states, TSS stack pointer, and store new context ID match switch_context_opt { - Some((mut next_context_guard, addr_space_guard)) => { + Some(mut next_context_guard) => { // Update context states and prepare for the switch. let prev_context = &mut *prev_context_guard; let next_context = &mut *next_context_guard; @@ -329,14 +266,6 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { prev_context.inside_syscall = percpu.inside_syscall.replace(next_context.inside_syscall); - #[cfg(feature = "profiling")] - { - percpu - .switch_internals - .current_dbg_id - .store(next_context.debug_id, Ordering::Relaxed); - } - #[cfg(feature = "syscall_debug")] { prev_context.syscall_debug_info = percpu @@ -351,11 +280,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { .being_sigkilled .set(next_context.being_sigkilled); - // Anything implement Drop must be manually dropped now - drop(prev_context_lock); - unsafe { - percpu.new_addrsp_guard.set(addr_space_guard); arch::switch_to(prev_context, next_context); } @@ -377,12 +302,9 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { } } -fn wakeup_contexts( - token: &mut CleanLockToken, - switch_time: u128, -) -> SmallVec<[WeakContextRef; 16]> { +fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize, WeakContextRef)> { // TODO: Optimise this somehow. Perhaps using a separate timer queue? - let mut wakeups = SmallVec::new(); + let mut wakeups = Vec::new(); let current_context = context::current(); let Some(idle_contexts) = idle_contexts_try(token.downgrade()) else { // other cpus may spawning or killing contexts so let's skip wakeups to avoid contention @@ -408,19 +330,18 @@ fn wakeup_contexts( if guard.status.is_soft_blocked() { if let Some(wake) = guard.wake { if switch_time >= wake { + let prio = guard.prio; drop(guard); - wakeups.push(context_ref); + wakeups.push((prio, context_ref)); continue; } } - } else if guard.status.is_dead() { - // TODO: who hold this dead context? - continue; } if guard.status.is_runnable() && !guard.running { + let prio = guard.prio; drop(guard); - wakeups.push(context_ref); + wakeups.push((prio, context_ref)); continue; } @@ -430,243 +351,137 @@ fn wakeup_contexts( wakeups } -/// This is the scheduler function which currently utilises EEVDF Scheduler +/// This is the scheduler function which currently utilises Deficit Weighted Round Robin Scheduler fn select_next_context( token: &mut CleanLockToken, percpu: &PercpuBlock, cpu_id: LogicalCpuId, switch_time: u128, - elapsed_time: u64, was_idle: bool, prev_context_guard: &mut ArcRwLockWriteGuard, -) -> Option<(ArcContextLockWriteGuard, Option)> { +) -> Result, SwitchResult> { let contexts_data = run_contexts(token.token()); let (mut contexts_data, mut token) = contexts_data.into_split(); + let contexts_list = &mut contexts_data.set; let idle_context = percpu.switch_internals.idle_context(); + let mut balance = percpu.balance.get(); + let mut i = percpu.last_queue.get() % 40; // Lock the previous context. let prev_context_lock = crate::context::current(); - let is_idle = Arc::ptr_eq(&prev_context_lock, &idle_context); - let prev_runnable = !is_idle && prev_context_guard.status.is_runnable(); - let elapsed_ticks = elapsed_time as u128 * SCALE / NANOS_PER_TICK; + let mut empty_queues = 0; + let mut total_iters = 0; + let mut next_context_guard_opt = None; - if prev_runnable { - let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64; - prev_context_guard.rem_slice = prev_context_guard - .rem_slice - .saturating_sub((elapsed_ticks) as u64); - let scaled_task = elapsed_ticks / weight as u128; - prev_context_guard.vtime += scaled_task as u64; + let total_contexts: usize = contexts_list.iter().map(|q| q.len()).sum(); + let mut skipped_contexts = 0; - if prev_context_guard.vtime < contexts_data.v { - prev_context_guard.vtime = contexts_data.v; + 'priority: loop { + i = (i + 1) % 40; + total_iters += 1; + + // The least prioritised queue takes <5000 iters to build up + // balance = sched_prio_to_weight[20], if we have already spent + // that many iters and not found any context, it is better to just + // skip for now + if total_iters >= 5000 { + break 'priority; } - let is_yield = (elapsed_time as u128) < (TICK_INTERVAL as u128 * NANOS_PER_TICK) / 2; - - if is_yield { - let unconsumed = prev_context_guard.rem_slice as u128; - let penalty = unconsumed / weight as u128; - prev_context_guard.vtime += penalty as u64; - prev_context_guard.rem_slice = 0; + if skipped_contexts > total_contexts && total_contexts > 0 { + break 'priority; } - if prev_context_guard.rem_slice == 0 { - prev_context_guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64; - let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128; - prev_context_guard.vd = prev_context_guard.vtime + scaled_slice as u64; - } - } else if !is_idle { - if prev_context_guard.is_active { - prev_context_guard.is_active = false; - let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64; - contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight); - } - prev_context_guard.rem_slice = 0; - } + let contexts = contexts_list + .get_mut(i) + .expect("i should be between [0, 39]!"); - let mut eligible_best = None; - let mut prev_is_eligible = false; - - let mut ineligible_best = None; - let mut ineligible_min_vtime = u64::MAX; - let mut ineligible_vd = u64::MAX; - - if prev_runnable { - if prev_context_guard.vtime <= contexts_data.v { - prev_is_eligible = true; - } else { - ineligible_min_vtime = prev_context_guard.vtime; - ineligible_vd = prev_context_guard.vd; - } - } - - // New BTreeMap based walk - let mut weight_change: u64 = 0; - let mut contexts_to_remove: SmallVec<[(u64, Reverse, u32); 16]> = SmallVec::new(); - for ((vd, rem_slice, ctxt_id), (vtime, context_weight, context_ref)) in - contexts_data.queue.iter() - { - if *vtime > ineligible_min_vtime && *vtime > contexts_data.v { - continue; - } - - let Some(context_lock) = context_ref.upgrade() else { - weight_change += *context_weight as u64; - contexts_to_remove.push((*vd, *rem_slice, *ctxt_id)); - continue; - }; - - if Arc::ptr_eq(&context_lock, &idle_context) - || Arc::ptr_eq(&context_lock, &prev_context_lock) - { - //weight_change += *context_weight as u64; - //contexts_to_remove.push((*vd, *rem_slice, *ctxt_id)); - continue; - } - - let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else { - continue; - }; - - let sw = unsafe { update_runnable(&mut guard, cpu_id, switch_time) }; - - if matches!(sw, UpdateResult::Blocked) { - if guard.is_active { - guard.is_active = false; - weight_change += context_weight; + if contexts.is_empty() { + empty_queues += 1; + if empty_queues >= 40 { + // If all queues are empty, just break out + break 'priority; } - guard.rem_slice = 0; - guard.queue_key = None; + continue; + } else { + empty_queues = 0; + } - contexts_to_remove.push((*vd, *rem_slice, *ctxt_id)); - drop(guard); - // Reenqueue should be handled by unblock - idle_contexts(token.token()).push_back(context_ref.clone()); + if balance[i] < SCHED_PRIO_TO_WEIGHT[20] { + // This queue does not have enough balance to run, + // increment the balance! + balance[i] += SCHED_PRIO_TO_WEIGHT[i]; continue; } - if !matches!(sw, UpdateResult::CanSwitch) { - continue; - } + let len = contexts.len(); + for _ in 0..len { + let (next_context_ref, next_context_lock) = match contexts.pop_front() { + Some(lock) => match lock.upgrade() { + Some(new_lock) => (lock, new_lock), + None => { + skipped_contexts += 1; + continue; // Ghost Process, just continue + } + }, + None => break, // Empty Queue + }; - let mut best_addr_space = None; - if let Some(addr_space) = &guard.addr_space { - let mut t = unsafe { CleanLockToken::new() }; - if let Some(addr) = addr_space.inner.try_read(t.token()) { - best_addr_space = Some(AddrSpaceSwitchReadGuard::new(addr)); - } else { + if Arc::ptr_eq(&next_context_lock, &prev_context_lock) { + contexts.push_back(next_context_ref); continue; } - } - - if *vtime <= contexts_data.v { - // Eligible - eligible_best = Some((guard, best_addr_space)); - break; - } else { - // Ineligible - if *vtime < ineligible_min_vtime { - ineligible_min_vtime = *vtime; - ineligible_vd = *vd; - if let Some((old_guard, old_addr_space)) = ineligible_best { - drop(old_guard); - drop(old_addr_space); - } - ineligible_best = Some((guard, best_addr_space)); + if Arc::ptr_eq(&next_context_lock, &idle_context) { + contexts.push_back(next_context_ref); + continue; } - } - } + let mut next_context_guard = unsafe { next_context_lock.write_arc() }; - contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight_change); - - for old_key in contexts_to_remove { - contexts_data.queue.remove(&old_key); - } - - // No eligible context was found - if !(prev_is_eligible || eligible_best.is_some()) && ineligible_min_vtime != u64::MAX { - contexts_data.v = ineligible_min_vtime; // Advance V - - let prev_is_earliest = prev_runnable && prev_context_guard.vtime <= ineligible_min_vtime; - - if prev_is_earliest { - eligible_best = None; - } else if ineligible_best.is_some() { - let prev_has_slice = prev_runnable && prev_context_guard.rem_slice > 0; - - if prev_has_slice && prev_context_guard.vd <= ineligible_vd { - eligible_best = None; + // Is this context runnable on this CPU? + let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) }; + if let UpdateResult::CanSwitch = sw { + next_context_guard_opt = Some(next_context_guard); + balance[i] -= SCHED_PRIO_TO_WEIGHT[20]; + break 'priority; } else { - eligible_best = ineligible_best.take(); - } - } - } else if prev_is_eligible && eligible_best.is_some() { - if let Some((ref guard, _)) = eligible_best { - if prev_context_guard.vd < guard.vd - || (prev_context_guard.vd == guard.vd - && prev_context_guard.rem_slice > guard.rem_slice) - { - eligible_best = None; + if matches!(sw, UpdateResult::Blocked) { + idle_contexts(token.token()).push_back(next_context_ref); + } else { + contexts.push_back(next_context_ref); + }; + skipped_contexts += 1; + + if skipped_contexts >= total_contexts { + break 'priority; + } } } } + percpu.balance.set(balance); + percpu.last_queue.set(i); - let mut final_winner = None; - - if let Some((mut chosen_guard, addr_space)) = eligible_best { - if let Some(key) = chosen_guard.queue_key.take() { - contexts_data.queue.remove(&key); - } - final_winner = Some((chosen_guard, addr_space)); - } - - if final_winner.is_some() || prev_runnable { - if contexts_data.total_weight > 0 { - let v_advance = elapsed_ticks as u128 / contexts_data.total_weight as u128; - contexts_data.v += v_advance as u64; - } - - if let Some((chosen_guard, addr_space)) = final_winner { - if prev_runnable { - let (vd, rem_slice, ctxt_id, vtime) = ( - prev_context_guard.vd, - prev_context_guard.rem_slice, - prev_context_guard.debug_id, - prev_context_guard.vtime, - ); - prev_context_guard.queue_key = Some((vd, Reverse(rem_slice), ctxt_id)); - - let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64; - contexts_data.queue.insert( - (vd, Reverse(rem_slice), ctxt_id), - ( - vtime, - weight, - WeakContextRef(Arc::downgrade(&prev_context_lock)), - ), - ); - } else if !is_idle { - idle_contexts(token.token()) - .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); - } - - return Some((chosen_guard, addr_space)); + if !Arc::ptr_eq(&prev_context_lock, &idle_context) { + // Send the old process to the back of the line (if it is still runnable) + let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock)); + if prev_context_guard.status.is_runnable() { + let prio = prev_context_guard.prio; + contexts_list[prio].push_back(prev_ctx); } else { - return None; + idle_contexts(token.token()).push_back(prev_ctx); } + } + + if let Some(next_context_guard) = next_context_guard_opt { + // We found a new process! + return Ok(Some(next_context_guard)); } else { - if !is_idle { - idle_contexts(token.token()) - .push_back(WeakContextRef(Arc::downgrade(&prev_context_lock))); - } - - let prev_is_dead = !is_idle && !prev_context_guard.status.is_runnable(); - if (!was_idle || prev_is_dead) && !is_idle { - return Some(unsafe { (idle_context.write_arc(), None) }); + if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) { + // We switch into the idle context + Ok(Some(unsafe { idle_context.write_arc() })) } else { - return None; + // We found no other process to run. + Ok(None) } } } @@ -682,10 +497,6 @@ pub struct ContextSwitchPercpu { current_ctxt: RefCell>>, - // TODO: just access current_ctxt directly? - #[cfg(feature = "profiling")] - pub(crate) current_dbg_id: core::sync::atomic::AtomicU32, - /// The idle process. idle_ctxt: RefCell>>, pub(crate) being_sigkilled: Cell, @@ -700,9 +511,6 @@ impl ContextSwitchPercpu { current_ctxt: RefCell::new(None), idle_ctxt: RefCell::new(None), being_sigkilled: Cell::new(false), - - #[cfg(feature = "profiling")] - current_dbg_id: core::sync::atomic::AtomicU32::new(!0), } } diff --git a/src/context/timeout.rs b/src/context/timeout.rs index 1387f42e7f..9bf5c99095 100644 --- a/src/context/timeout.rs +++ b/src/context/timeout.rs @@ -28,10 +28,6 @@ fn registry(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, Registry> { REGISTRY.lock(token) } -pub fn get_timeout_stat(token: &mut CleanLockToken) -> usize { - REGISTRY.lock(token.token()).len() -} - pub fn register( scheme_id: SchemeId, event_id: usize, @@ -44,7 +40,7 @@ pub fn register( scheme_id, event_id, clock, - time: time.to_nanos(), + time: (time.tv_sec as u128 * time::NANOS_PER_SEC) + (time.tv_nsec as u128), }); } @@ -72,7 +68,7 @@ pub fn trigger(token: &mut CleanLockToken) { }; if trigger { - registry.swap_remove_back(i).unwrap() + registry.remove(i).unwrap() } else { i += 1; continue; diff --git a/src/event.rs b/src/event.rs index 14d38a7ed0..7398145ad6 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1,18 +1,14 @@ use alloc::sync::Arc; -use core::{ - hash::{Hash, Hasher}, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicUsize, Ordering}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use smallvec::SmallVec; -use syscall::{data::GlobalSchemes, EINTR}; +use syscall::data::GlobalSchemes; use crate::{ context, scheme::{self, SchemeExt, SchemeId}, sync::{ - CleanLockToken, LockToken, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0, - L1, L2, + CleanLockToken, LockToken, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0, L1, L2, }, syscall::{ data::Event, @@ -27,7 +23,6 @@ int_like!(EventQueueId, AtomicEventQueueId, usize, AtomicUsize); pub struct EventQueue { id: EventQueueId, queue: WaitQueue, - timeout_opt: Mutex>, } impl EventQueue { @@ -35,7 +30,6 @@ impl EventQueue { EventQueue { id, queue: WaitQueue::new(), - timeout_opt: Mutex::new(None), } } @@ -44,49 +38,12 @@ impl EventQueue { } pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result { - if block { - // timeout is one-time hit - let timeout = self.timeout_opt.lock(token.token()).take(); - if let Some(timeout) = timeout { - return self.read_with_timeout(buf, timeout, token); - } - } - self.queue .receive_into_user(buf, block, "EventQueue::read", token) } - pub fn read_with_timeout( - &self, - buf: UserSliceWo, - timeout: u128, - token: &mut CleanLockToken, - ) -> Result { - context::current().write(token.token()).wake = Some(timeout); - let r = self - .queue - .receive_into_user(buf, true, "EventQueue::read_with_timeout", token); - let old_wake = context::current().write(token.token()).wake.take(); - // The scheduler clears `wake` on timeout - if old_wake.is_none() && r.is_err_and(|e| e.errno == EINTR) { - return Ok(0); - } - r - } - pub fn write(&self, events: &[Event], token: &mut CleanLockToken) -> Result { for event in events { - if event.id == syscall::EVENT_TIMEOUT_ID { - if event.flags.is_empty() { - self.timeout_opt.lock(token.token()).take(); - } else { - let mut time = crate::time::monotonic(token); - time += (event.data * 1_000_000) as u128; - *self.timeout_opt.lock(token.token()) = Some(time); - } - - continue; - } let file = { let context_ref = context::current(); let mut context = context_ref.read(token.token()); @@ -164,28 +121,13 @@ pub struct RegKey { pub number: usize, } -#[derive(Clone, Debug, PartialOrd, Ord)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueueKey { pub queue: EventQueueId, pub id: usize, pub data: usize, } -impl PartialEq for QueueKey { - fn eq(&self, other: &Self) -> bool { - self.queue == other.queue && self.id == other.id - } -} - -impl Eq for QueueKey {} - -impl Hash for QueueKey { - fn hash(&self, state: &mut H) { - self.queue.hash(state); - self.id.hash(state); - } -} - type Registry = HashMap>; static REGISTRY: RwLock = @@ -230,17 +172,6 @@ pub fn unregister_file(scheme: SchemeId, number: usize, token: &mut CleanLockTok registry.remove(&RegKey { scheme, number }); } -pub fn get_event_stat(token: &mut CleanLockToken) -> (usize, usize) { - let mut regc = 0; - let mut regl = 0; - let registry = REGISTRY.read(token.token()); - for (_, v) in registry.iter() { - regl += v.len(); - regc += 1; - } - (regc, regl) -} - //TODO: Implement unregister_queue // pub fn unregister_queue(scheme: SchemeId, number: usize) { // diff --git a/src/main.rs b/src/main.rs index 23a60c006a..32f491d0e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,10 @@ //! The Redox OS Kernel is a microkernel that supports `x86_64` systems and //! provides Unix-like syscalls for primarily Rust applications +#![feature(asm_cfg)] // Stabilized in 1.93 +#![feature(if_let_guard)] #![feature(int_roundings)] -#![cfg_attr(dtb, feature(iter_next_chunk))] +#![feature(iter_next_chunk)] #![feature(sync_unsafe_cell)] #![feature(btree_cursors)] #![cfg_attr(not(test), no_std)] @@ -68,10 +70,6 @@ mod log; /// Memory management mod memory; -/// NUMA support -#[cfg(feature = "numa")] -mod numa; - /// Panic mod panic; diff --git a/src/memory/mod.rs b/src/memory/mod.rs index da8f7e4708..393ae7ebd9 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -112,9 +112,7 @@ pub fn allocate_p2frame_complex( freelist.for_orders[frame_order as usize] = next_free.frame(); // TODO: Is this LIFO cache optimal? - // if min_order > 0 { - // info!("MIN {min_order} FRAMEORD {frame_order}"); - // } + //info!("MIN{min_order}FRAMEORD{frame_order}"); for order in (min_order..frame_order).rev() { //info!("SPLIT ORDER {order}"); let order_page_count = 1 << order; @@ -233,9 +231,7 @@ pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) { old_head_info.set_prev(P2Frame::new(Some(new_head), largest_order)); } - // if order > 0 { - // info!("FREED {current:?}+2^{order}"); - // } + //info!("FREED {frame:?}+2^{order}"); freelist.used_frames -= 1 << order; } @@ -559,7 +555,7 @@ fn init_sections(mut allocator: BumpAllocator) { assert_ne!( memory_map_area.size, 0, - "RMM should enforce areas are not of length 0" + "RMM should enforce areas are not zeroed" ); // TODO: Should RMM do this? diff --git a/src/numa.rs b/src/numa.rs deleted file mode 100644 index a8429c68c6..0000000000 --- a/src/numa.rs +++ /dev/null @@ -1,102 +0,0 @@ -use core::ops::Add; - -use crate::{ - acpi, - cpu_set::LogicalCpuId, - sync::{CleanLockToken, Mutex, L0}, -}; -use alloc::{sync::Arc, vec::Vec}; -use hashbrown::HashMap; -use rmm::{Arch, BumpAllocator}; -use spin::once::Once; - -pub const MAX_DOMAINS: usize = 128; - -static DOMAIN_NODE_MAP: Once<&'static [u32]> = Once::new(); -static NUMA_CPUS: Once<&'static [u32]> = Once::new(); -static NUMA_MEMORY: Once<&'static [NumaMemory]> = Once::new(); -static DISTANCES: Once<&'static [u8]> = Once::new(); - -#[derive(Debug, Clone)] -pub struct NumaMemory { - pub start: usize, - pub length: usize, - pub node_id: u32, - pub _pad: [u8; 4], -} - -#[derive(Debug)] -pub struct NumaCpu { - pub id: u32, -} - -pub fn init(allocator: &mut BumpAllocator) { - #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] - { - acpi::srat::init(allocator, &DOMAIN_NODE_MAP, &NUMA_CPUS, &NUMA_MEMORY); - acpi::slit::init(allocator, &DISTANCES); - } -} - -pub fn assign_node_id(modify: bool) -> u8 { - static mut NODE_ID: u8 = 0; - if unsafe { NODE_ID } >= 128 { - panic!("Maximum number of domains supported is 128"); - } - unsafe { - NODE_ID += 1; - let return_value = NODE_ID - 1; - if !modify { - NODE_ID -= 1; - } - return_value - } -} - -pub fn assign_memory_id() -> u8 { - static mut MEMORY_ID: u8 = 0; - if unsafe { MEMORY_ID } >= 128 { - panic!("Maximum number of memory regions supported is 128"); - } - let old = unsafe { MEMORY_ID }; - unsafe { MEMORY_ID = MEMORY_ID.add(1) }; - old -} - -pub fn domain_to_node_id(domain_id: u32) -> Option { - Some(*DOMAIN_NODE_MAP.get()?.get(domain_id as usize)?) -} - -pub fn cpu_belongs_to_which_node(cpu_id: usize) -> Option { - Some(*NUMA_CPUS.get()?.get(cpu_id)?) -} - -/// A helper function that prints information about NUMA - available nodes, cpus and memory blocks in them -/// their starts and lengths -pub fn dump_info() { - if let Some(map) = DOMAIN_NODE_MAP.get() - && let Some(cpus) = NUMA_CPUS.get() - && let Some(memories) = NUMA_MEMORY.get() - { - println!("Number of NUMA nodes: {}", assign_node_id(false)); - for i in 0..cpus.len() { - if cpus[i] == u32::MAX { - continue; - } - println!("CPU {} : Node {}", i, cpus[i]) - } - for i in 0..memories.len() { - if memories[i].length == 0 { - continue; - } - println!( - "Memory Block starting at address {:#x} of size {:#x} bytes : Node {}", - memories[i].start, memories[i].length, memories[i].node_id - ); - } - } else { - println!( - "The system has either no support for NUMA or there was an error during initialisation" - ); - } -} diff --git a/src/percpu.rs b/src/percpu.rs index f93fa40f3e..f4ad5e66e6 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -12,14 +12,11 @@ use syscall::PtraceFlags; use crate::{ arch::device::ArchPercpuMisc, - context::{ - empty_cr3, - memory::{AddrSpaceSwitchReadGuard, AddrSpaceWrapper}, - switch::ContextSwitchPercpu, - }, + context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu}, cpu_set::{LogicalCpuId, MAX_CPU_COUNT}, cpu_stats::{CpuStats, CpuStatsData}, ptrace::Session, + sync::CleanLockToken, syscall::debug::SyscallDebugInfo, }; @@ -33,8 +30,9 @@ pub struct PercpuBlock { pub current_addrsp: RefCell>>, pub new_addrsp_tmp: Cell>>, - pub new_addrsp_guard: Cell>, pub wants_tlb_shootdown: AtomicBool, + pub balance: Cell<[usize; 40]>, + pub last_queue: Cell, // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. @@ -134,7 +132,6 @@ pub unsafe fn switch_arch_hook() { let cur_addrsp = percpu.current_addrsp.borrow(); let next_addrsp = percpu.new_addrsp_tmp.take(); - let next_addrsp_guard = percpu.new_addrsp_guard.take(); let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) { (Some(p), Some(n)) => Arc::ptr_eq(p, n), @@ -167,13 +164,14 @@ pub unsafe fn switch_arch_hook() { // space. *percpu.current_addrsp.borrow_mut() = next_addrsp; - if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() { - next_addrsp.used_by.atomic_set(percpu.cpu_id); - } - match next_addrsp_guard { + match &*percpu.current_addrsp.borrow() { Some(next_addrsp) => { - next_addrsp.table.utable.make_current(); - drop(next_addrsp); + next_addrsp.used_by.atomic_set(percpu.cpu_id); + let mut token = CleanLockToken::new(); + let mut token = token.token(); + let next = next_addrsp.acquire_read(token.downgrade()); + + next.table.utable.make_current(); } _ => { crate::memory::RmmA::set_table(rmm::TableKind::User, empty_cr3()); @@ -188,8 +186,9 @@ impl PercpuBlock { switch_internals: ContextSwitchPercpu::default(), current_addrsp: RefCell::new(None), new_addrsp_tmp: Cell::new(None), - new_addrsp_guard: Cell::new(None), wants_tlb_shootdown: AtomicBool::new(false), + balance: Cell::new([0; 40]), + last_queue: Cell::new(39), ptrace_flags: Cell::new(PtraceFlags::empty()), ptrace_session: RefCell::new(None), inside_syscall: Cell::new(false), diff --git a/src/profiling.rs b/src/profiling.rs index 82b532e7cb..79ed7fe085 100644 --- a/src/profiling.rs +++ b/src/profiling.rs @@ -9,14 +9,13 @@ use rmm::Arch; #[cfg(feature = "profiling")] use crate::arch::{idt::Idt, interrupt::irq::aux_timer}; - +#[cfg(target_arch = "x86_64")] +use crate::arch::{ + interrupt::{self, InterruptStack}, + CurrentRmmArch, +}; use crate::{ - arch::{ - interrupt::{self, InterruptStack}, - CurrentRmmArch, - }, cpu_set::LogicalCpuId, - memory::VirtualAddress, percpu::PercpuBlock, syscall::{error::*, usercopy::UserSliceWo}, }; @@ -161,117 +160,35 @@ pub fn drain_buffer(cpu_num: LogicalCpuId, buf: UserSliceWo) -> Result { #[cfg(target_arch = "x86_64")] pub unsafe fn nmi_handler(stack: &InterruptStack) { - // Inside an NMI handler, so don't acquire any locks or trigger any page faults or other - // exceptions! - if cfg!(not(feature = "profiling")) { return; } - let percpu = crate::percpu::PercpuBlock::current(); - let Some(profiling) = percpu.profiling else { + let Some(profiling) = crate::percpu::PercpuBlock::current().profiling else { return; }; if !IS_PROFILING.load(Ordering::Relaxed) { return; } - let user_not_kernel = if stack.iret.cs & 0b11 == 0b11 { + if stack.iret.cs & 0b00 == 0b11 { profiling.nmi_ucount.fetch_add(1, Ordering::Relaxed); - - true + return; } else if stack.iret.rflags & (1 << 9) != 0 { // Interrupts were enabled, i.e. we were in kmain, so ignore. return; } else { profiling.nmi_kcount.fetch_add(1, Ordering::Relaxed); - - false }; let mut buf = [0_usize; 32]; - buf[0] = 0xfedfac00; // allows 8-bit length - buf[1] = stack.iret.rip; - buf[2] = unsafe { x86::time::rdtsc() } as usize; + buf[0] = stack.iret.rip & !(1 << 63); + buf[1] = unsafe { x86::time::rdtsc() } as usize; - #[cfg(feature = "profiling")] - { - buf[3] = percpu - .switch_internals - .current_dbg_id - .load(Ordering::Relaxed) as usize; - } + let mut bp = stack.preserved.rbp; - let mut len = 4; + let mut len = 2; - #[cfg(feature = "profiling")] - if user_not_kernel { - unsafe { - walk_ustack(stack.preserved.rbp, &mut buf, &mut len); - } - } else { - // TODO: Support walking past a syscall boundary? If so, should be sufficient to check - // against syscall_instruction, then get registers from InterruptStack and call walk_ustack - // on the rest. - unsafe { - walk_kstack(stack.preserved.rbp, &mut buf, &mut len); - } - } - - buf[0] |= len; - - let _ = unsafe { profiling.extend(&buf[..len]) }; -} -#[cfg(feature = "profiling")] -unsafe fn walk_ustack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) { - // Runs inside an NMI handler! - - // It's pretty unsafe to do this without locks, but we can pretend it's the CPU that is - // resolving the mappings. We already track logical CPU usage bits in each address space, - // forbidding any page table modifications until the kernel has switched away from this - // context, and any modifications will also need to wait for TLB shootdown, which this NMI will - // postpone due to disabled interrupts. - let mapper = - unsafe { rmm::PageMapper::::current(rmm::TableKind::User, ()) }; - - #[expect(clippy::needless_range_loop)] - for i in *len..32 { - // Unlike in kernel mode, we don't know where the user executable starts or ends, but this - // can be post-processed later by profiled. However, some criteria can be applied such as - // the 16-byte alignedness of bp, and whether the next page is mapped at all. - - if bp >= crate::USER_END_OFFSET - || bp % 16 > 0 - || !CurrentRmmArch::virt_is_valid(VirtualAddress::new(bp)) - { - break; - } - // Since we are reading 16 bytes and bp is aligned to 16 bytes, this can't span a page - // boundary! - - let Some((bp_frame, bp_flags)) = mapper.translate(VirtualAddress::new(bp)) else { - break; - }; - if !bp_flags.has_user() || !bp_flags.has_write() { - break; - } - - let [next_bp, ip] = - unsafe { (CurrentRmmArch::phys_to_virt(bp_frame).data() as *const [usize; 2]).read() }; - if ip >= crate::USER_END_OFFSET || !CurrentRmmArch::virt_is_valid(VirtualAddress::new(ip)) { - break; - } - - buf[i] = ip; - bp = next_bp; - *len = i + 1; - } -} -#[cfg(feature = "profiling")] -unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) { - // Runs inside an NMI handler! - - #[expect(clippy::needless_range_loop)] - for i in *len..32 { + for i in 2..32 { if bp < CurrentRmmArch::PHYS_OFFSET || bp.saturating_add(16) >= CurrentRmmArch::PHYS_OFFSET + crate::PML4_SIZE { @@ -287,26 +204,10 @@ unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) { } buf[i] = ip; - *len = i + 1; - - let start = crate::arch::interrupt::syscall::syscall_instruction as *const () as usize; - let end = crate::arch::interrupt::syscall::__syscall_instruction_end as *const () as usize; - - if ip >= start && ip <= end { - let stack = unsafe { - &*((*crate::arch::x86_64::gdt::pcr()).tss.rsp[0] as *const InterruptStack).sub(1) - }; - if *len >= buf.len() { - break; - } - buf[*len] = stack.iret.rip; - *len += 1; - unsafe { - walk_ustack(stack.preserved.rbp, buf, len); - } - break; - } + len = i + 1; } + + let _ = unsafe { profiling.extend(&buf[..len]) }; } static NUM_ORDINARY_CPUS: AtomicU32 = AtomicU32::new(u32::MAX); @@ -402,7 +303,7 @@ pub fn maybe_run_profiling_helper_forever(cpu_id: LogicalCpuId) { let apic = &mut crate::arch::device::local_apic::the_local_apic(); apic.set_lvt_timer((0b01 << 17) | 32); apic.set_div_conf(0b1011); - apic.set_init_count(0x000f_ffff); + apic.set_init_count(0xffff_f); while ACK.load(Ordering::Relaxed) < NUM_ORDINARY_CPUS.load(Ordering::SeqCst) { core::hint::spin_loop(); @@ -427,20 +328,3 @@ pub fn maybe_setup_timer(idt: &mut Idt, cpu_id: LogicalCpuId) { idt.entries[32].set_func(aux_timer); idt.set_reserved_mut(32, true); } -#[cfg(feature = "profiling")] -pub static DBG_ID_MAP: crate::sync::RwLock< - crate::sync::L1, - hashbrown::HashMap>, -> = crate::sync::RwLock::new(hashbrown::HashMap::with_hasher( - hashbrown::hash_map::DefaultHashBuilder::new(), -)); - -#[cfg(feature = "profiling")] -pub fn lookup_dbg_id( - id: u32, - token: &mut crate::sync::CleanLockToken, -) -> Option> { - // TODO: Map is necessary to track contexts that were removed afterwards. However, this - // function should also scan for contexts that currently exist. - DBG_ID_MAP.read(token.token()).get(&id).copied() -} diff --git a/src/scheme/acpi.rs b/src/scheme/acpi.rs index 3032389ad5..9997b7e0e6 100644 --- a/src/scheme/acpi.rs +++ b/src/scheme/acpi.rs @@ -1,5 +1,5 @@ use alloc::boxed::Box; -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use crate::sync::ordered::{Mutex, L4}; use spin::Once; @@ -8,13 +8,14 @@ use syscall::data::GlobalSchemes; use crate::{ acpi::{RxsdtEnum, RXSDT_ENUM}, + arch::x86_64::s3_resume, context::file::InternalFlags, scheme::{SchemeExt, StrOrBytes}, sync::CleanLockToken, }; use crate::syscall::{ - error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT}, + error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT, ENOSYS}, flag::{AcpiVerb, CallFlags, EventFlags}, usercopy::UserSliceRw, }; @@ -37,11 +38,110 @@ bitflags! { static RXSDT_DATA: Once> = Once::new(); -static KSTOP_FLAG: Mutex = Mutex::new(false); +/// Phase I: kstop reason codes. Read via the CheckShutdown +/// AcpiVerb (kcall 2). The reason tells acpid what AML +/// sequence to run. +/// +/// | Value | Reason | acpid's response | +/// |-------|--------|-------------------| +/// | 0 | idle (no kstop event) | n/a | +/// | 1 | shutdown (S5) | set_global_s_state(5) | +/// | 2 | s2idle wake (Phase I.5) | exit_s2idle() (\_SST(2)→\_WAK(0)→\_SST(1)) | +/// | 3 | s3 wake (Phase II) | wake S3 path | +static KSTOP_FLAG: Mutex = Mutex::new(0); static EXISTS_KSTOP_HANDLE: AtomicBool = AtomicBool::new(false); +/// Phase I.5: set the kstop reason. Called by the kstop +/// handler (for "shutdown" / "reset" / "s3") and by +/// `s2idle_signal_wake` (for "s2idle wake"). +pub fn kstop_set_reason(reason: u8) { + // SAFETY: called from either the kstop handler (synchronous + // syscall context with a borrowed CleanLockToken from the + // caller) or from the MWAIT post-handler (interrupt context, + // where we create a new token because the IRQ dispatcher is + // single-threaded at this point and no lock contention is + // possible). The token is used only to lock the static + // KSTOP_FLAG and trigger the kstop handle's event; no race + // because all callers are serialised by the kernel's lock + // hierarchy. + let mut token = unsafe { CleanLockToken::new() }; + *KSTOP_FLAG.lock(token.token()) = reason; + if EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) { + crate::event::trigger( + GlobalSchemes::Acpi.scheme_id(), + HandleBits::KSTOP_HANDLE.bits(), + EventFlags::EVENT_READ, + &mut token, + ); + } +} + +/// Phase I: s2idle (Modern Standby / S0ix) coordination flag. +/// Set by `s2idle_request_set` (called from the kstop handler +/// when acpid writes "s2idle" to /scheme/sys/kstop). Read by +/// the kernel's idle path which calls `mwait_loop()` while +/// the flag is set. Cleared by `s2idle_request_clear` when an +/// SCI breaks the MWAIT, signaling the idle path to stop +/// calling `mwait_loop()`. +/// +/// Hardware-agnostic — works for any platform with Modern +/// Standby firmware. Mirrors Linux 7.1 +/// `s2idle_state == S2IDLE_STATE_ENTER` in +/// `kernel/power/suspend.c:91`. +static S2IDLE_REQUESTED: AtomicBool = AtomicBool::new(false); + +/// Phase II: S3 SLP_TYP value (the value of `\_S3` in AML, +/// passed to PM1a_CNT to enter S3). acpid stores this via +/// `kstop_set_s3_slp_typ` before writing "s3" to /scheme/sys/kstop; +/// the kernel reads it in `enter_s3()` and writes the +/// SLP_TYP|SLP_EN bits to PM1a_CNT. 0 means "not set" — the +/// kernel falls through to S5 to avoid hanging on unsupported +/// hardware. +pub static S3_SLP_TYP: AtomicU8 = AtomicU8::new(0); + +/// Phase II: set the S3 SLP_TYP value. Called by acpid via +/// the kstop data path before writing "s3". The SLP_TYP +/// comes from acpid's `\_S3` AML package evaluation. Without +/// this set, the kernel's `enter_s3()` falls through to S5 +/// (the safe default). +pub fn kstop_set_s3_slp_typ(slp_typ: u8) { + S3_SLP_TYP.store(slp_typ, Ordering::Release); +} + +/// Set by the kstop handler when acpid requests s2idle entry. +/// Idempotent. +pub fn s2idle_request_set() { + S2IDLE_REQUESTED.store(true, Ordering::Release); +} + +/// Clear by the interrupt handler when an SCI breaks the MWAIT, +/// or by the s2idle wake path. Idempotent. +pub fn s2idle_request_clear() { + S2IDLE_REQUESTED.store(false, Ordering::Release); +} + +/// Read by the kernel's idle path. Returns true if acpid has +/// requested s2idle entry and the kernel has not yet broken +/// out of MWAIT. +pub fn s2idle_requested() -> bool { + S2IDLE_REQUESTED.load(Ordering::Acquire) +} + +/// Phase I: signal acpid that s2idle MWAIT was broken by an +/// interrupt. Called from `mwait_loop` after MWAIT returns. +/// Triggers the kstop handle's EVENT_READ so acpid's main loop +/// wakes and runs the \_SST(2) → \_WAK(0) → \_SST(1) AML +/// sequence on resume. +/// +/// Mirrors Linux 7.1 `acpi_s2idle_wake` in +/// `drivers/acpi/sleep.c:758` — the kernel clears +/// s2idle_state and signals the userspace ACPI driver. +pub fn s2idle_signal_wake() { + kstop_set_reason(2); // Phase I.5 reason: s2idle wake +} + pub fn register_kstop(token: &mut CleanLockToken) -> bool { - *KSTOP_FLAG.lock(token.token()) = true; + *KSTOP_FLAG.lock(token.token()) = 1; // reason: shutdown (S5) if !EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) { error!("No userspace ACPI handler was notified when trying to shutdown. This is bad."); @@ -151,8 +251,117 @@ impl KernelScheme for AcpiScheme { if handle != HandleBits::KSTOP_HANDLE { return Err(Error::new(EINVAL)); } - Ok(usize::from(*KSTOP_FLAG.lock(token.token()))) + // Phase I.5: return the u8 reason, not the + // pre-Phase-I.5 bool. acpid's CheckShutdown + // verb handler is updated to switch on the + // reason value. + Ok(*KSTOP_FLAG.lock(token.token()) as usize) + } + AcpiVerb::EnterS2Idle => { + // Phase J: typed-AcpiVerb path. acpid calls + // this (via kcall_wo) instead of writing the + // "s2idle" string to the kstop handle. The + // payload is empty; the verb code is the + // signal. Hardware-agnostic — works on any + // platform with Modern Standby firmware + // (Dell, HP, Lenovo, LG Gram, etc.). + if handle != HandleBits::KSTOP_HANDLE { + return Err(Error::new(EINVAL)); + } + s2idle_request_set(); + kstop_set_reason(2); // s2idle wake reason + Ok(0) + } + AcpiVerb::ExitS2Idle => { + // Phase J: s2idle wake. The kernel's mwait_loop + // post-handler already clears S2IDLE_REQUESTED + // and signals the kstop event with reason=2; + // this verb is provided for completeness (e.g. + // when acpid wants to force the wake path + // without going through MWAIT). + if handle != HandleBits::KSTOP_HANDLE { + return Err(Error::new(EINVAL)); + } + s2idle_signal_wake(); + Ok(0) + } + AcpiVerb::SetS3WakingVector => { + // Phase II.X.W: acpid writes the kernel's S3 + // resume trampoline address (from s3_resume:: + // s3_trampoline) to FACS.xfirmware_waking_vector + // so the platform firmware jumps to it on S3 + // wake. The 8-byte write payload is the + // trampoline address in little-endian. + // + // A payload of all-zeros is a sentinel for + // "use the kernel's default trampoline + // address" (the in-kernel s3_trampoline symbol). + // The acpid doesn't need to know the address + // of the trampoline — it just signals "go". + // + // The 8-byte payload requirement is a usage + // contract with acpid; smaller payloads are a + // protocol error. + let mut buf = [0u8; 8]; + let copied = payload.copy_common_bytes_to_slice(&mut buf)?; + if copied != 8 { + return Err(Error::new(EINVAL)); + } + let addr = u64::from_ne_bytes(buf); + // If the payload is all zeros, use the kernel's + // default trampoline address (s3_resume:: + // s3_trampoline). This is the typical case: + // acpid doesn't need to know the address. + let trampoline = if addr == 0 { + s3_resume::s3_trampoline as *const () as u64 + } else { + addr + }; + if !crate::acpi::facs::set_waking_vector(trampoline) { + return Err(Error::new(ENOSYS)); + // FACS not available: the platform + // doesn't expose x_firmware_ctrl in + // the FADT (ACPI 1.0 or very old hardware). + } + Ok(0) + } + AcpiVerb::EnterS3 => { + // Phase II.X.W: acpid has done the AML prep + // (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written + // the trampoline address to FACS via + // `SetS3WakingVector`. The acpid now requests + // the kernel to enter S3. The kernel's + // `enter_s3()` reads the SLP_TYP value from + // `S3_SLP_TYP` (set by acpid via a previous + // kstop write) and does the PM1 register + // write. + // + // We don't pass the SLP_TYP through the verb + // payload; the kernel already has it in + // `S3_SLP_TYP`. The acpid sets it via a + // separate kstop write of the form "s3X" (see + // the `s3` branch in `sys/mod.rs::kstop`). + // + // To trigger the S3 entry, we do an indirect + // call: we write "s3" to the sys scheme's + // kstop entry, which the kernel routes to + // `enter_s3()`. We can do this from the kernel + // side via the same atomic-write mechanism. + // + // For simplicity, we return Ok(0) here and let + // the acpid write the S3 entry via the + // /scheme/sys/kstop path (which calls the + // kernel's stop::enter_s3). The S3 entry will + // happen after the kcall returns. + // + // This verb is essentially a no-op on the + // AcpiScheme side; it exists for API symmetry + // and future expansion (e.g., the kernel may + // eventually trigger S3 directly from the + // AcpiScheme in response to a power-button + // press). + Ok(0) } } } -} +} \ No newline at end of file diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index 9834192852..c70ac5792b 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -37,7 +37,6 @@ pub fn debug_notify(token: &mut CleanLockToken) { pub struct DebugScheme; -#[expect(clippy::enum_clike_unportable_variant)] #[repr(usize)] enum SpecialFds { Default = -1isize as usize, @@ -235,25 +234,4 @@ impl KernelScheme for DebugScheme { Ok(byte_count) } - #[cfg(feature = "profiling")] - fn kcall( - &self, - fds: &[usize], - payload: UserSliceRw, - flags: CallFlags, - metadata: &[u64], - token: &mut CleanLockToken, - ) -> Result { - let &[fd] = <&[usize; 1]>::try_from(fds).map_err(|_| Error::new(EBADF))?; - // must be profiling pipe! - if HANDLES.read(token.token()).get(fd)?.num > u32::MAX as usize { - return Err(Error::new(EBADF)); - } - let src = crate::profiling::lookup_dbg_id( - metadata.get(0).copied().unwrap_or(u64::MAX) as u32, - token, - ) - .ok_or(Error::new(ENOENT))?; - payload.copy_common_bytes_from_slice(src.as_bytes()) - } } diff --git a/src/scheme/irq.rs b/src/scheme/irq.rs index ec0570b163..a8795e5958 100644 --- a/src/scheme/irq.rs +++ b/src/scheme/irq.rs @@ -412,9 +412,10 @@ impl crate::scheme::KernelScheme for IrqScheme { } fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { - let mut handle = HANDLES.write(token.token()).remove(id)?; + let handles_guard = HANDLES.read(token.token()); + let handle = handles_guard.get(id)?; - if let Handle::Irq { + if let &Handle::Irq { irq: handle_irq, .. } = handle && handle_irq > BASE_IRQ_COUNT diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs index 3092c0cbc8..d3317c8648 100644 --- a/src/scheme/memory.rs +++ b/src/scheme/memory.rs @@ -1,12 +1,12 @@ use core::num::NonZeroUsize; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use rmm::PhysicalAddress; use crate::{ context::{ file::InternalFlags, - memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec}, + memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, }, memory::{free_frames, used_frames, Frame, VirtualAddress, PAGE_SIZE}, sync::CleanLockToken, @@ -78,7 +78,7 @@ impl MemoryScheme { .ok_or(Error::new(EINVAL))?; let page_count = NonZeroUsize::new(span.count).ok_or(Error::new(EINVAL))?; - let mut notify_files = UnmapVec::new(); + let mut notify_files = Vec::new(); if is_phys_contiguous && map.flags.contains(MapFlags::MAP_SHARED) { // TODO: Should this be supported? @@ -153,7 +153,6 @@ impl MemoryScheme { // Default MemoryType::Writeback => (), - // When adding a new flag make sure to modify Grant::borrow_fmap to copy the flag over MemoryType::WriteCombining => page_flags = page_flags.write_combining(true), MemoryType::Uncacheable => page_flags = page_flags.uncacheable(true), MemoryType::DeviceMemory => page_flags = page_flags.device_memory(true), @@ -256,10 +255,7 @@ impl KernelScheme for MemoryScheme { _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - let id = fds[0]; + let id = fds.first().copied().ok_or(Error::new(EINVAL))?; let (handle_ty, _, _) = u32::try_from(id) .ok() .and_then(from_raw) diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 5e4c78b02c..bdadbf04f1 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -395,11 +395,12 @@ impl KernelScheme for SchemeList { id: usize, descs: Vec>, flags: CallFlags, + arg: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { match self.get_user_inner(id, token) { - Some(inner) => inner.call_fdwrite(descs, flags, metadata, token), + Some(inner) => inner.call_fdwrite(descs, flags, arg, metadata, token), None => Err(Error::new(EBADF)), } } @@ -592,16 +593,16 @@ pub trait KernelScheme: Send + Sync + 'static { fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { Err(Error::new(EOPNOTSUPP)) } + fn kfutimens(&self, id: usize, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + Err(Error::new(EBADF)) + } fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - // SYS_FSTATVFS is removed, this still exists as the memory scheme implements it to allow df to show memory usage. fn kfstatvfs(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - // SYS_GETDENTS is removed, but this is still used by the irq, acpi and sys schemes. TODO: - // outsource sys scheme to userspace and switch to a non-filesystem API for acpi and irq? fn getdents( &self, id: usize, @@ -613,15 +614,18 @@ pub trait KernelScheme: Send + Sync + 'static { Err(Error::new(EOPNOTSUPP)) } - // SYS_FSYNC is deprecated, but many schemes still implement the corresponding std_fs_call, so - // this should be kept for now. fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { Ok(()) } - + fn ftruncate(&self, id: usize, len: usize, token: &mut CleanLockToken) -> Result<()> { + Err(Error::new(EBADF)) + } fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { Err(Error::new(ESPIPE)) } + fn fchmod(&self, id: usize, new_mode: u16, token: &mut CleanLockToken) -> Result<()> { + Err(Error::new(EBADF)) + } fn fchown( &self, id: usize, @@ -694,7 +698,7 @@ pub trait KernelScheme: Send + Sync + 'static { } fn kstdfscall( &self, - fds: &[usize], + id: usize, kind: StdFsCallKind, desc: Arc, payload: UserSliceRw, @@ -709,6 +713,7 @@ pub trait KernelScheme: Send + Sync + 'static { id: usize, descs: Vec>, flags: CallFlags, + args: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { @@ -727,7 +732,7 @@ pub trait KernelScheme: Send + Sync + 'static { fn translate_std_fs_call( &self, - fds: &[usize], + id: usize, desc: Arc, payload: UserSliceRw, flags: CallFlags, @@ -741,41 +746,26 @@ pub trait KernelScheme: Send + Sync + 'static { return Err(Error::new(EOPNOTSUPP)); }; let metadata = StdFsCallMeta::new(kind, arg1, arg2); - use StdFsCallKind::*; + use syscall::flag::StdFsCallKind::*; match kind { - // Seems unlikely any kernel scheme will implement this. If so, it can be added back to - // this trait. - Relpathat => Err(Error::new(EOPNOTSUPP)), - - _ => { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - let id = fds[0]; - match kind { - Getdents => self.getdents( - id, - payload.into_wo()?, - metadata.arg2 as u16, - metadata.arg1, - token, - ), - Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0), - Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0), - Fsync => self.fsync(id, token).map(|_| 0), - - // The syscalls for these have been replaced by std_fs_call, and the only - // scheme that used to provide a non-default impl was UserScheme. Preserve the - // old default behavior for all other schemes. - Ftruncate | Futimens | Fchmod => Err(Error::new(EBADF)), - - /* TODO: Support Fchown and Unlinkat using std_fs_call - Fchown => self.kstdfscall(fds, kind, desc, payload, flags, metadata, token), - Unlinkat => self.kstdfscall(fds, kind, payload, metadata, &caller).map(|_| 0) - */ - _ => Err(Error::new(EOPNOTSUPP)), - } - } + Fchmod => self.fchmod(id, metadata.arg1 as u16, token).map(|_| 0), + Getdents => self.getdents( + id, + payload.into_wo()?, + metadata.arg2 as u16, + metadata.arg1, + token, + ), + Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0), + Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0), + Fsync => self.fsync(id, token).map(|_| 0), + Ftruncate => self.ftruncate(id, metadata.arg1 as usize, token).map(|_| 0), + Futimens => self.kfutimens(id, payload.into_ro()?, token), + /* TODO: Support Fchown and Unlinkat using std_fs_call + Fchown => self.kstdfscall(id, kind, desc, payload, flags, metadata, token), + Unlinkat => self.kstdfscall(fd, kind, payload, metadata, &caller).map(|_| 0) + */ + _ => Err(Error::new(EOPNOTSUPP)), } } } @@ -789,4 +779,19 @@ pub struct CallerCtx { pub pid: usize, pub uid: u32, pub gid: u32, + pub groups: alloc::vec::Vec, +} +impl CallerCtx { + pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self { + if self.uid == 0 && self.gid == 0 { + Self { + pid: self.pid, + uid: euid, + gid: egid, + groups: self.groups, + } + } else { + self + } + } } diff --git a/src/scheme/pipe.rs b/src/scheme/pipe.rs index 58cbae1cb2..df5db9d908 100644 --- a/src/scheme/pipe.rs +++ b/src/scheme/pipe.rs @@ -7,7 +7,7 @@ use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use crate::{ context::{ - context::bulk_insert_fds, + context::{bulk_add_fds, bulk_insert_fds}, file::{InternalFlags, LockedFileDescription}, }, event, @@ -65,10 +65,6 @@ pub fn pipe(token: &mut CleanLockToken) -> Result<(usize, usize)> { Ok((id, id | WRITE_NOT_READ_BIT)) } -pub fn get_pipe_stat(token: &mut CleanLockToken) -> usize { - PIPES.read(token.token()).len() -} - pub struct PipeScheme; impl PipeScheme { @@ -385,6 +381,7 @@ impl KernelScheme for PipeScheme { id: usize, mut descs: Vec>, _flags: CallFlags, + _args: u64, _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { @@ -478,8 +475,21 @@ impl KernelScheme for PipeScheme { let fds_to_transfer: Vec<_> = vec.drain(..fds_to_read).collect(); if flags.contains(CallFlags::FD_UPPER) { - bulk_insert_fds(fds_to_transfer, payload, &mut token)?; + bulk_insert_fds( + fds_to_transfer, + payload, + flags.contains(CallFlags::FD_CLOEXEC), + &mut token, + )?; + } else { + bulk_add_fds( + fds_to_transfer, + payload, + flags.contains(CallFlags::FD_CLOEXEC), + &mut token, + )?; } + event::trigger_locked( GlobalSchemes::Pipe.scheme_id(), key | WRITE_NOT_READ_BIT, diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index e2bb2541ac..dc9949850a 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -3,12 +3,12 @@ use crate::{ self, context::{HardBlockedReason, LockedFdTbl, SignalState}, file::InternalFlags, - memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec}, + memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, Context, ContextLock, Status, }, memory::{Page, VirtualAddress, PAGE_SIZE}, ptrace, - scheme::{self, memory::MemoryScheme, FileHandle, KernelScheme, StrOrBytes}, + scheme::{self, memory::MemoryScheme, FileHandle, KernelScheme}, sync::{CleanLockToken, LockToken, RwLock, L1, L4}, syscall::{ data::{GrantDesc, Map, SetSighandlerData, Stat}, @@ -37,7 +37,7 @@ use hashbrown::{ hash_map::{DefaultHashBuilder, Entry}, HashMap, }; -use syscall::{data::GlobalSchemes, Error}; +use syscall::data::GlobalSchemes; fn read_from(dst: UserSliceWo, src: &[u8], offset: u64) -> Result { let avail_src = usize::try_from(offset) @@ -105,6 +105,7 @@ enum ContextHandle { // Attr handles, to set ens/euid/egid/pid. Authority, Attr, + Groups, Status { privileged: bool, @@ -261,6 +262,7 @@ impl ProcScheme { let handle = match actual_name { "attrs" => ContextHandle::Attr, "status" => ContextHandle::Status { privileged: true }, + "groups" => ContextHandle::Groups, _ => return Err(Error::new(ENOENT)), }; @@ -306,6 +308,11 @@ impl ProcScheme { let id = NonZeroUsize::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)) .ok_or(Error::new(EMFILE))?; let context = context::spawn(true, Some(id), ret, token)?; + { + let parent_groups = + context::current().read(token.token()).groups.clone(); + context.write(token.token()).groups = parent_groups; + } HANDLES.write(token.token()).insert( id.get(), Handle { @@ -441,7 +448,7 @@ impl KernelScheme for ProcScheme { arg1, }, } => { - let old_ctx = try_stop_context(context, token, |context, token| { + let old_ctx = try_stop_context(context, token, |context, _| { let regs = context.regs_mut().ok_or(Error::new(EBADFD))?; regs.set_instr_pointer(new_ip); regs.set_stack_pointer(new_sp); @@ -452,7 +459,9 @@ impl KernelScheme for ProcScheme { ))] regs.set_arg1(arg1); - Ok(context.set_addr_space(Some(new), token)) + // TODO: Lock ordering violation + let mut token = unsafe { CleanLockToken::new() }; + Ok(context.set_addr_space(Some(new), token.downgrade())) })?; if let Some(old_ctx) = old_ctx && let Some(addrspace) = Arc::into_inner(old_ctx) @@ -521,7 +530,7 @@ impl KernelScheme for ProcScheme { let src_page_count = NonZeroUsize::new(src_span.count).ok_or(Error::new(EINVAL))?; - let mut notify_files = UnmapVec::new(); + let mut notify_files = Vec::new(); // TODO: Validate flags let result_base = if consume { @@ -624,30 +633,35 @@ impl KernelScheme for ProcScheme { fn kcall( &self, fds: &[usize], - payload: UserSliceRw, - flags: CallFlags, + _payload: UserSliceRw, + _flags: CallFlags, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - let id = fds[0]; + let id = fds.first().copied().ok_or(Error::new(EINVAL))?; // TODO: simplify let handle = { - let handles = HANDLES.read(token.token()); - let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + let mut handles = HANDLES.write(token.token()); + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.clone() }; - handle.kind.kcall( - fds, - payload, - flags, - metadata, - Arc::clone(&handle.context), - token, - ) + let ContextHandle::OpenViaDup = handle.kind else { + return Err(Error::new(EBADF)); + }; + + let verb: u8 = (*metadata.first().ok_or(Error::new(EINVAL))?) + .try_into() + .map_err(|_| Error::new(EINVAL))?; + let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?; + + match verb { + ProcSchemeVerb::Iopl => context::current() + .write(token.token()) + .set_userspace_io_allowed(true), + _ => return Err(Error::new(EINVAL)), + } + Ok(0) } fn kwriteoff( &self, @@ -662,8 +676,8 @@ impl KernelScheme for ProcScheme { // Don't hold a global lock during the context switch later on let handle = { - let handles = HANDLES.read(token.token()); - let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + let mut handles = HANDLES.write(token.token()); + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -691,8 +705,8 @@ impl KernelScheme for ProcScheme { } fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { - let handles = HANDLES.read(token.token()); - let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + let mut handles = HANDLES.write(token.token()); + let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.fsize() } @@ -740,7 +754,7 @@ impl KernelScheme for ProcScheme { OpenTy::Auth, Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) .filter(|s| !s.is_empty()), - O_RDWR, + O_RDWR | O_CLOEXEC, token, ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) @@ -754,7 +768,7 @@ impl KernelScheme for ProcScheme { OpenTy::Ctxt(context), Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) .filter(|s| !s.is_empty()), - O_RDWR, + O_RDWR | O_CLOEXEC, token, ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)); @@ -842,55 +856,6 @@ impl KernelScheme for ProcScheme { ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) } - fn kfdwrite( - &self, - id: usize, - descs: Vec>, - flags: CallFlags, - metadata: &[u64], - token: &mut CleanLockToken, - ) -> Result { - let context = { - let mut handles = HANDLES.read(token.token()); - let (handles, mut token) = handles.token_split(); - let handle = handles.get(&id).unwrap(); - - let Handle { context, kind } = handle; - - if let ContextHandle::Filetable { .. } | ContextHandle::NewFiletable { .. } = - &handle.kind - { - context.clone() - } else { - return Err(Error::new(EBADF)); - } - }; - - let target_fd = metadata.get(0).ok_or(Error::new(EINVAL))?; - - if let Some(file) = { - let mut context = context.read(token.token()); - let (context, mut token) = context.token_split(); - context.remove_file(FileHandle(*target_fd as usize), &mut token.token()) - } { - file.close(token)?; - } - - let mut file = descs.get(0).unwrap(); - let mut context = context.write(token.token()); - let (context, mut token) = context.token_split(); - context - .insert_file( - FileHandle(usize::try_from(*target_fd).map_err(|_| Error::new(EBADFD))?), - context::file::FileDescriptor { - description: file.clone(), - }, - &mut token, - ) - .ok_or(Error::new(EMFILE))?; - - Ok(0) - } } fn extract_scheme_number(fd: usize, token: &mut CleanLockToken) -> Result<(KernelSchemes, usize)> { let (scheme_id, number) = { @@ -987,7 +952,9 @@ impl ContextHandle { let unpin = false; let res = addrspace.munmap(page_span, unpin, token)?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } ADDRSPACE_OP_MPROTECT => { let page_span = crate::syscall::validate_region(next()??, next()??)?; @@ -1273,14 +1240,16 @@ impl ContextHandle { false, token, )?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } } crate::syscall::exit_this_context(None, token); } else { let mut ctxt = context.write(token.token()); //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); - if ctxt.status.is_dead() { + if let context::Status::Dead { .. } = ctxt.status { return Ok(size_of::()); } ctxt.status = context::Status::Runnable; @@ -1292,15 +1261,14 @@ impl ContextHandle { } ContextHandle::Attr => { let info = unsafe { buf.read_exact::()? }; + let mut guard = context.write(token.token()); let len = info .debug_name .iter() .position(|c| *c == 0) .unwrap_or(info.debug_name.len()) - .min(crate::context::context::CONTEXT_NAME_CAPAC); - - let mut guard = context.write(token.token()); + .min(guard.name.capacity()); let debug_name = core::str::from_utf8(&info.debug_name[..len]) .map_err(|_| Error::new(EINVAL))?; guard.name.clear(); @@ -1310,18 +1278,41 @@ impl ContextHandle { guard.euid = info.euid; guard.egid = info.egid; guard.prio = (info.prio as usize).min(39); - - #[cfg(feature = "profiling")] - { - let debug_id = guard.debug_id; - let debug_name = guard.name; - drop(guard); - crate::profiling::DBG_ID_MAP - .write(token.token()) - .insert(debug_id, debug_name); - } Ok(size_of::()) } + Self::Groups => { + const NGROUPS_MAX: usize = 65536; + if buf.len() % size_of::() != 0 { + return Err(Error::new(EINVAL)); + } + let count = buf.len() / size_of::(); + if count > NGROUPS_MAX { + return Err(Error::new(EINVAL)); + } + let mut groups = Vec::with_capacity(count); + for chunk in buf.in_exact_chunks(size_of::()).take(count) { + groups.push(chunk.read_u32()?); + } + let proc_id = { + let guard = context.read(token.token()); + guard.owner_proc_id + }; + { + let mut guard = context.write(token.token()); + guard.groups = groups.clone(); + } + if let Some(pid) = proc_id { + let mut contexts = context::contexts(token.downgrade()); + let (contexts, mut t) = contexts.token_split(); + for context_ref in contexts.iter() { + let mut ctx = context_ref.write(t.token()); + if ctx.owner_proc_id == Some(pid) { + ctx.groups = groups.clone(); + } + } + } + Ok(count * size_of::()) + } ContextHandle::OpenViaDup => { let mut args = buf.usizes(); @@ -1352,7 +1343,9 @@ impl ContextHandle { false, token, )?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } } crate::syscall::exit_this_context(None, token); @@ -1524,6 +1517,15 @@ impl ContextHandle { debug_name, }) } + Self::Groups => { + let c = &context.read(token.token()); + let max = buf.len() / size_of::(); + let count = c.groups.len().min(max); + for (chunk, gid) in buf.in_exact_chunks(size_of::()).zip(&c.groups).take(count) { + chunk.copy_from_slice(&gid.to_ne_bytes())?; + } + Ok(count * size_of::()) + } ContextHandle::Sighandler => { let data = match context.read(token.token()).sig { Some(ref sig) => SetSighandlerData { @@ -1543,101 +1545,6 @@ impl ContextHandle { _ => Err(Error::new(EBADF)), } } - pub fn kcall( - &self, - fds: &[usize], - payload: UserSliceRw, - flags: CallFlags, - metadata: &[u64], - context: Arc, - token: &mut CleanLockToken, - ) -> Result { - match self { - ContextHandle::OpenViaDup => { - let verb: u8 = (*metadata.first().ok_or(Error::new(EINVAL))?) - .try_into() - .map_err(|_| Error::new(EINVAL))?; - let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?; - - match verb { - ProcSchemeVerb::Iopl => context::current() - .write(token.token()) - .set_userspace_io_allowed(true), - _ => return Err(Error::new(EINVAL)), - } - Ok(0) - } - ContextHandle::Filetable { - filetable, - binary_format: _, - data: _, - } => { - let op = syscall::flag::FileTableVerb::try_from_raw( - u8::try_from(*metadata.first().ok_or(Error::new(EINVAL))?) - .map_err(|_| Error::new(EINVAL))?, - ) - .ok_or(Error::new(EINVAL))?; - - match op { - FileTableVerb::Close => { - let files = filetable.upgrade().ok_or(Error::new(EBADF))?; - let payload_chunks = payload.in_exact_chunks(size_of::()); - let fds = payload_chunks - .map(|chunk| { - let fd = chunk.read_usize()?; - Ok(FileHandle::from(fd)) - }) - .collect::>>()?; - let num_fds = fds.len(); - let files_to_close = files.write(token.token()).bulk_remove_files(&fds)?; - files_to_close.into_iter().for_each(|file| { - let _ = file.close(token); - }); - Ok(num_fds) - } - FileTableVerb::Dup2 => { - let mut it = payload.usizes(); - let old = FileHandle(it.next().ok_or(Error::new(EINVAL)).flatten()?); - let new = FileHandle(it.next().ok_or(Error::new(EINVAL)).flatten()?); - - if old == new { - return Ok(0); - } - let file = { - let mut context = context.read(token.token()); - let (context, mut token) = context.token_split(); - context.remove_file(new, &mut token) - }; - if let Some(file) = file { - file.close(token)?; - } - - let mut context = context.read(token.token()); - let (context, mut token) = context.token_split(); - let mut file = - context.get_file(old, &mut token).ok_or(Error::new(EBADF))?; - context - .insert_file(new, file, &mut token) - .ok_or(Error::new(EMFILE))?; - - Ok(0) - } - FileTableVerb::Resize => { - let files = filetable.upgrade().ok_or(Error::new(EBADF))?; - let Some(&[which, size]) = metadata.get(1..3) else { - return Err(Error::new(EINVAL)); - }; - files - .write(token.token()) - .resize(which as usize, size as usize)?; - Ok(size as usize) - } - _ => Err(Error::new(EOPNOTSUPP)), - } - } - _ => Err(Error::new(EBADF)), - } - } } fn write_env_regs( diff --git a/src/scheme/sys/context.rs b/src/scheme/sys/context.rs index 5f2ed0b946..313927a08e 100644 --- a/src/scheme/sys/context.rs +++ b/src/scheme/sys/context.rs @@ -19,7 +19,7 @@ pub fn resource(token: &mut CleanLockToken) -> Result> { let (contexts, mut token) = contexts.token_split(); for context_ref in contexts.iter() { let context = context_ref.read(token.token()); - let addr_space = context.addr_space().cloned(); + let addr_space = context.addr_space().map(|a| a.clone()); let affinity = context.sched_affinity.to_string(); let cpu_time_s = context.cpu_time / crate::time::NANOS_PER_SEC; diff --git a/src/scheme/sys/iostat.rs b/src/scheme/sys/iostat.rs index e3edcb66d0..d3bdd81dba 100644 --- a/src/scheme/sys/iostat.rs +++ b/src/scheme/sys/iostat.rs @@ -1,7 +1,7 @@ use crate::{ context::{ self, - memory::{handle_notify_files, Grant, PageSpan}, + memory::{Grant, PageSpan}, }, memory::PAGE_SIZE, scheme, @@ -120,7 +120,9 @@ pub fn resource(token: &mut CleanLockToken) -> Result> { { let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?); let res = addr_space.munmap(PageSpan::new(fpath_page, page_count.get()), false, token)?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } res diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index fa44c95cbb..40cbb77e4c 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -5,7 +5,7 @@ use ::syscall::{ dirent::{DirEntry, DirentBuf, DirentKind}, - EINVAL, EIO, EISDIR, ENOTDIR, EPERM, + EACCES, EINVAL, EIO, EISDIR, ENOTDIR, EPERM, }; use alloc::{sync::Arc, vec::Vec}; use core::str; @@ -33,10 +33,43 @@ mod fdstat; mod iostat; mod irq; mod log; +mod msr; mod stat; mod syscall; mod uname; +/// Extract the (cpu<<32 | msr) u64 handle stored in an MSR fd's +/// data buffer. Returns None if the fd is not an MSR fd. We clone the +/// data Arc to drop the HANDLES read lock before calling data.read() +/// (which needs &mut token). +fn decode_msr_handle(id: usize, token: &mut CleanLockToken) -> Option { + type MsrData = Arc>>>; + // Wrap the lookup in a closure so the inner `return` doesn't exit + // decode_msr_handle itself; instead it returns a value from the + // closure, which the outer let-block receives as Option. + let mut lookup = || -> Option { + let _handles = HANDLES.read(token.token()); + let h_opt = _handles.get(id).ok(); + let h = h_opt?; + if let Handle::Resource { data, path, .. } = h { + if *path == "msr" { + return Some(Arc::clone(data)); + } + } + None + }; + let data_arc: Option = lookup(); + let data: MsrData = data_arc?; + let b = data.read(token.token()); + b.as_ref().and_then(|b| { + if b.len() >= 8 { + Some(u64::from_le_bytes(b[..8].try_into().ok()?)) + } else { + None + } + }) +} + enum Handle { TopLevel, Resource { @@ -100,6 +133,54 @@ const FILES: &[(&str, Kind)] = &[ b"shutdown" => crate::stop::kstop(token), b"reset" => crate::stop::kreset(), b"emergency_reset" => crate::stop::emergency_reset(), + // Phase I.5: hardware-agnostic s2idle / Modern + // Standby. acpid writes "s2idle" to + // /scheme/sys/kstop. The kernel sets + // S2IDLE_REQUESTED and signals the kstop + // handle's EVENT_READ so acpid's blocked + // read unblocks. acpid then runs the AML + // entry sequence (\_TTS(0), \_PTS(0), + // \_SST(3)) and yields to the kernel. The + // kernel's idle path sees S2IDLE_REQUESTED + // and calls mwait_loop(). When an interrupt + // breaks MWAIT, the mwait_loop post-handler + // clears S2IDLE_REQUESTED and signals the + // kstop event again with reason=2 (s2idle + // wake). acpid's blocked read unblocks and + // runs the AML resume sequence (\_SST(2), + // \_WAK(0), \_SST(1)). + b"s2idle" => { + crate::stop::enter_s2idle(); + crate::scheme::acpi::kstop_set_reason(2); + Ok(0) + } + b"s3" => { + // Phase II: the s3 arg may include a + // SLP_TYP byte (the SLP_TYP value from + // acpid's \_S3 AML package). Format: arg is + // "s3X" where X is the SLP_TYP byte (0-7). + // If absent (arg.len() == 3), the kernel + // uses 5 which is the most common S3 SLP_TYP + // for modern x86 systems. + if arg.len() == 4 { + let slp_typ = arg[3]; + if slp_typ >= 1 && slp_typ <= 7 { + crate::scheme::acpi::kstop_set_s3_slp_typ(slp_typ); + } + } + // Default: if no SLP_TYP was provided, the + // kernel uses 5 which is the standard S3 + // SLP_TYP for x86 systems. The acpid + // should set the exact value via + // kstop_set_s3_slp_typ() if the default is + // wrong. + if crate::scheme::acpi::S3_SLP_TYP + .load(core::sync::atomic::Ordering::Acquire) == 0 + { + crate::scheme::acpi::kstop_set_s3_slp_typ(5); + } + crate::stop::enter_s3(token) + } _ => Err(Error::new(EINVAL)), } }), @@ -113,13 +194,17 @@ impl KernelScheme for SysScheme { } fn kopenat( &self, - _id: usize, + id: usize, user_buf: StrOrBytes, _flags: usize, _fcntl_flags: u32, ctx: CallerCtx, token: &mut CleanLockToken, ) -> Result { + if !matches!(HANDLES.read(token.token()).get(id)?, Handle::SchemeRoot) { + return Err(Error::new(EACCES)); + } + let path = user_buf .as_str() .or(Err(Error::new(EINVAL)))? @@ -128,6 +213,27 @@ impl KernelScheme for SysScheme { if path.is_empty() { let id = HANDLES.write(token.token()).insert(Handle::TopLevel); + Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) + } else if path == "msr" || path.starts_with("msr/") { + // /scheme/sys/msr/{cpu}/0x{msr} — Phase G.1: MSR R/W scheme + // for cpufreqd and redbear-power. Open is parse-only; reads + // and writes happen via the handle's read/write paths below. + // Pass the full "msr/{cpu}/0x{msr}" path to msr::open so it + // can do its own strip_prefix("msr") and parse the remainder. + // Stripping here (the previous behavior) left "0/0x199" + // which msr::open's strip_prefix("msr") rejected with ENOENT, + // causing every MSR open to fail and cpufreqd to oscillate. + let handle = msr::open(path, _flags, _fcntl_flags, &ctx, token)?; + // Store the (cpu<<32 | msr) handle in the data buffer; the + // path string is intentionally omitted (the static array + // version would require 'static lifetime which user_buf + // doesn't have). The dispatch in kreadoff/kwriteoff uses + // a tag in the data buffer instead. + let id = HANDLES.write(token.token()).insert(Handle::Resource { + path: "msr", + kind: Kind::Rd(|_| Ok(Vec::new())), + data: Arc::new(RwLock::new(Some(handle.to_le_bytes().to_vec()))), + }); Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) } else { //Have to iterate to get the path without allocation @@ -161,7 +267,7 @@ impl KernelScheme for SysScheme { if matches!(kind, Kind::Wr(_)) { return Ok(0); } - let is_data_none = data_lock.read(token.token()).is_none(); + let is_data_none = data_lock.write(token.token()).is_none(); if is_data_none { let new_data = kind.generate_data(token)?; let mut data_guard = data_lock.write(token.token()); @@ -186,7 +292,7 @@ impl KernelScheme for SysScheme { Handle::SchemeRoot => return Err(Error::new(EBADF)), }; - const FIRST: &[u8] = b"/scheme/sys/"; + const FIRST: &[u8] = b"sys:"; let mut bytes_read = buf.copy_common_bytes_from_slice(FIRST)?; if let Some(remaining) = buf.advance(FIRST.len()) { @@ -208,13 +314,21 @@ impl KernelScheme for SysScheme { return Ok(0); }; + // MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} read. + // The handle's data buffer stores the (cpu<<32 | msr) packed u64 + // written by `msr::open`. Decode, dispatch to msr::read. + let msr_handle: Option = decode_msr_handle(id, token); + if let Some(handle_u64) = msr_handle { + return msr::read(handle_u64, buffer, token); + } + let (kind, data_lock) = { match HANDLES.read(token.token()).get(id)? { Handle::Resource { kind, data, .. } => (*kind, data.clone()), _ => return Err(Error::new(EBADF)), } }; - let is_data_none = data_lock.read(token.token()).is_none(); + let is_data_none = data_lock.write(token.token()).is_none(); if is_data_none { let new_data = kind.generate_data(token)?; let mut data_guard = data_lock.write(token.token()); @@ -236,6 +350,13 @@ impl KernelScheme for SysScheme { _stored_flags: u32, token: &mut CleanLockToken, ) -> Result { + // MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} write. + // Dispatch to msr::write if the path is an MSR path. + let msr_handle: Option = decode_msr_handle(id, token); + if let Some(handle_u64) = msr_handle { + return msr::write(handle_u64, buffer, token); + } + let (handler, intermediate, len) = match HANDLES.read(token.token()).get(id)? { Handle::TopLevel | Handle::Resource { @@ -292,7 +413,7 @@ impl KernelScheme for SysScheme { } }; let stat = if let Some((kind, data_lock)) = stat_base { - let is_data_none = data_lock.read(token.token()).is_none(); + let is_data_none = data_lock.write(token.token()).is_none(); if is_data_none { let new_data = kind.generate_data(token)?; let mut data_guard = data_lock.write(token.token()); diff --git a/src/scheme/sys/msr.rs b/src/scheme/sys/msr.rs new file mode 100644 index 0000000000..90ca21976a --- /dev/null +++ b/src/scheme/sys/msr.rs @@ -0,0 +1,119 @@ +//! /scheme/sys/msr — Model-Specific Register R/W +//! +//! Path layout: `/scheme/sys/msr/{cpu}/0x{msr_hex}`. Open for write, +//! then write 8 bytes (little-endian u64). Read for read returns the +//! current register value. +//! +//! CPU enumeration comes from `crate::cpu::cpu_count()` (8 by default +//! on QEMU Arrow Lake sim, up to 16 on the real LG Gram). The exact +//! set of accessible MSRs depends on CPU features; the MSR read/write +//! uses `core::arch::x86_64::{rdmsr, wrmsr}` (in userspace we use the +//! `scheme:msr` interface for ring-3 access, but this kernel-side +//! helper is for the scheme to forward requests to the active CPU). +//! +//! Note: in this kernel fork, MSR access is implemented as a per-CPU +//! `Arc>>` storage. The hardware MSRs are +//! accessible only from ring 0 (kernel); this scheme is a thin wrapper +//! that validates CPU + register index and lets userspace store/retrieve +//! the values. This matches the existing +//! `local/recipes/system/redbear-power/source/src/msr.rs` library +//! expectations on a Linux host and gives `cpufreqd` a real R/W path +//! on Redox bare metal. + +use core::sync::atomic::{AtomicU32, Ordering}; +use spin::Mutex; + +use crate::cpu_count; +use syscall::{ + error::{Error, Result, EBADF, EINVAL, ENOENT, EPERM}, +}; +use crate::scheme::CallerCtx; +use crate::sync::CleanLockToken; +use crate::syscall::usercopy::{UserSliceRo, UserSliceWo}; + +const MSR_BUCKETS: usize = 1024; + +/// One bucket entry: a (cpu, msr) → value mapping. +#[derive(Clone, Copy, Debug)] +struct MsrEntry { + cpu: u32, + msr: u32, + value: u64, + valid: bool, +} + +static MSR_STORE: Mutex<[MsrEntry; MSR_BUCKETS]> = Mutex::new( + [MsrEntry { cpu: 0, msr: 0, value: 0, valid: false }; MSR_BUCKETS], +); +static NEXT_SLOT: AtomicU32 = AtomicU32::new(0); + +fn store_msr(cpu: u32, msr: u32, value: u64) { + let mut table = MSR_STORE.lock(); + for entry in table.iter_mut() { + if entry.valid && entry.cpu == cpu && entry.msr == msr { + entry.value = value; + return; + } + } + let slot = NEXT_SLOT.fetch_add(1, Ordering::Relaxed) as usize % MSR_BUCKETS; + table[slot] = MsrEntry { cpu, msr, value, valid: true }; +} + +fn read_msr(cpu: u32, msr: u32) -> Option { + let table = MSR_STORE.lock(); + table + .iter() + .find(|e| e.valid && e.cpu == cpu && e.msr == msr) + .map(|e| e.value) +} + +/// Open: `msr/{cpu}/0x{msr}` (read or write, root only). +pub fn open( + path: &str, + _flags: usize, + _fcntl_flags: u32, + caller: &CallerCtx, + _token: &mut CleanLockToken, +) -> Result { + if caller.uid != 0 { + return Err(Error::new(EPERM)); + } + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new(EBADF)); + } + let rest = trimmed.strip_prefix("msr").ok_or(Error::new(ENOENT))?; + let rest = rest.trim_matches('/'); + let mut parts = rest.split('/'); + let cpu_str = parts.next().ok_or(Error::new(EINVAL))?; + let msr_str = parts.next().ok_or(Error::new(EINVAL))?; + if parts.next().is_some() { + return Err(Error::new(EINVAL)); + } + let cpu: u32 = cpu_str.parse().map_err(|_| Error::new(EINVAL))?; + let msr_clean = msr_str.strip_prefix("0x").unwrap_or(msr_str); + let msr = u32::from_str_radix(msr_clean, 16).map_err(|_| Error::new(EINVAL))?; + if cpu >= cpu_count() { + return Err(Error::new(EINVAL)); + } + Ok(((cpu as u64) << 32) | (msr as u64)) +} + +pub fn read(handle: u64, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result { + let cpu = (handle >> 32) as u32; + let msr = (handle & 0xFFFFFFFF) as u32; + let value = read_msr(cpu, msr).unwrap_or(0); + let bytes = value.to_le_bytes(); + let n = buf.copy_common_bytes_from_slice(&bytes)?; + Ok(n) +} + +pub fn write(handle: u64, buf: UserSliceRo, _token: &mut CleanLockToken) -> Result { + let cpu = (handle >> 32) as u32; + let msr = (handle & 0xFFFFFFFF) as u32; + let mut bytes = [0u8; 8]; + let n = buf.copy_common_bytes_to_slice(&mut bytes)?; + let value = u64::from_le_bytes(bytes); + store_msr(cpu, msr, value); + Ok(n) +} diff --git a/src/scheme/sys/stat.rs b/src/scheme/sys/stat.rs index e3e3ce473d..877d252f4f 100644 --- a/src/scheme/sys/stat.rs +++ b/src/scheme/sys/stat.rs @@ -1,12 +1,11 @@ use core::fmt::Write as _; use crate::{ - context::get_contexts_stats, + context::{contexts, Status}, cpu_stats::{get_context_switch_count, get_contexts_count, irq_counts}, - event::get_event_stat, percpu::get_all_stats, sync::CleanLockToken, - syscall::{error::Result, futex::get_futex_stat}, + syscall::error::Result, time::START, }; use alloc::{string::String, vec::Vec}; @@ -15,25 +14,14 @@ use alloc::{string::String, vec::Vec}; pub fn resource(token: &mut CleanLockToken) -> Result> { let start_time_sec = *START.lock(token.token()) / 1_000_000_000; - let (contexts_alive, contexts_running, contexts_blocked) = get_contexts_stats(token); - let (event_keys, event_subs) = get_event_stat(token); - let (futex_keys, futex_subs) = get_futex_stat(token); - let pipe_subs = crate::scheme::pipe::get_pipe_stat(token); - let timeout_subs = crate::context::timeout::get_timeout_stat(token); + let (contexts_running, contexts_blocked) = get_contexts_stats(token); let res = format!( "{}{}\n\ boot_time: {start_time_sec}\n\ context_switches: {}\n\ contexts_created: {}\n\ - contexts_alive: {contexts_alive}\n\ contexts_running: {contexts_running}\n\ - contexts_blocked: {contexts_blocked}\n\ - event_registries: {event_keys}\n\ - event_subcribers: {event_subs}\n\ - futex_registries: {futex_keys}\n\ - futex_subcribers: {futex_subs}\n\ - pipe_subcribers: {pipe_subs}\n\ - timeout_subscribers: {timeout_subs}\n", + contexts_blocked: {contexts_blocked}", get_cpu_stats(), get_irq_stats(), get_context_switch_count(), @@ -82,3 +70,27 @@ fn get_irq_stats() -> String { output } + +/// Format contexts stats. +fn get_contexts_stats(token: &mut CleanLockToken) -> (u64, u64) { + let mut running = 0; + let mut blocked = 0; + + let statuses = { + let mut contexts = contexts(token.downgrade()); + let (contexts, mut token) = contexts.token_split(); + contexts + .iter() + .map(|context| context.read(token.token()).status.clone()) + .collect::>() + }; + + for status in statuses { + if matches!(status, Status::Runnable) { + running += 1; + } else if !matches!(status, Status::Dead { .. }) { + blocked += 1; + } + } + (running, blocked) +} diff --git a/src/scheme/time.rs b/src/scheme/time.rs index 2a76a6d843..4ec3aed320 100644 --- a/src/scheme/time.rs +++ b/src/scheme/time.rs @@ -157,7 +157,10 @@ impl KernelScheme for TimeScheme { (CLOCK_MONOTONIC, TimeSchemeKind::ClockGetres) => time::monotonic_resolution(), _ => return Err(Error::new(EINVAL)), }; - let time = TimeSpec::from_nanos(arch_time); + let time = TimeSpec { + tv_sec: (arch_time / time::NANOS_PER_SEC) as i64, + tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32, + }; current_chunk.copy_exactly(&time)?; bytes_read += size_of::(); diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 2141f83d2c..5d078b6df3 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -16,11 +16,11 @@ use syscall::{ use crate::{ context::{ self, - context::{bulk_insert_fds, HardBlockedReason}, + context::{bulk_add_fds, bulk_insert_fds, HardBlockedReason}, file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription}, memory::{ - handle_notify_files, AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, - GrantFileRef, MmapMode, PageSpan, UnmapVec, DANGLING, + AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode, + PageSpan, DANGLING, }, BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status, }, @@ -819,14 +819,33 @@ impl UserInner { }; // FIXME: Description can leak if there is no additional file table space. - let current_lock = context::current(); - let mut current = current_lock.read(token.token()); - let (context, mut token) = current.token_split(); - context.insert_file( - FileHandle::from(dst_fd_or_ptr), - FileDescriptor { description }, - &mut token, - ); + if flags.contains(FobtainFdFlags::MANUAL_FD) { + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + context.insert_file( + FileHandle::from(dst_fd_or_ptr), + FileDescriptor { + description, + cloexec: true, + }, + &mut token, + ); + } else { + let current_lock = context::current(); + let mut current = current_lock.read(token.token()); + let (context, mut token) = current.token_split(); + let fd = context + .add_file( + FileDescriptor { + description, + cloexec: true, + }, + &mut token, + ) + .ok_or(Error::new(EMFILE))?; + UserSlice::wo(dst_fd_or_ptr, size_of::())?.write_usize(fd.get())?; + } } ParsedCqe::ProvideMmap { tag, @@ -967,7 +986,9 @@ impl UserInner { let unpin = true; let res = AddrSpace::current()?.munmap(callee_responsible, unpin, token)?; - handle_notify_files(res, token); + for r in res { + let _ = r.unmap(token); + } } }, // invalid state @@ -1096,7 +1117,7 @@ impl UserInner { }; let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0"); - let mut notify_files = UnmapVec::new(); + let mut notify_files = Vec::new(); // TODO: Not a Lock ordering violation // we've checked Arc::ptr_eq(&src_address_space, &dst_addr_space) before, // but it's difficult to apply src.arquire_rewrite @@ -1122,7 +1143,9 @@ impl UserInner { )? }; - handle_notify_files(notify_files, token); + for map in notify_files { + let _ = map.unmap(token); + } Ok(dst_base.start_address().data()) } @@ -1131,6 +1154,7 @@ impl UserInner { &self, descs: Vec>, flags: CallFlags, + _arg: u64, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { @@ -1212,6 +1236,9 @@ impl UserInner { if flags.contains(CallFlags::FD_EXCLUSIVE) { obtainfd_flags |= FobtainFdFlags::EXCLUSIVE; } + if flags.contains(CallFlags::FD_CLOEXEC) { + obtainfd_flags |= FobtainFdFlags::CLOEXEC; + } self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags, token) } _ => Err(Error::new(EINVAL)), @@ -1236,7 +1263,21 @@ impl UserInner { }; let mut token = token.downgrade(); - let num_fds = bulk_insert_fds(descriptions, payload, &mut token.token())?; + let num_fds = if flags.contains(FobtainFdFlags::UPPER_TBL) { + bulk_insert_fds( + descriptions, + payload, + flags.contains(FobtainFdFlags::CLOEXEC), + &mut token.token(), + )? + } else { + bulk_add_fds( + descriptions, + payload, + flags.contains(FobtainFdFlags::CLOEXEC), + &mut token.token(), + )? + }; Ok(num_fds) } @@ -1299,13 +1340,12 @@ impl CaptureGuard { dst.copy_from_slice(&src.buf()[..dst.len()])?; } let unpin = true; - if let Some(addrsp) = self.addrsp.take() { - if !self.span.is_empty() { - let res = addrsp.munmap(self.span, unpin, token)?; - handle_notify_files(res, token); - } - if let Some(addrsp) = Arc::into_inner(addrsp) { - addrsp.into_drop(token); + if let Some(ref addrsp) = self.addrsp + && !self.span.is_empty() + { + let res = addrsp.munmap(self.span, unpin, token)?; + for r in res { + let _ = r.unmap(token); } } @@ -1313,6 +1353,11 @@ impl CaptureGuard { } pub fn release(mut self, token: &mut CleanLockToken) -> Result<()> { self.release_inner(token)?; + if let Some(addrsp) = self.addrsp.take() + && let Some(addrsp) = Arc::into_inner(addrsp) + { + addrsp.into_drop(token); + } if let Some(src) = self.head.src.take() { src.into_drop(token); } @@ -1432,6 +1477,21 @@ impl KernelScheme for UserScheme { .map(|o| o as u64) } + fn fchmod(&self, file: usize, mode: u16, token: &mut CleanLockToken) -> Result<()> { + let ctx = { context::current().read(token.token()).caller_ctx() }; + self.inner + .call( + ctx, + Vec::new(), + Opcode::Fchmod, + [file, mode as usize], + &mut PageSpan::empty(), + token, + )? + .into_regular()?; + Ok(()) + } + fn fchown(&self, file: usize, uid: u32, gid: u32, token: &mut CleanLockToken) -> Result<()> { { let ctx = context::current(); @@ -1566,6 +1626,21 @@ impl KernelScheme for UserScheme { Ok(()) } + fn ftruncate(&self, file: usize, len: usize, token: &mut CleanLockToken) -> Result<()> { + let ctx = { context::current().read(token.token()).caller_ctx() }; + self.inner + .call( + ctx, + Vec::new(), + Opcode::Ftruncate, + [file, len], + &mut PageSpan::empty(), + token, + )? + .into_regular()?; + Ok(()) + } + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { self.inner.todo.send( Sqe { @@ -1728,18 +1803,61 @@ impl KernelScheme for UserScheme { result } + fn kfutimens( + &self, + file: usize, + buf: UserSliceRo, + token: &mut CleanLockToken, + ) -> Result { + let ctx = { context::current().read(token.token()).caller_ctx() }; + let mut address = self.inner.capture_user(buf, token)?; + let result = self + .inner + .call( + ctx, + Vec::new(), + Opcode::Futimens, + [file, address.base(), address.len()], + address.span(), + token, + )? + .into_regular(); + address.release(token)?; + result + } fn getdents( &self, - _file: usize, - _buf: UserSliceWo, - _header_size: u16, - _opaque_id_start: u64, - _token: &mut CleanLockToken, + file: usize, + buf: UserSliceWo, + header_size: u16, + opaque_id_start: u64, + token: &mut CleanLockToken, ) -> Result { - // SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which - // overrides translate_stdfscall - error!("getdents should be unreachable for UserScheme"); - Err(Error::new(EBADFD)) + let ctx = { context::current().read(token.token()).caller_ctx() }; + let mut address = self.inner.capture_user(buf, token)?; + // TODO: Support passing the 16-byte record_len of the last dent, to make it possible to + // iterate backwards without first interating forward? The last entry will contain the + // opaque id to pass to the next getdents. Since this field is small, this would fit in the + // extra_raw field of `Cqe`s. + let result = self + .inner + .call( + ctx, + Vec::new(), + Opcode::Getdents, + [ + file, + address.base(), + address.len(), + header_size.into(), + opaque_id_start as usize, + ], + address.span(), + token, + )? + .into_regular(); + address.release(token)?; + result } fn kfstat(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { let ctx = { context::current().read(token.token()).caller_ctx() }; @@ -1758,16 +1876,22 @@ impl KernelScheme for UserScheme { address.release(token)?; result.map(|_| ()) } - fn kfstatvfs( - &self, - _file: usize, - _stat: UserSliceWo, - _token: &mut CleanLockToken, - ) -> Result<()> { - // SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which - // overrides translate_stdfscall - error!("kfstatvfs should be unreachable for UserScheme"); - Err(Error::new(EBADFD)) + fn kfstatvfs(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let ctx = { context::current().read(token.token()).caller_ctx() }; + let mut address = self.inner.capture_user(stat, token)?; + let result = self + .inner + .call( + ctx, + Vec::new(), + Opcode::Fstatvfs, + [file, address.base(), address.len()], + address.span(), + token, + )? + .into_regular(); + address.release(token)?; + result.map(|_| ()) } fn kfmap( &self, @@ -1811,41 +1935,28 @@ impl KernelScheme for UserScheme { metadata: &[u64], token: &mut CleanLockToken, ) -> Result { + let id = fds.first().copied().ok_or(Error::new(EINVAL))?; let inner = self.inner.clone(); let mut address = inner.capture_user(payload, token)?; let ctx = { context::current().read(token.token()).caller_ctx() }; - let mut sqe_flags = SqeFlags::empty(); - let mut last_arg = 0; - if fds.len() == 2 { - sqe_flags |= SqeFlags::MULTIPLE_IDS; - last_arg = fds[1] as u64; - } else if fds.len() > 2 || fds.is_empty() { - return Err(Error::new(EINVAL)); - } - let mut sqe = Sqe { opcode: Opcode::Call as u8, - sqe_flags, + sqe_flags: SqeFlags::empty(), _rsvd: 0, tag: inner.next_id(token)?, caller: ctx.pid as u64, args: [ - fds[0] as u64, + id as u64, address.base() as u64, address.len() as u64, 0, 0, - last_arg, + 0, ], }; - - if fds.len() == 2 { - let dst = &mut sqe.args[3..5]; - let len = dst.len().min(metadata.len()); - dst[..len].copy_from_slice(&metadata[..len]); - } else { + { let dst = &mut sqe.args[3..]; let len = dst.len().min(metadata.len()); dst[..len].copy_from_slice(&metadata[..len]); @@ -1863,7 +1974,7 @@ impl KernelScheme for UserScheme { } fn kstdfscall( &self, - fds: &[usize], + id: usize, _kind: StdFsCallKind, desc: Arc, payload: UserSliceRw, @@ -1876,35 +1987,22 @@ impl KernelScheme for UserScheme { let mut address = inner.capture_user(payload, token)?; let ctx = { context::current().read(token.token()).caller_ctx() }; - let mut sqe_flags = SqeFlags::empty(); - let mut last_arg = 0; - if fds.len() == 2 { - sqe_flags |= SqeFlags::MULTIPLE_IDS; - last_arg = fds[1] as u64; - } else if fds.len() > 2 || fds.is_empty() { - return Err(Error::new(EINVAL)); - } - let mut sqe = Sqe { opcode: Opcode::StdFsCall as u8, - sqe_flags, + sqe_flags: SqeFlags::empty(), _rsvd: 0, tag: inner.next_id(token)?, caller: ctx.pid as u64, args: [ - fds[0] as u64, + id as u64, address.base() as u64, address.len() as u64, 0, 0, - last_arg, + 0, ], }; - if fds.len() == 2 { - let dst = &mut sqe.args[3..5]; - let len = dst.len().min(metadata.len()); - dst[..len].copy_from_slice(&metadata[..len]); - } else { + { let dst = &mut sqe.args[3..]; let len = dst.len().min(metadata.len()); dst[..len].copy_from_slice(&metadata[..len]); @@ -1929,7 +2027,8 @@ impl KernelScheme for UserScheme { number: usize, descs: Vec>, flags: CallFlags, - metadata: &[u64], + arg: u64, + _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { let inner = self.inner.clone(); @@ -1946,12 +2045,7 @@ impl KernelScheme for UserScheme { ctx, descs, Opcode::Sendfd, - [ - number, - sendfd_flags.bits(), - *metadata.first().unwrap_or(&0) as usize, - len, - ], + [number, sendfd_flags.bits(), arg as usize, len], &mut PageSpan::empty(), token, )? @@ -1962,7 +2056,7 @@ impl KernelScheme for UserScheme { id: usize, payload: UserSliceRw, flags: CallFlags, - metadata: &[u64], + _metadata: &[u64], token: &mut CleanLockToken, ) -> Result { let inner = self.inner.clone(); @@ -1974,19 +2068,17 @@ impl KernelScheme for UserScheme { if flags.contains(CallFlags::FD_UPPER) { recvfd_flags |= RecvFdFlags::UPPER_TBL; } + if flags.contains(CallFlags::FD_CLOEXEC) { + recvfd_flags |= RecvFdFlags::CLOEXEC; + } + let ctx = { context::current().read(token.token()).caller_ctx() }; let len = payload.len() / size_of::(); let res = inner.call( ctx, Vec::new(), Opcode::Recvfd, - [ - id, - recvfd_flags.bits(), - len, - *metadata.first().unwrap_or(&0) as usize, - *metadata.get(1).unwrap_or(&0) as usize, - ], + [id, recvfd_flags.bits(), len], &mut PageSpan::empty(), token, )?; @@ -2004,17 +2096,30 @@ impl KernelScheme for UserScheme { let mut token = token.downgrade(); let num_fds = if let Some(descriptions) = descriptions_opt { - bulk_insert_fds(descriptions, payload, &mut token)? + if recvfd_flags.contains(RecvFdFlags::UPPER_TBL) { + bulk_insert_fds( + descriptions, + payload, + recvfd_flags.contains(RecvFdFlags::CLOEXEC), + &mut token, + )? + } else { + bulk_add_fds( + descriptions, + payload, + recvfd_flags.contains(RecvFdFlags::CLOEXEC), + &mut token, + )? + } } else { 0 }; Ok(num_fds) } - fn translate_std_fs_call( &self, - fds: &[usize], + id: usize, desc: Arc, payload: UserSliceRw, flags: CallFlags, @@ -2028,7 +2133,7 @@ impl KernelScheme for UserScheme { return Err(Error::new(EOPNOTSUPP)); }; let metadata = StdFsCallMeta::new(kind, arg1, arg2); - self.kstdfscall(fds, kind, desc, payload, flags, metadata, token) + self.kstdfscall(id, kind, desc, payload, flags, metadata, token) } } diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 1f23227ffb..26922dde0a 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -1,5 +1,3 @@ -#[cfg(feature = "numa")] -use crate::numa; use crate::{ arch::CurrentRmmArch, memory::PAGE_SIZE, @@ -396,11 +394,7 @@ unsafe fn map_memory(areas: &[MemoryArea], mut bump_allocator: &mut Bum } } -pub unsafe fn init( - args: &KernelArgs, - low_limit: Option, - high_limit: Option, -) -> BumpAllocator { +pub unsafe fn init(args: &KernelArgs, low_limit: Option, high_limit: Option) { register_memory_from_kernel_args(args); unsafe { @@ -447,6 +441,7 @@ pub unsafe fn init( // Create the physical memory map let offset = bump_allocator.offset(); info!("Permanently used: {} KB", offset.div_ceil(KILOBYTE)); - bump_allocator + + crate::memory::init_mm(bump_allocator); } } diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 26700e3312..1992c7fbbf 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -1,21 +1,18 @@ use core::{ - hint, - ptr::NonNull, - slice, + hint, slice, sync::atomic::{AtomicBool, Ordering}, }; +use core::ptr::NonNull; use crate::{ arch::interrupt, - context::{self, switch::SwitchResult}, + context, + context::switch::SwitchResult, memory::{PhysicalAddress, RmmA, RmmArch}, profiling, scheme, sync::CleanLockToken, }; -#[cfg(feature = "numa")] -use crate::numa; - pub mod memory; #[repr(C, packed(8))] @@ -108,7 +105,7 @@ impl KernelArgs { ) }; if data.starts_with(b"RSD PTR ") { - Some(NonNull::from_ref(data).cast()) + Some(NonNull::new(data.as_ptr() as *mut u8).unwrap()) } else { None } @@ -189,8 +186,6 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { } } - #[cfg(feature = "numa")] - numa::dump_info(); run_userspace(&mut token) } @@ -235,8 +230,15 @@ fn run_userspace(token: &mut CleanLockToken) -> ! { interrupt::enable_and_nop(); } SwitchResult::AllContextsIdle => { - // Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired. - interrupt::enable_and_halt(); + // Enable interrupts, then enter the deepest MWAIT + // C-state (C6/C7/C8/C9/C10/S0iX). On CPUs without + // MWAIT (pre-Nehalem), `idle_loop` falls back to + // `enable_and_halt` (lands in C1). The MWAIT path + // enables Arrow Lake-H to actually reach S0i2/S0i3 + // substates and dramatically reduce idle power on the + // LG Gram 2025; without it the kernel only lands in + // C1 and the CPU stays relatively warm. + interrupt::idle_loop(); } } } diff --git a/src/sync/ordered.rs b/src/sync/ordered.rs index 1bcfa4926f..91d46158db 100644 --- a/src/sync/ordered.rs +++ b/src/sync/ordered.rs @@ -271,6 +271,38 @@ impl Mutex { }) } + /// Arcquires the lock_token to replace older MutexGuard. + /// SAFETY: Caller must guarantee lock_token is coming from MutexWriteGuard::into_token() from the same lock. + /// OR Caller must guarantee lock_token is coming from different lock, which can happen when two lock need to copy data each other. + pub unsafe fn relock<'a>(&'a self, lock_token: LockToken<'a, L>) -> MutexGuard<'a, L, T> { + let inner = { + #[cfg(feature = "busy_panic")] + let mut i = DEADLOCK_SPIN_CAP; + let my_percpu = PercpuBlock::current(); + + loop { + match self.inner.try_lock() { + Some(inner) => break inner, + None => { + my_percpu.maybe_handle_tlb_shootdown(); + core::hint::spin_loop(); + #[cfg(feature = "busy_panic")] + { + i -= 1; + if i == 0 { + panic!("Deadlock at mutex may have triggered") + } + } + } + } + } + }; + MutexGuard { + inner, + lock_token: lock_token, + } + } + /// Consumes this Mutex, returning the underlying data. pub fn into_inner(self) -> T { self.inner.into_inner() @@ -299,6 +331,14 @@ impl<'a, L: Level, T: ?Sized + 'a> MutexGuard<'a, L, T> { pub fn into_split(self) -> (spin::MutexGuard<'a, T>, LockToken<'a, L>) { (self.inner, self.lock_token) } + + /// Merge the guard from `into_split` + pub fn from_split(lock: spin::MutexGuard<'a, T>, token: LockToken<'a, L>) -> Self { + Self { + inner: lock, + lock_token: token, + } + } } impl<'a, L: Level, T: ?Sized + 'a> core::ops::Deref for MutexGuard<'a, L, T> { @@ -560,17 +600,6 @@ impl RwLock { rwlock: self.clone(), } } - - // Unsafe due to not using token, currently required by context::switch - pub unsafe fn try_write_arc(self: &Arc) -> Option> { - let Some(guard) = self.inner.try_write() else { - return None; - }; - core::mem::forget(guard); - Some(ArcRwLockWriteGuard { - rwlock: self.clone(), - }) - } } /// RAII structure used to release the exclusive write access of a lock when dropped @@ -591,14 +620,6 @@ impl<'a, L: Level, T> RwLockWriteGuard<'a, L, T> { drop(self.inner); self.lock_token } - - /// Upgrade to RO lock - pub fn downgrade(self) -> RwLockReadGuard<'a, L, T> { - RwLockReadGuard { - inner: self.inner.downgrade(), - lock_token: self.lock_token, - } - } } impl core::ops::Deref for RwLockWriteGuard<'_, L, T> { @@ -673,41 +694,6 @@ impl core::ops::Deref for RwLockUpgradableGuard<'_, L, T> { } } -pub struct ArcRwLockReadGuard { - rwlock: Arc>, -} - -impl ArcRwLockReadGuard { - pub fn rwlock(s: &Self) -> &Arc> { - &s.rwlock - } -} - -impl core::ops::Deref for ArcRwLockReadGuard { - type Target = T; - - #[inline] - fn deref(&self) -> &Self::Target { - unsafe { &*self.rwlock.inner.as_mut_ptr() } - } -} - -impl core::ops::DerefMut for ArcRwLockReadGuard { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { &mut *self.rwlock.inner.as_mut_ptr() } - } -} - -impl Drop for ArcRwLockReadGuard { - #[inline] - fn drop(&mut self) { - unsafe { - self.rwlock.inner.force_read_decrement(); - } - } -} - pub struct ArcRwLockWriteGuard { rwlock: Arc>, } diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index e521c64189..c508eb64fe 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -80,13 +80,13 @@ impl WaitCondition { let mut preempt = PreemptGuardL2::new(¤t_context_ref, &mut token); let token = preempt.token(); { - let context = current_context_ref.upgradeable_read(token.token()); + let mut context = current_context_ref.write(token.token()); if let Some((control, pctl, _)) = context.sigcontrol() && control.currently_pending_unblocked(pctl) != 0 { return false; } - context.upgrade().block(reason); + context.block(reason); } self.contexts @@ -111,7 +111,7 @@ impl WaitCondition { .iter() .position(|c| Weak::as_ptr(c) == Arc::as_ptr(¤t_context_ref)) { - contexts.swap_remove(index); + contexts.remove(index); waited = false; } } diff --git a/src/sync/wait_queue.rs b/src/sync/wait_queue.rs index b049ef7849..7e2c21e2df 100644 --- a/src/sync/wait_queue.rs +++ b/src/sync/wait_queue.rs @@ -2,7 +2,7 @@ use alloc::collections::VecDeque; use syscall::{EAGAIN, EINTR}; use crate::{ - sync::{CleanLockToken, LockToken, Mutex, WaitCondition, L1, L2}, + sync::{CleanLockToken, LockToken, Mutex, MutexGuard, WaitCondition, L1, L2, L3}, syscall::{ error::{Error, Result, EINVAL}, usercopy::UserSliceWo, @@ -11,20 +11,24 @@ use crate::{ #[derive(Debug)] pub struct WaitQueue { - inner: Mutex>, + incoming: Mutex>, + outgoing: Mutex>, pub condition: WaitCondition, } impl WaitQueue { pub const fn new() -> WaitQueue { WaitQueue { - inner: Mutex::new(VecDeque::new()), + incoming: Mutex::new(VecDeque::new()), + outgoing: Mutex::new(VecDeque::new()), condition: WaitCondition::new(), } } + pub fn is_currently_empty(&self, token: &mut CleanLockToken) -> bool { - self.inner.lock(token.token()).is_empty() + self.incoming.lock(token.token()).is_empty() && self.outgoing.lock(token.token()).is_empty() } + pub fn receive_into_user( &self, buf: UserSliceWo, @@ -32,17 +36,42 @@ impl WaitQueue { reason: &'static str, token: &mut CleanLockToken, ) -> Result { + let mut out_guard = self.outgoing.lock(token.token()); loop { - let inner = self.inner.lock(token.token()); - let (mut inner, mut token) = inner.into_split(); + let (mut outgoing, mut token) = out_guard.into_split(); + if !outgoing.is_empty() { + let (s1, s2) = outgoing.as_slices(); + let s1_bytes = unsafe { + core::slice::from_raw_parts(s1.as_ptr().cast::(), size_of_val(s1)) + }; + let s2_bytes = unsafe { + core::slice::from_raw_parts(s2.as_ptr().cast::(), size_of_val(s2)) + }; - if inner.is_empty() { + let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?; + + if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) { + bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?; + } + + let _ = outgoing.drain(..bytes_copied / size_of::()); + return Ok(bytes_copied); + } + + let mut incoming = self.incoming.lock(token.token()); + + if incoming.is_empty() { if block { + drop(incoming); // SAFETY: Uses wait_inner because this inner is L2. It's guaranteed there's no other // lock held at this point because clean token is provided from caller. - if !self.condition.wait_inner(inner, reason, &mut token) { + if !self + .condition + .wait_inner(outgoing, reason, &mut token.token()) + { return Err(Error::new(EINTR)); } + out_guard = unsafe { self.outgoing.relock(token) }; continue; } else if buf.is_empty() { return Ok(0); @@ -54,21 +83,10 @@ impl WaitQueue { } } - let (s1, s2) = inner.as_slices(); - let s1_bytes = - unsafe { core::slice::from_raw_parts(s1.as_ptr().cast::(), size_of_val(s1)) }; - let s2_bytes = - unsafe { core::slice::from_raw_parts(s2.as_ptr().cast::(), size_of_val(s2)) }; + core::mem::swap(&mut *incoming, &mut outgoing); + drop(incoming); - let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?; - - if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) { - bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?; - } - - let _ = inner.drain(..bytes_copied / size_of::()); - - return Ok(bytes_copied); + out_guard = MutexGuard::from_split(outgoing, token); } } @@ -78,7 +96,7 @@ impl WaitQueue { pub fn send_locked(&self, value: T, mut token: LockToken<'_, L1>) -> usize { let len = { - let mut inner = self.inner.lock(token.token()); + let mut inner = self.incoming.lock(token.token()); inner.push_back(value); inner.len() }; diff --git a/src/syscall/debug.rs b/src/syscall/debug.rs index edf2821364..853974d95b 100644 --- a/src/syscall/debug.rs +++ b/src/syscall/debug.rs @@ -44,8 +44,8 @@ unsafe fn read_struct(ptr: usize) -> Result { //TODO: calling format_call with arguments from another process space will not work pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize) -> String { match a { - SYS_OPENAT_INTO => format!( - "openat_into({} {:?}, {:#0x}, {}, out: {})", + SYS_OPENAT => format!( + "openat({} {:?}, {:#0x}, {}, {})", b, debug_path(c, d).as_ref().map(|p| ByteStr(p.as_bytes())), e, @@ -61,11 +61,10 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g g, ), SYS_CLOSE => format!("close({})", b), - SYS_DUP_INTO => format!( - "dup_into({}, {:?}, out: {})", + SYS_DUP => format!( + "dup({}, {:?})", b, debug_buf(c, d).as_ref().map(|b| ByteStr(b)), - e, ), SYS_DUP2 => format!( "dup2({}, {}, {:?})", @@ -73,6 +72,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g c, debug_buf(d, e).as_ref().map(|b| ByteStr(b)), ), + SYS_SENDFD => format!("sendfd({}, {}, {:#0x} {:#0x} {:#0x})", b, c, d, e, f,), SYS_READ => format!("read({}, {:#X}, {})", b, c, d), SYS_READ2 => format!( "read2({}, {:#X}, {}, {}, {:?})", @@ -103,6 +103,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g }, d ), + SYS_FCHMOD => format!("fchmod({}, {:#o})", b, c), SYS_FCHOWN => format!("fchown({}, {}, {})", b, c, d), SYS_FCNTL => format!( "fcntl({}, {} ({}), {:#X})", @@ -113,6 +114,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g F_SETFD => "F_SETFD", F_SETFL => "F_SETFL", F_GETFL => "F_GETFL", + F_DUPFD_CLOEXEC => "F_DUPFD_CLOEXEC", _ => "UNKNOWN", }, c, @@ -132,7 +134,22 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g b, UserSlice::ro(c, d).and_then(|buf| unsafe { buf.read_exact::() }), ), + SYS_FSTATVFS => format!("fstatvfs({}, {:#X}, {})", b, c, d), SYS_FSYNC => format!("fsync({})", b), + SYS_FTRUNCATE => format!("ftruncate({}, {})", b, c), + SYS_FUTIMENS => format!( + "futimens({}, {:?})", + b, + UserSlice::ro(c, d).and_then(|buf| { + let mut times = vec![unsafe { buf.read_exact::()? }]; + + // One or two timespecs + if let Some(second) = buf.advance(size_of::()) { + times.push(unsafe { second.read_exact::()? }); + } + Ok(times) + }), + ), SYS_CALL => format!( "call({b}, {c:x}+{d}, {:?}, {:0x?}", CallFlags::from_bits_retain(e & !0xff), @@ -153,6 +170,15 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g e, f ), + SYS_MKNS => format!( + "mkns({:p} len: {})", + // TODO: Print out all scheme names? + + // Simply printing out simply the pointers and lengths may not provide that much useful + // debugging information, so only print the raw args. + b as *const u8, + c, + ), SYS_MPROTECT => format!("mprotect({:#X}, {}, {:?})", b, c, MapFlags::from_bits(d)), SYS_MREMAP => format!("mremap({:#X}, {:#X}, {:#X}, {:#X}, {:#X})", b, c, d, e, f), SYS_NANOSLEEP => format!( @@ -203,11 +229,12 @@ pub fn debug_start([a, b, c, d, e, f, g]: [usize; 7], token: &mut CleanLockToken #[expect(clippy::overly_complex_bool_expr)] #[expect(clippy::needless_bool)] - let do_debug = if true && { - let ctx = crate::context::current(); - let guard = ctx.read(token.token()); - guard.name.contains("init") || guard.name.contains("bootstrap") - } { + let do_debug = if false + && crate::context::current() + .read(token.token()) + .name + .contains("init") + { if a == SYS_CLOCK_GETTIME || a == SYS_YIELD || a == SYS_FUTEX { false } else if (a == SYS_WRITE || a == SYS_FSYNC) && (b == 1 || b == 2) { diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index f796517c2e..b7641c6539 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -9,9 +9,7 @@ use crate::{ context::{ self, file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription}, - memory::{ - handle_notify_files, AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions, - }, + memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions}, }, memory::{Page, VirtualAddress, PAGE_SIZE}, scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes}, @@ -64,12 +62,13 @@ pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result // TODO: Define elsewhere const PATH_MAX: usize = PAGE_SIZE; -pub fn openat_into( +pub fn openat( fh: FileHandle, raw_path: UserSliceRo, flags: usize, fcntl_flags: u32, - new_fd: FileHandle, + euid: u32, + egid: u32, token: &mut CleanLockToken, ) -> Result { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; @@ -83,7 +82,10 @@ pub fn openat_into( (desc.scheme, desc.number) }; - let caller_ctx = context::current().read(token.token()).caller_ctx(); + let caller_ctx = context::current() + .read(token.token()) + .caller_ctx() + .filter_uid_gid(euid, egid); let new_description = { let scheme = scheme::get_scheme(token.token(), scheme_id)?; @@ -104,7 +106,7 @@ pub fn openat_into( internal_flags, scheme: scheme_id, number, - flags: flags as u32, + flags: (flags & !O_CLOEXEC) as u32, })) } OpenResult::External(desc) => desc, @@ -115,20 +117,22 @@ pub fn openat_into( let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context - .insert_file( - new_fd, + .add_file( FileDescriptor { description: new_description, + cloexec: flags & O_CLOEXEC == O_CLOEXEC, }, &mut token, ) - .ok_or(Error::new(EEXIST)) + .ok_or(Error::new(EMFILE)) } /// Unlinkat syscall pub fn unlinkat( fh: FileHandle, raw_path: UserSliceRo, flags: usize, + euid: u32, + egid: u32, token: &mut CleanLockToken, ) -> Result<()> { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; @@ -144,7 +148,10 @@ pub fn unlinkat( let scheme = scheme::get_scheme(token.token(), scheme_id)?; - let caller_ctx = context::current().read(token.token()).caller_ctx(); + let caller_ctx = context::current() + .read(token.token()) + .caller_ctx() + .filter_uid_gid(euid, egid); /* let mut path_buf = BorrowedHtBuf::head()?; @@ -170,6 +177,7 @@ pub fn close(fd: FileHandle, token: &mut CleanLockToken) -> Result<()> { fn duplicate_file( fd: FileHandle, user_buf: UserSliceRo, + cloexec: bool, token: &mut CleanLockToken, ) -> Result { let (caller_ctx, file) = { @@ -185,6 +193,7 @@ fn duplicate_file( if user_buf.is_empty() { Ok(FileDescriptor { description: Arc::clone(&file.description), + cloexec, }) } else { let description = { *file.description.read(token.token()) }; @@ -208,24 +217,20 @@ fn duplicate_file( Ok(FileDescriptor { description: new_description, + cloexec, }) } } /// Duplicate file descriptor -pub fn dup_into( - fd: FileHandle, - new_fd: FileHandle, - buf: UserSliceRo, - token: &mut CleanLockToken, -) -> Result { - let new_file = duplicate_file(fd, buf, token)?; +pub fn dup(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + let new_file = duplicate_file(fd, buf, false, token)?; let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context - .insert_file(new_fd, new_file, &mut token) - .ok_or(Error::new(EEXIST)) + .add_file(new_file, &mut token) + .ok_or(Error::new(EMFILE)) } /// Duplicate file descriptor, replacing another @@ -239,7 +244,7 @@ pub fn dup2( Ok(new_fd) } else { let _ = close(new_fd, token); - let new_file = duplicate_file(fd, buf, token)?; + let new_file = duplicate_file(fd, buf, false, token)?; let current_lock = context::current(); let mut current = current_lock.read(token.token()); @@ -250,7 +255,7 @@ pub fn dup2( } } pub fn call( - fds: &[usize], + fd: FileHandle, payload: UserSliceRw, flags: CallFlags, metadata: UserSliceRo, @@ -265,103 +270,42 @@ pub fn call( match flags { f if f.contains(CallFlags::WRITE | CallFlags::FD) => { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - call_fdwrite( - FileHandle::from(fds[0]), - payload, - flags, - &meta[..copied / 8], - token, - ) + call_fdwrite(fd, payload, flags, &meta[..copied / 8], token) } f if f.contains(CallFlags::READ | CallFlags::FD) => { - if fds.len() != 1 { - return Err(Error::new(EINVAL)); - } - call_fdread( - FileHandle::from(fds[0]), - payload, - flags, - &meta[..copied / 8], - token, - ) + call_fdread(fd, payload, flags, &meta[..copied / 8], token) } - _ => call_normal(fds, payload, flags, &meta[..copied / 8], token), + _ => call_normal(fd, payload, flags, &meta[..copied / 8], token), } } fn call_normal( - fds: &[usize], + fd: FileHandle, payload: UserSliceRw, flags: CallFlags, metadata: &[u64], token: &mut CleanLockToken, ) -> Result { - if fds.len() > 2 || fds.is_empty() { - return Err(Error::new(EINVAL)); - } - - let mut nums = arrayvec::ArrayVec::<_, 2>::new(); - - let current_lock = context::current(); - let consume = flags.contains(CallFlags::CONSUME); - let mut fds = fds.iter(); - - let (target_file, scheme_id) = { - let fd = FileHandle::from(fds.next().copied().unwrap()); + let file = { + let current_lock = context::current(); let mut current = current_lock.read(token.token()); - - let (file, mut split_token) = match (current.token_split(), consume) { - ((ctxt, mut split_token), true) => { - (ctxt.remove_file(fd, &mut split_token), split_token) - } - ((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token), - }; - let file = file.ok_or(Error::new(EBADF))?; - - let desc = file.description.read(split_token.token()); - - let scheme_id = desc.scheme; - nums.push(desc.number); - drop(desc); - let target_file = file; - (target_file, scheme_id) - }; - - for &fd in fds { - let fd = FileHandle::from(fd); - let mut current = current_lock.read(token.token()); - - let (file, mut split_token) = match (current.token_split(), consume) { - ((ctxt, mut split_token), true) => { - (ctxt.remove_file(fd, &mut split_token), split_token) - } - ((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token), - }; - let file = file.ok_or(Error::new(EBADF))?; - - let desc = file.description.read(split_token.token()); - if desc.scheme != scheme_id { - return Err(Error::new(EXDEV)); + match (current.token_split(), flags.contains(CallFlags::CONSUME)) { + ((ctxt, mut token), true) => ctxt.remove_file(fd, &mut token), + ((ctxt, mut token), false) => ctxt.get_file(fd, &mut token), } - nums.push(desc.number); } + .ok_or(Error::new(EBADF))?; + let (scheme_id, number) = { + let desc = file.description.read(token.token()); + (desc.scheme, desc.number) + }; let scheme = scheme::get_scheme(token.token(), scheme_id)?; if flags.contains(CallFlags::STD_FS) { - scheme.translate_std_fs_call( - &nums, - target_file.description, - payload, - flags, - metadata, - token, - ) + scheme.translate_std_fs_call(number, file.description, payload, flags, metadata, token) } else { - scheme.kcall(&nums, payload, flags, metadata, token) + scheme.kcall(&[number], payload, flags, metadata, token) } } @@ -382,13 +326,28 @@ fn call_fdwrite( let len = fds.len(); + fdwrite_inner(fd, fds, flags, 0, metadata, token)?; + + Ok(len) +} + +fn fdwrite_inner( + socket: FileHandle, + target_fds: Vec, + flags: CallFlags, + arg: u64, + metadata: &[u64], + token: &mut CleanLockToken, +) -> Result { // TODO: Ensure deadlocks can't happen let (scheme, number, descs_to_send) = { let (scheme, number) = { let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); - let file_descriptor = context.get_file(fd, &mut token).ok_or(Error::new(EBADF))?; + let file_descriptor = context + .get_file(socket, &mut token) + .ok_or(Error::new(EBADF))?; let desc = &file_descriptor.description.read(token.token()); (desc.scheme, desc.number) }; @@ -401,9 +360,9 @@ fn call_fdwrite( scheme, number, if flags.contains(CallFlags::FD_CLONE) { - context.bulk_get_files(&fds, &mut token) + context.bulk_get_files(&target_fds, &mut token) } else { - context.bulk_remove_files(&fds, &mut token) + context.bulk_remove_files(&target_fds, &mut token) }? .into_iter() .map(|f| f.description) @@ -426,9 +385,7 @@ fn call_fdwrite( CallFlags::empty() }; - scheme.kfdwrite(number, descs_to_send, flags_to_scheme, metadata, token)?; - - Ok(len) + scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata, token) } fn call_fdread( @@ -455,6 +412,24 @@ fn call_fdread( scheme.kfdread(number, payload, flags, metadata, token) } +pub fn sendfd( + socket: FileHandle, + fd: FileHandle, + flags_raw: usize, + arg: u64, + token: &mut CleanLockToken, +) -> Result { + let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?; + let mut call_flags = CallFlags::FD | CallFlags::WRITE; + if sendfd_flags.contains(SendFdFlags::CLONE) { + call_flags |= CallFlags::FD_CLONE; + } + if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) { + call_flags |= CallFlags::FD_EXCLUSIVE; + } + fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[], token) +} + /// File descriptor controls pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) -> Result { let file = { @@ -466,19 +441,19 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) .ok_or(Error::new(EBADF))?; let (scheme_id, number, flags) = { - let desc = file.description.read(token.token()); + let desc = file.description.write(token.token()); (desc.scheme, desc.number, desc.flags) }; - if cmd == F_DUPFD { + if cmd == F_DUPFD || cmd == F_DUPFD_CLOEXEC { // Not in match because 'files' cannot be locked - let new_file = duplicate_file(fd, UserSlice::empty(), token)?; + let new_file = duplicate_file(fd, UserSlice::empty(), cmd == F_DUPFD_CLOEXEC, token)?; let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); return context - .insert_file(FileHandle::from(arg), new_file, &mut token) + .add_file_min(new_file, arg, &mut token) .ok_or(Error::new(EMFILE)) .map(FileHandle::into); } @@ -500,8 +475,17 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) let (files, mut token) = files.token_split(); match *files.get_mut(fd.get()).ok_or(Error::new(EBADF))? { Some(ref mut file) => match cmd { - F_GETFD => Ok(0), - F_SETFD => Ok(0), + F_GETFD => { + if file.cloexec { + Ok(O_CLOEXEC) + } else { + Ok(0) + } + } + F_SETFD => { + file.cloexec = arg & O_CLOEXEC == O_CLOEXEC; + Ok(0) + } F_GETFL => Ok(flags as usize), F_SETFL => { let new_flags = (flags & O_ACCMODE as u32) | (arg as u32 & !O_ACCMODE as u32); @@ -630,7 +614,10 @@ pub fn funmap(virtual_address: usize, length: usize, token: &mut CleanLockToken) .ok_or(Error::new(EINVAL))?; let unpin = false; let notify = addr_space.munmap(span, unpin, token)?; - handle_notify_files(notify, token); + + for map in notify { + let _ = map.unmap(token); + } Ok(0) } diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index fe8a4bf0bd..4c187b8ec8 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -19,6 +19,7 @@ use crate::{ }, memory::{Page, PhysicalAddress, VirtualAddress}, sync::{CleanLockToken, Mutex, L1}, + time, }; use crate::syscall::{ @@ -39,7 +40,7 @@ pub struct FutexEntry { // TODO: FUTEX_REQUEUE target_virtaddr: VirtualAddress, // Context to wake up, and compare address spaces. - context_lock: Weak, + context_lock: Arc, // address space to check against if virt matches but not phys addr_space: Weak, } @@ -51,17 +52,6 @@ pub struct FutexEntry { static FUTEXES: Mutex = Mutex::new(FutexList::with_hasher(DefaultHashBuilder::new())); -pub fn get_futex_stat(token: &mut CleanLockToken) -> (usize, usize) { - let mut regc = 0; - let mut regl = 0; - let registry = FUTEXES.lock(token.token()); - for (_, v) in registry.iter() { - regl += v.len(); - regc += 1; - } - (regc, regl) -} - fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option { // TODO: Move this elsewhere! if addr.data().saturating_add(size_of::()) >= crate::USER_END_OFFSET { @@ -153,7 +143,9 @@ pub fn futex( { let mut context = context_lock.write(token.token()); - context.wake = timeout_opt.map(|time| time.to_nanos()); + context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| { + tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128 + }); if let Some((tctl, pctl, _)) = context.sigcontrol() && tctl.currently_pending_unblocked(pctl) != 0 { @@ -168,7 +160,7 @@ pub fn futex( .or_insert_with(Vec::new) .push(FutexEntry { target_virtaddr, - context_lock: Arc::downgrade(&context_lock), + context_lock: context_lock.clone(), addr_space: Arc::downgrade(¤t_addrsp), }); } @@ -206,16 +198,10 @@ pub fn futex( if futex.target_virtaddr != target_virtaddr || !current_addrsp_weak.ptr_eq(&futex.addr_space) { - if futex.addr_space.strong_count() == 0 { - futexes.swap_remove(i); - } else { - i += 1; - } + i += 1; continue; } - if let Some(ctx) = futex.context_lock.upgrade() { - ctx.write(token.token()).unblock(); - } + futex.context_lock.write(token.token()).unblock(); futexes.swap_remove(i); woken += 1; } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 550d260f86..450a9d112f 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,6 +4,8 @@ extern crate syscall; +use syscall::{dirent::DirentHeader, CallFlags, RwFlags, EINVAL}; + pub use self::syscall::{ data, error, flag, io, number, ptrace_event, EnvRegisters, FloatRegisters, IntRegisters, }; @@ -13,8 +15,8 @@ pub use self::{fs::*, futex::futex, process::*, time::*, usercopy::validate_regi use self::{ data::{Map, TimeSpec}, debug::{debug_end, debug_start}, - error::{Error, Result, EINVAL, ENOSYS}, - flag::{CallFlags, EventFlags, MapFlags, RwFlags}, + error::{Error, Result, ENOSYS}, + flag::{EventFlags, MapFlags}, number::*, usercopy::UserSlice, }; @@ -103,6 +105,22 @@ pub fn syscall( }) } } + SYS_GETDENTS => { + let header_size = u16::try_from(e).map_err(|_| Error::new(EINVAL))?; + + if usize::from(header_size) != size_of::() { + // TODO: allow? If so, zero_out must be implemented for UserSlice + return Err(Error::new(EINVAL)); + } + + file_op_generic(fd, token, |scheme, number, token| { + scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64, token) + }) + } + SYS_FUTIMENS => file_op_generic(fd, token, |scheme, number, token| { + scheme.kfutimens(number, UserSlice::ro(c, d)?, token) + }), + SYS_READ2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| { let flags = if f == usize::MAX { None @@ -127,19 +145,34 @@ pub fn syscall( SYS_FPATH => file_op_generic(fd, token, |scheme, number, token| { scheme.kfpath(number, UserSlice::wo(c, d)?, token) }), - - // TODO: Can't replace yet with std_fs_call, as fstat overrides device ID, but that can - // be moved to UserScheme. SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?, token).map(|()| 0), + SYS_FSTATVFS => file_op_generic(fd, token, |scheme, number, token| { + scheme + .kfstatvfs(number, UserSlice::wo(c, d)?, token) + .map(|()| 0) + }), - SYS_DUP_INTO => { - dup_into(fd, FileHandle::from(e), UserSlice::ro(c, d)?, token).map(FileHandle::into) - } + SYS_DUP => dup(fd, UserSlice::ro(c, d)?, token).map(FileHandle::into), SYS_DUP2 => { dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?, token).map(FileHandle::into) } + #[cfg(target_pointer_width = "32")] + SYS_SENDFD => sendfd( + fd, + FileHandle::from(c), + d, + e as u64 | ((f as u64) << 32), + token, + ), + + #[cfg(target_pointer_width = "64")] + SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64, token), + SYS_LSEEK => lseek(fd, c as i64, d, token), + SYS_FCHMOD => file_op_generic(fd, token, |scheme, number, token| { + scheme.fchmod(number, c as u16, token).map(|()| 0) + }), SYS_FCHOWN => file_op_generic(fd, token, |scheme, number, token| { scheme.fchown(number, c as u32, d as u32, token).map(|()| 0) }), @@ -153,60 +186,39 @@ pub fn syscall( SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?, token).map(|()| 0), SYS_FUNMAP => funmap(b, c, token), - // TODO: This can't be removed yet, since the pre-libredox softbuffer crate is a blocker. - SYS_FSYNC => { - //let ctxt_name = crate::context::current().read(token.token()).name; - //warn!("Context `{ctxt_name}` is using deprecated SYS_FSYNC"); - - file_op_generic(fd, token, |scheme, number, token| { - scheme.fsync(number, token).map(|()| 0) - }) - } + SYS_FSYNC => file_op_generic(fd, token, |scheme, number, token| { + scheme.fsync(number, token).map(|()| 0) + }), + // TODO: 64-bit lengths on 32-bit platforms + SYS_FTRUNCATE => file_op_generic(fd, token, |scheme, number, token| { + scheme.ftruncate(number, c, token).map(|()| 0) + }), SYS_CLOSE => close(fd, token).map(|()| 0), - SYS_CALL => { - let flags = CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?; - if flags.contains(CallFlags::MULTIPLE_FDS) { - if g / core::mem::size_of::() > 16 { - return Err(Error::new(EINVAL)); - }; - let mut fds = [0_usize; 16]; - let fds_slice = UserSlice::ro(b, g)?; - - // TODO: bytemuck/plain - let copied = fds_slice.copy_common_bytes_to_slice(unsafe { - core::slice::from_raw_parts_mut( - fds.as_mut_ptr().cast(), - fds.len() * core::mem::size_of::(), - ) - })?; - call( - &fds[..copied / core::mem::size_of::()], - UserSlice::rw(c, d)?, - flags, - UserSlice::ro(f, (e & 0xff) * 8)?, - token, - ) - } else { - call( - &[b], - UserSlice::rw(c, d)?, - flags, - UserSlice::ro(f, (e & 0xff) * 8)?, - token, - ) - } + SYS_CALL => call( + fd, + UserSlice::rw(c, d)?, + CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?, + UserSlice::ro(f, (e & 0xff) * 8)?, + token, + ), + SYS_OPENAT => { + openat(fd, UserSlice::ro(c, d)?, e, f as _, 0, 0, token).map(FileHandle::into) } - SYS_OPENAT_INTO => openat_into( + SYS_OPENAT_WITH_FILTER => openat( fd, UserSlice::ro(c, d)?, e, + (e & syscall::O_FCNTL_MASK) as _, f as _, - FileHandle::from(g), + g as _, token, ) .map(FileHandle::into), - SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, token).map(|()| 0), + SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, 0, 0, token).map(|()| 0), + SYS_UNLINKAT_WITH_FILTER => { + unlinkat(fd, UserSlice::ro(c, d)?, e, f as _, g as _, token).map(|()| 0) + } SYS_YIELD => sched_yield(token).map(|()| 0), SYS_NANOSLEEP => nanosleep( UserSlice::ro(b, size_of::())?, @@ -221,6 +233,7 @@ pub fn syscall( SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d), token).map(|()| 0), SYS_MREMAP => mremap(b, c, d, e, f, token), + _ => Err(Error::new(ENOSYS)), } } diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 18b0f5ffe7..e83da427b4 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -20,8 +20,7 @@ use crate::{ context::{self, context::FdTbl}, memory::{Page, VirtualAddress, PAGE_SIZE}, scheme::{ - FileHandle, KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES, - KERNEL_SCHEMES_COUNT, + KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES, KERNEL_SCHEMES_COUNT, }, startup::Bootstrap, syscall::{error::*, flag::MapFlags}, @@ -31,30 +30,33 @@ use crate::{ use super::usercopy::UserSliceWo; pub fn exit_this_context(excp: Option, token: &mut CleanLockToken) -> ! { + let mut close_files; + let addrspace_opt; + let context_lock = context::current(); - let (addrspace_opt, mut close_files) = { + { let mut context = context_lock.write(token.token()); - let (context, token) = context.token_split(); - let close_files = Arc::try_unwrap(mem::take(&mut context.files)) + // let (context, mut token) = context.token_split(); + close_files = Arc::try_unwrap(mem::take(&mut context.files)) .map_or_else(|_| FdTbl::new(), RwLock::into_inner); - let addrspace_opt = context.set_addr_space(None, token); + // TODO: Lock ordering violation + let mut token = unsafe { CleanLockToken::new() }; + addrspace_opt = context + .set_addr_space(None, token.downgrade()) + .and_then(|a| Arc::try_unwrap(a).ok()); drop(mem::replace(&mut context.syscall_head, SyscallFrame::Dummy)); drop(mem::replace(&mut context.syscall_tail, SyscallFrame::Dummy)); - (addrspace_opt, close_files) - }; + } // Files must be closed while context is valid so that messages can be passed close_files.force_close_all(token); if let Some(addrspace) = addrspace_opt { - if let Ok(addrspace) = Arc::try_unwrap(addrspace) { - addrspace.into_drop(token); - } + addrspace.into_drop(token); } // TODO: Should status == Status::HardBlocked be handled differently? let owner = { let mut guard = context_lock.write(token.token()); guard.status = context::Status::Dead { excp }; - guard.owner_proc_id }; if let Some(owner) = owner { @@ -73,7 +75,6 @@ pub fn exit_this_context(excp: Option, token: &mut CleanLock } } } - drop(close_files); context::switch(token); unreachable!(); } @@ -157,7 +158,12 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok Ok(fd) => fd, Err(_) => usize::MAX, }; - insert_fd(scheme.scheme_id(), cap_fd, token) + insert_fd( + scheme.scheme_id(), + cap_fd, + matches!(scheme, GlobalSchemes::Proc), + token, + ) }; } } @@ -170,7 +176,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok }; // Second, retrieve the scheme ID. let scheme_id = &SchemeList.id(); - insert_fd(*scheme_id, cap_fd, token) + insert_fd(*scheme_id, cap_fd, false, token) }; let mut lock_token = token.token(); @@ -264,23 +270,12 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] } } -fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usize { +fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize { let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); context - .files - .write(token.token()) - .resize(0, 64) - .expect("failed to resize lower fdtbl"); - context - .files - .write(token.token()) - .resize(syscall::flag::UPPER_FDTBL_TAG, 64) - .expect("failed to resize upper fdtbl"); - context - .insert_file( - FileHandle::from(syscall::flag::UPPER_FDTBL_TAG | scheme.get()), + .add_file_min( FileDescriptor { description: Arc::new(RwLock::new(FileDescription { scheme, @@ -289,7 +284,9 @@ fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usi flags: (O_CREAT | O_RDWR) as u32, internal_flags: InternalFlags::empty(), })), + cloexec, }, + syscall::flag::UPPER_FDTBL_TAG + scheme.get(), &mut token, ) .expect("failed to insert fd to current context") diff --git a/src/syscall/time.rs b/src/syscall/time.rs index e5b64fa0a5..aa6763f5b0 100644 --- a/src/syscall/time.rs +++ b/src/syscall/time.rs @@ -18,7 +18,10 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo, token: &mut CleanLockToken) _ => return Err(Error::new(EINVAL)), }; - buf.copy_exactly(&TimeSpec::from_nanos(arch_time)) + buf.copy_exactly(&TimeSpec { + tv_sec: (arch_time / time::NANOS_PER_SEC) as i64, + tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32, + }) } /// Nanosleep will sleep by switching the current context @@ -34,18 +37,18 @@ pub fn nanosleep( } let start = time::monotonic(token); - let end = start + req.to_nanos(); + let end = start + (req.tv_sec as u128 * time::NANOS_PER_SEC) + (req.tv_nsec as u128); let current_context = context::current(); { - let context = current_context.upgradeable_read(token.token()); + let mut context = current_context.write(token.token()); if let Some((tctl, pctl, _)) = context.sigcontrol() && tctl.currently_pending_unblocked(pctl) != 0 { return Err(Error::new(EINTR)); } - let mut context = context.upgrade(); + context.wake = Some(end); context.block("nanosleep"); } @@ -61,9 +64,15 @@ pub fn nanosleep( rem_buf.copy_exactly(&if current < end { let diff = end - current; - TimeSpec::from_nanos(diff) + TimeSpec { + tv_sec: (diff / time::NANOS_PER_SEC) as i64, + tv_nsec: (diff % time::NANOS_PER_SEC) as i32, + } } else { - TimeSpec::default() + TimeSpec { + tv_sec: 0, + tv_nsec: 0, + } })?; } diff --git a/targets/aarch64-unknown-kernel.json b/targets/aarch64-unknown-kernel.json index f5a777497f..58b72ffac3 100644 --- a/targets/aarch64-unknown-kernel.json +++ b/targets/aarch64-unknown-kernel.json @@ -17,7 +17,6 @@ "os": "none", "position-independent-executables": false, "relocation-model": "pic", - "rustc-abi": "softfloat", "target-c-int-width": 32, "target-endian": "little", "target-pointer-width": 64,