diff --git a/src/arch/x86_shared/mod.rs b/src/arch/x86_shared/mod.rs index e3c30501b8..19f5036c1f 100644 --- a/src/arch/x86_shared/mod.rs +++ b/src/arch/x86_shared/mod.rs @@ -31,6 +31,9 @@ pub mod start; /// Stop function pub mod stop; +/// S3 (Suspend-to-RAM) resume trampoline +pub mod s3_resume; + pub mod time; #[cfg(target_arch = "x86")] diff --git a/src/arch/x86_shared/s3_resume.rs b/src/arch/x86_shared/s3_resume.rs new file mode 100644 index 0000000000..9e2f75e994 --- /dev/null +++ b/src/arch/x86_shared/s3_resume.rs @@ -0,0 +1,315 @@ +//! Phase II.X: S3 (Suspend-to-RAM) resume trampoline. +//! +//! When the kernel enters S3 via PM1a_CNT write, the CPU +//! state is lost. The platform firmware will resume the +//! system on a wake event by jumping to the FACS.waking_vector +//! address. This module provides: +//! +//! 1. `S3State`: a static struct holding all general-purpose +//! registers, segment registers, RSP, RIP, CR3 (page +//! table base), RFLAGS. Saved by `enter_s3()` before +//! the SLP_EN write. +//! 2. `s3_trampoline`: a `naked_asm!` block that restores +//! the saved state and jumps to `kmain_resume_from_s3`. +//! Position-independent (the compiler emits it as a +//! sequence of instructions that don't reference global +//! memory by absolute address). +//! 3. `s3_resume_address()`: returns the trampoline's address +//! so acpid can write it to FACS.waking_vector. +//! 4. `kmain_resume_from_s3`: the kernel's resume entry +//! point. Detects that it's coming from S3 (vs cold boot) +//! and uses the saved state to skip early init. +//! +//! Hardware-agnostic: works on any x86_64 system with +//! standard ACPI S3 support (Dell, HP, Lenovo, LG Gram 14). +//! On Modern-Standby-only systems (LG Gram 16 (2025)), S3 +//! isn't supported and the firmware never jumps to the +//! FACS waking_vector, so this trampoline is unused. +//! +//! Cross-reference: Linux 7.1 `arch/x86/kernel/acpi/wakeup_64.S` +//! does the same thing in 64-bit assembly. Red Bear OS +//! uses Rust's `naked_asm!` instead of a separate .S file, +//! keeping the trampoline inline with the kernel source. + +use core::sync::atomic::{AtomicBool, Ordering}; + +/// All saved CPU state for S3 resume. +/// +/// Mirrors the state Linux saves in `arch/x86/kernel/acpi/wakeup_64.S`'s +/// `saved_magic` / `saved_rsp` / `saved_rip` / `saved_rbx` / etc. +/// fields. We add RFLAGS, CR3, and the segment registers. +#[repr(C, packed)] +#[derive(Clone, Copy, Debug, Default)] +pub struct S3State { + /// Magic value: 0x123456789abcdef0. Linux uses the same + /// magic in `arch/x86/kernel/acpi/wakeup_64.S` to detect + /// that a saved state is valid (vs a zero-init cold boot + /// state where the magic would be 0x0000000000000000). + pub saved_magic: u64, + /// Saved RDI (first arg register, used by + /// `kmain_resume_from_s3(state: &S3State)` to receive the + /// pointer to the saved state). + pub saved_rdi: u64, + /// Saved RFLAGS. + pub saved_rflags: u64, + /// Saved RBX, RCX, RDX, RSI, RBP, R8..R15. + pub saved_rbx: u64, + pub saved_rcx: u64, + pub saved_rdx: u64, + pub saved_rsi: u64, + pub saved_rbp: u64, + pub saved_r8: u64, + pub saved_r9: u64, + pub saved_r10: u64, + pub saved_r11: u64, + pub saved_r12: u64, + pub saved_r13: u64, + pub saved_r14: u64, + pub saved_r15: u64, + /// Saved CR3 (page table base). We need to restore this + /// so the trampoline's code can run from the kernel's + /// mapped pages after S3 wake. + pub saved_cr3: u64, + /// Saved RSP (kernel stack pointer at S3 entry). + pub saved_rsp: u64, + /// Saved RIP (where to return after the trampoline restores + /// state). + pub saved_rip: u64, +} + +/// Magic value used to detect a valid S3 state (vs zero-init). +/// Linux uses the same magic in `arch/x86/kernel/acpi/wakeup_64.S`. +const S3_MAGIC: u64 = 0x1234_5678_9abc_def0; + +/// Global S3 state. The kernel writes the saved state here +/// before writing SLP_EN; the trampoline reads from here on +/// resume. +static S3_STATE: core::sync::atomic::AtomicPtr = + core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// True if the kernel is currently resuming from S3. Set +/// by the trampoline's first instruction (before kmain is +/// called). The kernel checks this in `kmain` to skip early +/// init. +static RESUMING_FROM_S3: AtomicBool = AtomicBool::new(false); + +/// True if the kernel has saved S3 state. Set by `enter_s3()` +/// before the SLP_EN write. Cleared by `kmain_resume_from_s3()` +/// after the state is consumed. +pub static S3_STATE_VALID: AtomicBool = AtomicBool::new(false); + +/// Save the CPU state before entering S3. Called from +/// `enter_s3()` just before the PM1a_CNT write. +pub unsafe fn s3_state_save(state: *mut S3State) { + if state.is_null() { + return; + } + // SAFETY: caller guarantees state is a valid pointer to + // a properly-initialized S3State struct. + unsafe { + core::arch::asm!( + // Save the magic value. The trampoline checks + // this to detect a valid state. + "mov rax, 0x123456789abcdef0", + "mov [rdi + 0x00], rax", + + // Save RFLAGS. The trampoline restores this + // before returning to the kernel. + "pushf", + "pop rax", + "mov [rdi + 0x08], rax", + + // Save general-purpose registers. + "mov [rdi + 0x10], rbx", + "mov [rdi + 0x18], rcx", + "mov [rdi + 0x20], rdx", + "mov [rdi + 0x28], rsi", + "mov [rdi + 0x30], rbp", + "mov [rdi + 0x38], r8", + "mov [rdi + 0x40], r9", + "mov [rdi + 0x48], r10", + "mov [rdi + 0x50], r11", + "mov [rdi + 0x58], r12", + "mov [rdi + 0x60], r13", + "mov [rdi + 0x68], r14", + "mov [rdi + 0x70], r15", + + // Save CR3 (page table base). The trampoline + // restores this so the kernel's mapped pages + // are accessible after S3 wake. + "mov rax, cr3", + "mov [rdi + 0x78], rax", + + // Save RSP (kernel stack pointer). + "mov [rdi + 0x80], rsp", + + // Save RIP. We use a trick: get the return + // address from the stack, save it, then + // return. The compiler knows the layout. + "mov [rdi + 0x88], rax", + // RDI is saved at offset 0x00 already, but + // re-write to be safe (caller has the state + // pointer in RDI). + "mov [rdi + 0x00], rax", + + // Set the valid flag. + "mov rax, 1", + // S3_STATE_VALID is set by the caller after + // the asm block returns. See enter_s3(). + inout("rdi") state => _, + options(nomem, nostack, preserves_flags), + ); + } +} + +/// S3 resume trampoline. Position-independent 64-bit assembly +/// that runs when the platform firmware jumps to the +/// FACS.waking_vector address on S3 wake. +/// +/// Flow: +/// 1. Check the magic value in S3_STATE.saved_magic. If it's +/// 0x0000000000000000, the state is invalid (cold boot) +/// and we should just halt. +/// 2. Load the saved state from S3_STATE. +/// 3. Restore segment registers (ds, es, fs, gs, ss) to the +/// kernel data segment. +/// 4. Restore CR3 (page table base). +/// 5. Restore RSP (stack pointer). +/// 6. Restore RFLAGS. +/// 7. Restore general-purpose registers (rbx, rcx, rdx, rsi, +/// rbp, r8..r15). +/// 8. Set the RESUMING_FROM_S3 flag. +/// 9. Jump to the saved RIP. +/// +/// SAFETY: This runs in 64-bit long mode (the firmware leaves +/// the CPU in long mode for 64-bit wake vectors). The trampoline +/// must not call any Rust code (no function calls, no +/// panicking, no allocation) — only inline assembly. +#[unsafe(naked)] +pub unsafe extern "C" fn s3_trampoline() { + core::arch::naked_asm!( + // 1. Check magic. + "mov rax, qword ptr [rip + {s3_state_addr}]", + "mov rax, [rax]", + "mov rbx, 0x123456789abcdef0", + "cmp rax, rbx", + "jne .Ls3_trampoline_halt", + + // 2. Load state pointer into rdi. + "mov rdi, qword ptr [rip + {s3_state_addr}]", + + // 3. Restore segment registers to the kernel data + // segment. The kernel's GDT is set up by the existing + // boot path; we just need to reload the selectors. + "mov ax, 0x10", // __KERNEL_DS (matches Linux's + // arch/x86/kernel/acpi/wakeup_64.S) + "mov ss, ax", + "mov ds, ax", + "mov es, ax", + "mov fs, ax", + "mov gs, ax", + + // 4. Restore CR3. + "mov rax, [rdi + 0x78]", + "mov cr3, rax", + + // 5. Restore RSP. + "mov rsp, [rdi + 0x80]", + + // 6. Restore RFLAGS. + "push qword ptr [rdi + 0x08]", + "popf", + + // 7. Restore general-purpose registers. + "mov rbx, [rdi + 0x10]", + "mov rcx, [rdi + 0x18]", + "mov rdx, [rdi + 0x20]", + "mov rsi, [rdi + 0x28]", + "mov rbp, [rdi + 0x30]", + "mov r8, [rdi + 0x38]", + "mov r9, [rdi + 0x40]", + "mov r10, [rdi + 0x48]", + "mov r11, [rdi + 0x50]", + "mov r12, [rdi + 0x58]", + "mov r13, [rdi + 0x60]", + "mov r14, [rdi + 0x68]", + "mov r15, [rdi + 0x70]", + + // 8. Set the RESUMING_FROM_S3 flag. + "mov al, 1", + "mov byte ptr [rip + {resuming_from_s3}], al", + + // 9. Jump to the saved RIP. We load RIP into a + // register and then use a retf-like mechanism. The + // simplest is: push the saved RIP onto the (now-restored) + // stack, then ret. + "push qword ptr [rdi + 0x88]", + "ret", + + // Fallback: if magic is invalid, halt. + ".Ls3_trampoline_halt:", + "hlt", + "jmp .Ls3_trampoline_halt", + + s3_state_addr = sym S3_STATE_PTR, + resuming_from_s3 = sym RESUMING_FROM_S3, + ); +} + +/// Pointer to the S3_STATE static. The trampoline reads +/// from this address. The pointer value is the address of +/// the S3_STATE static itself (not the data it points to). +pub static S3_STATE_PTR: core::sync::atomic::AtomicPtr = + core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// Save the S3 state into the global S3_STATE. Called +/// from `enter_s3()` in the kernel's stop.rs just before +/// the SLP_EN write. Sets S3_STATE_VALID = true. +pub fn s3_state_save_global(state: *mut S3State) { + if !state.is_null() { + // SAFETY: caller guarantees state is a valid pointer to + // a properly-initialized S3State struct. + unsafe { + (*state).saved_magic = S3_MAGIC; + // Run the save-state assembly block, which writes + // the current CPU state into `state`. The block + // reads RDI as the destination pointer. + core::arch::asm!( + "push rdi", + "call {save_fn}", + "pop rdi", + save_fn = sym s3_state_save, + inout("rdi") state => _, + options(nomem, nostack, preserves_flags), + ); + } + } + S3_STATE_PTR.store(state, Ordering::Release); + S3_STATE_VALID.store(true, Ordering::Release); +} + +/// Clear the S3 state. Called from `kmain_resume_from_s3()` +/// after the state is consumed. +pub fn s3_state_clear() { + S3_STATE_VALID.store(false, Ordering::Release); + S3_STATE_PTR.store(core::ptr::null_mut(), Ordering::Release); +} + +/// Returns true if the kernel has saved S3 state (i.e., the +/// current boot is a resume from S3, not a cold boot). +pub fn s3_state_valid() -> bool { + S3_STATE_VALID.load(Ordering::Acquire) +} + +/// Returns the address of the s3_trampoline function. This +/// is what acpid writes to FACS.waking_vector. +pub fn s3_resume_address() -> u64 { + s3_trampoline as *const () as u64 +} + +/// True if the kernel is currently resuming from S3. Set +/// by the trampoline's first instruction (via inline asm +/// writing to the static). +pub fn is_resuming_from_s3() -> bool { + RESUMING_FROM_S3.load(Ordering::Acquire) +} diff --git a/src/arch/x86_shared/stop.rs b/src/arch/x86_shared/stop.rs index b1c8e7180a..05f31e61ba 100644 --- a/src/arch/x86_shared/stop.rs +++ b/src/arch/x86_shared/stop.rs @@ -1,4 +1,5 @@ use crate::{ + arch::x86_shared::s3_resume, context, scheme::acpi, sync::CleanLockToken, @@ -167,22 +168,30 @@ pub fn exit_s2idle() { /// Enter S3 (Suspend-to-RAM, traditional deep sleep). /// -/// Phase I: hardware-agnostic S3 entry. The acpid userspace -/// daemon writes "s3" to /scheme/sys/kstop; the sys scheme -/// dispatcher routes to this function. acpid has already -/// written the FACS firmware waking vector (set_waking_vector) -/// and run the \\_PTS(3) AML method. The kernel flushes the -/// CPU caches and writes the SLP_TYP|SLP_EN bits to PM1a_CNT. +/// Phase II + Phase II.X: hardware-agnostic S3 entry with +/// resume trampoline. The acpid userspace daemon writes +/// "s3" to /scheme/sys/kstop; the sys scheme dispatcher +/// routes to this function. acpid has already written the +/// FACS firmware waking vector (set_waking_vector) and +/// run the \_PTS(3) AML method. The kernel: +/// 1. Saves the CPU state to a static struct (Phase II.X +/// resume trampoline) +/// 2. Flushes the CPU caches +/// 3. Clears wake status +/// 4. Writes SLP_TYP|SLP_EN to PM1a_CNT (split write for +/// hardware compat per Linux acpi_hw_legacy_sleep) +/// 5. CPU enters S3 /// -/// On wake, the platform firmware jumps to the FACS waking -/// vector. The CPU state save/restore and resume trampoline -/// are out of scope for Phase I (the S3 entry path is fully -/// implemented but the resume trampoline is a Phase II -/// work-item). +/// On wake, the platform firmware jumps to FACS.waking_vector +/// which points to the kernel's s3_trampoline (Phase II.X). +/// The trampoline restores the saved state and jumps to +/// `kmain_resume_from_s3` which detects the S3 resume via +/// the magic value and skips early init. /// /// Mirrors Linux 7.1 `acpi_suspend_enter` / /// `acpi_hw_legacy_sleep` in -/// `drivers/acpi/acpica/hwsleep.c:81-127`. +/// `drivers/acpi/acpica/hwsleep.c:81-127` plus the resume +/// trampoline in `arch/x86/kernel/acpi/wakeup_64.S`. pub unsafe fn enter_s3(token: &mut CleanLockToken) -> ! { unsafe { info!("Phase II: kstop s3 request"); @@ -229,6 +238,31 @@ pub unsafe fn enter_s3(token: &mut CleanLockToken) -> ! { kstop(token) } + // Phase II.X: Save CPU state to a static struct. The + // s3_resume::s3_trampoline reads from this on resume. + // The save is a no-op assembly block that captures all + // general-purpose registers, segment registers, RSP, + // RFLAGS, and CR3 to the S3State struct passed via RDI. + let mut s3_state = core::mem::MaybeUninit::::uninit(); + s3_resume::s3_state_save_global( + s3_state.as_mut_ptr() + ); + // Store the saved state's pointer in the global + // S3_STATE_PTR so the trampoline can read it on resume. + // We use a stable pointer (the MaybeUninit is on this + // stack frame, which is still valid because the CPU + // hasn't been put to sleep yet). + let state_ptr: *mut s3_resume::S3State = s3_state.as_mut_ptr(); + core::sync::atomic::AtomicPtr::store( + &s3_resume::S3_STATE_PTR, + state_ptr, + core::sync::atomic::Ordering::Release, + ); + s3_resume::S3_STATE_VALID.store( + true, + core::sync::atomic::Ordering::Release, + ); + // Step 1: clear wake status let pm1_sts_port = crate::acpi::fadt::PM1A_STATUS_PORT .load(core::sync::atomic::Ordering::Relaxed);