diff --git a/src/arch/aarch64/interrupt/handler.rs b/src/arch/aarch64/interrupt/handler.rs index 0fc15dc559..85b2b4546c 100644 --- a/src/arch/aarch64/interrupt/handler.rs +++ b/src/arch/aarch64/interrupt/handler.rs @@ -390,15 +390,15 @@ macro_rules! exception_stack { unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { unsafe { $code }} - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Backup all userspace registers to stack push_preserved!(), push_scratch!(), push_special!(), // Call inner function with pointer to stack - "mov x29, sp\n", - "mov x0, sp\n", + "mov x29, sp", + "mov x0, sp", "bl {}", // Restore all userspace registers @@ -406,19 +406,21 @@ macro_rules! exception_stack { pop_scratch!(), pop_preserved!(), - "eret\n", - ), sym inner); + "eret", + + sym inner, + ); } }; } #[unsafe(naked)] pub unsafe extern "C" fn enter_usermode() -> ! { - core::arch::naked_asm!(concat!( - "blr x28\n", + core::arch::naked_asm!( + "blr x28", // Restore all userspace registers pop_special!(), pop_scratch!(), pop_preserved!(), - "eret\n", - )); + "eret", + ); } diff --git a/src/arch/aarch64/vectors.rs b/src/arch/aarch64/vectors.rs index 2b61f9d9a7..f8bf0e46d1 100644 --- a/src/arch/aarch64/vectors.rs +++ b/src/arch/aarch64/vectors.rs @@ -105,7 +105,7 @@ __vec_14: __vec_15: b unhandled_exception b __vec_15 - + .align 7 exception_vector_end: " diff --git a/src/arch/riscv64/interrupt/handler.rs b/src/arch/riscv64/interrupt/handler.rs index 1be5b9298c..339fc2ab46 100644 --- a/src/arch/riscv64/interrupt/handler.rs +++ b/src/arch/riscv64/interrupt/handler.rs @@ -319,15 +319,15 @@ macro_rules! pop_registers { #[unsafe(naked)] pub unsafe extern "C" fn enter_usermode() -> ! { - core::arch::naked_asm!(concat!( - "jalr s11\n", - "li t0, 1 << 8\n", // force U mode on sret - "csrc sstatus, t0\n", - "li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode) - "csrs sstatus, t0\n", - "addi t0, sp, 32 * 8\n", // save S mode stack to percpu - "sd t0, 8(tp)\n", + core::arch::naked_asm!( + "jalr s11", + "li t0, 1 << 8", // force U mode on sret + "csrc sstatus, t0", + "li t0, 0x6000", // set FS to dirty (enable FPU in U mode) + "csrs sstatus, t0", + "addi t0, sp, 32 * 8", // save S mode stack to percpu + "sd t0, 8(tp)", pop_registers!(), - "sret\n", - )) + "sret", + ) } diff --git a/src/arch/x86/interrupt/handler.rs b/src/arch/x86/interrupt/handler.rs index cbf5051ade..59d9caadce 100644 --- a/src/arch/x86/interrupt/handler.rs +++ b/src/arch/x86/interrupt/handler.rs @@ -296,9 +296,9 @@ macro_rules! interrupt_stack { $code } } - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Backup all userspace registers to stack - "push eax\n", + "push eax", push_scratch!(), push_preserved!(), @@ -324,9 +324,9 @@ macro_rules! interrupt_stack { pop_preserved!(), pop_scratch!(), - "iretd\n", - ), - inner = sym inner, + "iretd", + + inner = sym inner, ); } }; @@ -343,9 +343,9 @@ macro_rules! interrupt { $code } - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Backup all userspace registers to stack - "push eax\n", + "push eax", push_scratch!(), // Enter kernel TLS segment @@ -355,7 +355,7 @@ macro_rules! interrupt { // $crate::arch::x86::pti::map(); // Call inner function with pointer to stack - "call {inner}\n", + "call {inner}", // TODO: Unmap PTI // $crate::arch::x86::pti::unmap(); @@ -366,9 +366,9 @@ macro_rules! interrupt { // Restore all userspace registers pop_scratch!(), - "iretd\n", - ), - inner = sym inner, + "iretd", + + inner = sym inner, ); } }; @@ -384,10 +384,10 @@ macro_rules! interrupt_error { $code } - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Move eax into code's place, put code in last instead (to be // compatible with InterruptStack) - "xchg [esp], eax\n", + "xchg [esp], eax", // Push all userspace registers push_scratch!(), @@ -397,7 +397,7 @@ macro_rules! interrupt_error { enter_gs!(), // Put code in, it's now in eax - "push eax\n", + "push eax", // TODO: Map PTI // $crate::arch::x86::pti::map(); @@ -413,7 +413,7 @@ macro_rules! interrupt_error { // $crate::arch::x86::pti::unmap(); // Pop previous esp and code - "add esp, 8\n", + "add esp, 8", // Exit kernel TLS segment exit_gs!(), @@ -423,9 +423,9 @@ macro_rules! interrupt_error { pop_scratch!(), // The error code has already been popped, so use the regular macro. - "iretd\n", - ), - inner = sym inner); + "iretd", + inner = sym inner, + ); } }; } @@ -461,7 +461,7 @@ impl ArchIntCtx for InterruptStack { #[unsafe(naked)] pub unsafe extern "C" fn enter_usermode() { - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // TODO: Unmap PTI // $crate::arch::x86::pti::unmap(); @@ -470,6 +470,6 @@ pub unsafe extern "C" fn enter_usermode() { // Restore all userspace registers pop_preserved!(), pop_scratch!(), - "iretd\n", - )) + "iretd", + ) } diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index 2186dee468..e3a0d71c0c 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -383,13 +383,13 @@ macro_rules! interrupt_stack { unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) { $code } - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", // Backup all userspace registers to stack $save1!(), - "push rax\n", + "push rax", push_scratch!(), push_preserved!(), @@ -414,13 +414,12 @@ macro_rules! interrupt_stack { pop_scratch!(), $rstor1!(), - "iretq\n", - ), + "iretq", - inner = sym inner, - IA32_GS_BASE = const(x86::msr::IA32_GS_BASE), + inner = sym inner, + IA32_GS_BASE = const(x86::msr::IA32_GS_BASE), - PCR_GDT_OFFSET = const(core::mem::offset_of!($crate::gdt::ProcessorControlRegion, gdt)), + PCR_GDT_OFFSET = const(core::mem::offset_of!($crate::gdt::ProcessorControlRegion, gdt)), ); } }; @@ -437,20 +436,20 @@ macro_rules! interrupt { $code } - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", // Backup all userspace registers to stack swapgs_iff_ring3_fast!(), - "push rax\n", + "push rax", push_scratch!(), // TODO: Map PTI // $crate::arch::x86_64::pti::map(); // Call inner function with pointer to stack - "call {inner}\n", + "call {inner}", // TODO: Unmap PTI // $crate::arch::x86_64::pti::unmap(); @@ -459,10 +458,9 @@ macro_rules! interrupt { pop_scratch!(), swapgs_iff_ring3_fast!(), - "iretq\n", - ), + "iretq", - inner = sym inner, + inner = sym inner, ); } }; @@ -477,7 +475,7 @@ macro_rules! interrupt_error { $code } - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", @@ -512,10 +510,9 @@ macro_rules! interrupt_error { // The error code has already been popped, so use the regular macro. swapgs_iff_ring3_fast!(), "iretq;", - ), - inner = sym inner, - rax_offset = const(::core::mem::size_of::<$crate::interrupt::handler::PreservedRegisters>() + ::core::mem::size_of::<$crate::interrupt::handler::ScratchRegisters>() - 8), + inner = sym inner, + rax_offset = const(::core::mem::size_of::<$crate::interrupt::handler::PreservedRegisters>() + ::core::mem::size_of::<$crate::interrupt::handler::ScratchRegisters>() - 8), ); } }; diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index ca6d1f7a45..ca86e5f737 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -95,7 +95,7 @@ pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) #[unsafe(naked)] pub unsafe extern "C" fn syscall_instruction() { - core::arch::naked_asm!(concat!( + core::arch::naked_asm!( // Yes, this is magic. No, you don't need to understand "swapgs;", // Swap KGSBASE with GSBASE, allowing fast TSS access. "mov gs:[{sp}], rsp;", // Save userspace stack pointer @@ -193,7 +193,7 @@ pub unsafe extern "C" fn syscall_instruction() { xor rcx, rcx xor r11, r11 iretq - "), + ", sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)), ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)),