diff --git a/src/arch/aarch64/interrupt/handler.rs b/src/arch/aarch64/interrupt/handler.rs index 7083329589..c21d0e9fd3 100644 --- a/src/arch/aarch64/interrupt/handler.rs +++ b/src/arch/aarch64/interrupt/handler.rs @@ -381,7 +381,7 @@ macro_rules! exception_stack { unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { $code } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Backup all userspace registers to stack push_preserved!(), push_scratch!(), @@ -398,21 +398,18 @@ macro_rules! exception_stack { pop_preserved!(), "eret\n", - ), sym inner, options(noreturn)); + ), sym inner); } }; } #[naked] pub unsafe extern "C" fn enter_usermode() -> ! { - core::arch::asm!( - concat!( - "blr x28\n", - // Restore all userspace registers - pop_special!(), - pop_scratch!(), - pop_preserved!(), - "eret\n", - ), - options(noreturn) - ); + core::arch::naked_asm!(concat!( + "blr x28\n", + // Restore all userspace registers + pop_special!(), + pop_scratch!(), + pop_preserved!(), + "eret\n", + )); } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 739cbbf854..16714c189d 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -43,7 +43,7 @@ pub use arch_copy_to_user as arch_copy_from_user; #[link_section = ".usercopy-fns"] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { // x0, x1, x2 - core::arch::asm!( + core::arch::naked_asm!( " mov x4, x0 mov x0, 0 @@ -61,8 +61,7 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - b 2b 3: ret - ", - options(noreturn) + " ); } diff --git a/src/arch/riscv64/interrupt/handler.rs b/src/arch/riscv64/interrupt/handler.rs index f495f6676d..e7c2914325 100644 --- a/src/arch/riscv64/interrupt/handler.rs +++ b/src/arch/riscv64/interrupt/handler.rs @@ -302,18 +302,15 @@ macro_rules! pop_registers { #[naked] pub unsafe extern "C" fn enter_usermode() -> ! { - core::arch::asm!( - concat!( - "jalr s11\n", - "li t0, 1 << 8\n", // force U mode on sret - "csrc sstatus, t0\n", - "li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode) - "csrs sstatus, t0\n", - "addi t0, sp, 32 * 8\n", // save S mode stack to percpu - "sd t0, 8(tp)\n", - pop_registers!(), - "sret\n", - ), - options(noreturn) - ) + core::arch::naked_asm!(concat!( + "jalr s11\n", + "li t0, 1 << 8\n", // force U mode on sret + "csrc sstatus, t0\n", + "li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode) + "csrs sstatus, t0\n", + "addi t0, sp, 32 * 8\n", // save S mode stack to percpu + "sd t0, 8(tp)\n", + pop_registers!(), + "sret\n", + )) } diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs index f2bed7a666..62d1afbd3b 100644 --- a/src/arch/riscv64/mod.rs +++ b/src/arch/riscv64/mod.rs @@ -14,14 +14,14 @@ pub mod stop; pub mod time; pub use ::rmm::RiscV64Sv48Arch as CurrentRmmArch; -use core::arch::asm; +use core::arch::naked_asm; pub use arch_copy_to_user as arch_copy_from_user; #[link_section = ".usercopy-fns"] #[naked] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - asm!( + naked_asm!( " addi sp, sp, -16 sd fp, 0(sp) @@ -58,8 +58,7 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - bne a2, x0, 4b 5: mv a0, x0 ret - ", - options(noreturn) + " ) } diff --git a/src/arch/x86/interrupt/handler.rs b/src/arch/x86/interrupt/handler.rs index c8f3e27eed..973298e7cf 100644 --- a/src/arch/x86/interrupt/handler.rs +++ b/src/arch/x86/interrupt/handler.rs @@ -276,7 +276,7 @@ macro_rules! interrupt_stack { $code } } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Backup all userspace registers to stack "push eax\n", push_scratch!(), @@ -306,11 +306,7 @@ macro_rules! interrupt_stack { "iretd\n", ), - inner = sym inner, - - options(noreturn), - ); } }; @@ -327,7 +323,7 @@ macro_rules! interrupt { $code } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Backup all userspace registers to stack "push eax\n", push_scratch!(), @@ -352,10 +348,7 @@ macro_rules! interrupt { "iretd\n", ), - inner = sym inner, - - options(noreturn), ); } }; @@ -373,7 +366,7 @@ macro_rules! interrupt_error { } } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Move eax into code's place, put code in last instead (to be // compatible with InterruptStack) "xchg [esp], eax\n", @@ -414,24 +407,20 @@ macro_rules! interrupt_error { // The error code has already been popped, so use the regular macro. "iretd\n", ), - - inner = sym inner, - - options(noreturn)); + inner = sym inner); } }; } #[naked] unsafe extern "C" fn usercopy_trampoline() { - core::arch::asm!( + core::arch::naked_asm!( " pop esi pop edi mov eax, 1 ret - ", - options(noreturn) + " ); } @@ -454,18 +443,15 @@ impl ArchIntCtx for InterruptStack { #[naked] pub unsafe extern "C" fn enter_usermode() { - core::arch::asm!( - concat!( - // TODO: Unmap PTI - // $crate::arch::x86::pti::unmap(); + core::arch::naked_asm!(concat!( + // TODO: Unmap PTI + // $crate::arch::x86::pti::unmap(); - // Exit kernel TLS segment - exit_gs!(), - // Restore all userspace registers - pop_preserved!(), - pop_scratch!(), - "iretd\n", - ), - options(noreturn) - ) + // Exit kernel TLS segment + exit_gs!(), + // Restore all userspace registers + pop_preserved!(), + pop_scratch!(), + "iretd\n", + )) } diff --git a/src/arch/x86/mod.rs b/src/arch/x86/mod.rs index ca26ade330..a3f78082c2 100644 --- a/src/arch/x86/mod.rs +++ b/src/arch/x86/mod.rs @@ -32,7 +32,7 @@ pub mod flags { #[naked] #[link_section = ".usercopy-fns"] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - core::arch::asm!( + core::arch::naked_asm!( " push edi push esi @@ -47,8 +47,7 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - xor eax, eax ret - ", - options(noreturn) + " ); } pub use arch_copy_to_user as arch_copy_from_user; diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index f3c94985fb..547e0c38e3 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -369,7 +369,7 @@ macro_rules! interrupt_stack { $code } } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", @@ -407,9 +407,6 @@ macro_rules! interrupt_stack { IA32_GS_BASE = const(x86::msr::IA32_GS_BASE), PCR_GDT_OFFSET = const(core::mem::offset_of!(crate::gdt::ProcessorControlRegion, gdt)), - - options(noreturn), - ); } }; @@ -426,7 +423,7 @@ macro_rules! interrupt { $code } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", @@ -452,8 +449,6 @@ macro_rules! interrupt { ), inner = sym inner, - - options(noreturn), ); } }; @@ -471,7 +466,7 @@ macro_rules! interrupt_error { } } - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", @@ -510,8 +505,7 @@ macro_rules! interrupt_error { inner = sym inner, rax_offset = const(::core::mem::size_of::<$crate::interrupt::handler::PreservedRegisters>() + ::core::mem::size_of::<$crate::interrupt::handler::ScratchRegisters>() - 8), - - options(noreturn)); + ); } }; } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 55b6d1f4c2..828908539a 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -85,7 +85,7 @@ pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) #[naked] #[allow(named_asm_labels)] pub unsafe extern "C" fn syscall_instruction() { - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( // Yes, this is magic. No, you don't need to understand "swapgs;", // Swap KGSBASE with GSBASE, allowing fast TSS access. "mov gs:[{sp}], rsp;", // Save userspace stack pointer @@ -189,8 +189,6 @@ pub unsafe extern "C" fn syscall_instruction() { ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)), ss_sel = const(SegmentSelector::new(gdt::GDT_USER_DATA as u16, x86::Ring::Ring3).bits()), cs_sel = const(SegmentSelector::new(gdt::GDT_USER_CODE as u16, x86::Ring::Ring3).bits()), - - options(noreturn), ); } extern "C" { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 9057c35049..59823f4cfe 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -44,10 +44,9 @@ pub mod flags { pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { // TODO: spectre_v1 - core::arch::asm!( - alternative!( - feature: "smap", - then: [" + core::arch::naked_asm!(alternative!( + feature: "smap", + then: [" xor eax, eax mov rcx, rdx stac @@ -55,15 +54,13 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - clac ret "], - default: [" + default: [" xor eax, eax mov rcx, rdx rep movsb ret "] - ), - options(noreturn) - ); + )); } pub use arch_copy_to_user as arch_copy_from_user; diff --git a/src/context/arch/aarch64.rs b/src/context/arch/aarch64.rs index fa6e272b62..844164cf86 100644 --- a/src/context/arch/aarch64.rs +++ b/src/context/arch/aarch64.rs @@ -4,7 +4,7 @@ use crate::{ percpu::PercpuBlock, syscall::FloatRegisters, }; -use core::{arch::asm, mem, mem::offset_of, ptr, sync::atomic::AtomicBool}; +use core::{mem, mem::offset_of, ptr, sync::atomic::AtomicBool}; use rmm::TableKind; use spin::Once; use syscall::{EnvRegisters, Error, Result, ENOMEM}; @@ -206,7 +206,7 @@ pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { #[target_feature(enable = "neon")] #[naked] unsafe extern "C" fn fp_save(float_regs: &mut FloatRegisters) { - asm!( + core::arch::naked_asm!( "stp q0, q1, [x0, {0} + 16 * 0]", "stp q2, q3, [x0, {0} + 16 * 2]", "stp q4, q5, [x0, {0} + 16 * 4]", @@ -232,14 +232,13 @@ unsafe extern "C" fn fp_save(float_regs: &mut FloatRegisters) { const mem::offset_of!(FloatRegisters, fp_simd_regs), const mem::offset_of!(FloatRegisters, fpcr), const mem::offset_of!(FloatRegisters, fpsr), - options(noreturn), ); } #[target_feature(enable = "neon")] #[naked] unsafe extern "C" fn fp_load(float_regs: &mut FloatRegisters) { - asm!( + core::arch::naked_asm!( "ldp q0, q1, [x0, {0} + 16 * 0]", "ldp q2, q3, [x0, {0} + 16 * 2]", "ldp q4, q5, [x0, {0} + 16 * 4]", @@ -265,7 +264,6 @@ unsafe extern "C" fn fp_load(float_regs: &mut FloatRegisters) { const mem::offset_of!(FloatRegisters, fp_simd_regs), const mem::offset_of!(FloatRegisters, fpcr), const mem::offset_of!(FloatRegisters, fpsr), - options(noreturn), ); } @@ -287,7 +285,7 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { #[naked] unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { - core::arch::asm!( + core::arch::naked_asm!( " str x19, [x0, #{off_x19}] ldr x19, [x1, #{off_x19}] @@ -383,7 +381,6 @@ unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { off_sp = const(offset_of!(Context, sp)), switch_hook = sym crate::context::switch_finish_hook, - options(noreturn), ); } diff --git a/src/context/arch/riscv64.rs b/src/context/arch/riscv64.rs index 298ab83278..27bac6e4be 100644 --- a/src/context/arch/riscv64.rs +++ b/src/context/arch/riscv64.rs @@ -149,7 +149,7 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { #[naked] unsafe extern "C" fn switch_to_inner(prev: &mut Context, next: &mut Context) { - core::arch::asm!(r#" + core::arch::naked_asm!(r#" sd s1, {off_s1}(a0) ld s1, {off_s1}(a1) @@ -219,7 +219,6 @@ unsafe extern "C" fn switch_to_inner(prev: &mut Context, next: &mut Context) { off_sstatus = const(offset_of!(Context, sstatus)), switch_hook = sym crate::context::switch_finish_hook, - options(noreturn), ); } diff --git a/src/context/arch/x86.rs b/src/context/arch/x86.rs index 2e258f9536..4a98d9ebc4 100644 --- a/src/context/arch/x86.rs +++ b/src/context/arch/x86.rs @@ -252,7 +252,7 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { unsafe extern "cdecl" fn switch_to_inner() { use Context as Cx; - core::arch::asm!( + core::arch::naked_asm!( // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): // // - the current parameters are passed in the registers `edi`, `esi`, @@ -306,7 +306,6 @@ unsafe extern "cdecl" fn switch_to_inner() { off_esp = const(offset_of!(Cx, esp)), switch_hook = sym crate::context::switch_finish_hook, - options(noreturn), ); } diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 789a194759..21cecac460 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -329,7 +329,7 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { use Context as Cx; - core::arch::asm!( + core::arch::naked_asm!( // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): // // - the current parameters are passed in the registers `rdi`, `rsi`, @@ -389,7 +389,6 @@ unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Conte off_rsp = const(offset_of!(Cx, rsp)), switch_hook = sym crate::context::switch_finish_hook, - options(noreturn), ); }