diff --git a/linkers/aarch64.ld b/linkers/aarch64.ld index f679c9af3a..c5f1e469a8 100644 --- a/linkers/aarch64.ld +++ b/linkers/aarch64.ld @@ -25,9 +25,6 @@ SECTIONS { .text : AT(ADDR(.text) - KERNEL_OFFSET) { __text_start = .; *(.text*) - __usercopy_start = .; - *(.usercopy-fns) - __usercopy_end = .; . = ALIGN(4096); __text_end = .; } diff --git a/linkers/i686.ld b/linkers/i686.ld index f7eb9c4295..c1b0c25875 100644 --- a/linkers/i686.ld +++ b/linkers/i686.ld @@ -23,9 +23,6 @@ SECTIONS { .text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) { __text_start = .; *(.text*) - __usercopy_start = .; - *(.usercopy-fns) - __usercopy_end = .; } .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { diff --git a/linkers/riscv64.ld b/linkers/riscv64.ld index 634b5c2cab..5c00859c41 100644 --- a/linkers/riscv64.ld +++ b/linkers/riscv64.ld @@ -27,9 +27,6 @@ SECTIONS { *(.early_init.text*) . = ALIGN(4096); *(.text*) - __usercopy_start = .; - *(.usercopy-fns) - __usercopy_end = .; . = ALIGN(4096); __text_end = .; } diff --git a/linkers/x86_64.ld b/linkers/x86_64.ld index 6610509aef..00b75b1229 100644 --- a/linkers/x86_64.ld +++ b/linkers/x86_64.ld @@ -23,9 +23,6 @@ SECTIONS { .text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) { __text_start = .; *(.text*) - __usercopy_start = .; - *(.usercopy-fns) - __usercopy_end = .; } .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 300fc8a85c..1e9e203328 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -37,11 +37,12 @@ pub use ::rmm::AArch64Arch as CurrentRmmArch; pub use arch_copy_to_user as arch_copy_from_user; #[unsafe(naked)] -#[unsafe(link_section = ".usercopy-fns")] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { // x0, x1, x2 core::arch::naked_asm!( " + .global __usercopy_start + __usercopy_start: mov x4, x0 mov x0, 0 2: @@ -58,6 +59,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - b 2b 3: ret + .global __usercopy_end + __usercopy_end: " ); } diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs index 2cfee5c263..6893aa44d8 100644 --- a/src/arch/riscv64/mod.rs +++ b/src/arch/riscv64/mod.rs @@ -15,11 +15,12 @@ use core::arch::naked_asm; pub use arch_copy_to_user as arch_copy_from_user; -#[unsafe(link_section = ".usercopy-fns")] #[unsafe(naked)] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { naked_asm!( " + .global __usercopy_start + __usercopy_start: addi sp, sp, -16 sd fp, 0(sp) sd ra, 8(sp) @@ -55,6 +56,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - bne a2, x0, 4b 5: mv a0, x0 ret + .global __usercopy_end + __usercopy_end: " ) } diff --git a/src/arch/x86/mod.rs b/src/arch/x86/mod.rs index fe3c36ed35..3fe8d7c52b 100644 --- a/src/arch/x86/mod.rs +++ b/src/arch/x86/mod.rs @@ -17,10 +17,11 @@ pub mod flags { } #[unsafe(naked)] -#[unsafe(link_section = ".usercopy-fns")] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { core::arch::naked_asm!( " + .global __usercopy_start + __usercopy_start: push edi push esi @@ -34,6 +35,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - xor eax, eax ret + .global __usercopy_end + __usercopy_end: " ); } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 40417aeb61..a43557ec0d 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -24,27 +24,32 @@ pub mod flags { // TODO: Maybe support rewriting relocations (using LD's --emit-relocs) when working with entire // functions? #[unsafe(naked)] -#[unsafe(link_section = ".usercopy-fns")] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { // TODO: spectre_v1 - core::arch::naked_asm!(alternative!( - feature: "smap", - then: [" - xor eax, eax - mov rcx, rdx - stac - rep movsb - clac - ret - "], - default: [" - xor eax, eax - mov rcx, rdx - rep movsb - ret - "] - )); + core::arch::naked_asm!( + ".global __usercopy_start + __usercopy_start:", + alternative!( + feature: "smap", + then: [" + xor eax, eax + mov rcx, rdx + stac + rep movsb + clac + ret + "], + default: [" + xor eax, eax + mov rcx, rdx + rep movsb + ret + "] + ), + ".global __usercopy_end + __usercopy_end:" + ); } pub use arch_copy_to_user as arch_copy_from_user;