relibc: cap mmap_min_addr at canonical user-space top

ROOT CAUSE: fexec_impl maps the 8 MB stack at STACK_TOP - STACK_SIZE =
0x7FFFFFF800000 (with STACK_TOP = 1 << 47 = 0x800000000000 on x86_64).
The update_min_mmap_addr closure computes (addr + size).next_multiple_of(PAGE_SIZE)
= 0x800000000000 as the new mmap_min. But 0x800000000000 is the FIRST
non-canonical address on x86-64 — the canonical lower half ends at
0x7FFFFFFFFFFF. With mmap_min = 0x800000000000, every subsequent anonymous
mmap fails because find_free_near can't find any hole at or above
0x800000000000.

This breaks ld.so's stage2 Tcb::new(0) call (8 KB anonymous mmap),
which uses .expect_notls() and panics → ud2. The init process crashes
before main() with 'Invalid opcode fault' at RIP 0x21cd1.

FIX: Add USER_CANONICAL_TOP_PAGE constant per architecture and cap
mmap_min_addr at that value. On x86_64/aarch64 this is 0x7FFFFFFFF000
(last page in the 48-bit canonical lower half). On riscv64-sv39 this
is (1 << 38) - 0x1000. On i686 this is (1 << 31) - 0x1000.

redox-rt/src/arch/aarch64.rs

@@ -14,6 +14,7 @@ use super::ForkScratchpad;
 // Setup a stack starting from the very end of the address space, and then growing downwards.
 pub const STACK_TOP: usize = 1 << 47;
 pub const STACK_SIZE: usize = 1024 * 1024;
+pub const USER_CANONICAL_TOP_PAGE: usize = 0x7FFFFFFFF000;
 
 #[derive(Debug, Default)]
 #[repr(C)]

redox-rt/src/arch/i686.rs

@@ -13,6 +13,7 @@ use super::ForkScratchpad;
 // Setup a stack starting from the very end of the address space, and then growing downwards.
 pub const STACK_TOP: usize = 1 << 31;
 pub const STACK_SIZE: usize = 1024 * 1024;
+pub const USER_CANONICAL_TOP_PAGE: usize = (1 << 31) - 0x1000;
 
 #[derive(Debug, Default)]
 #[repr(C)]

redox-rt/src/arch/riscv64.rs

@@ -14,6 +14,7 @@ use super::ForkScratchpad;
 // Setup a stack starting from the very end of the address space, and then growing downwards.
 pub const STACK_TOP: usize = 1 << 38;
 pub const STACK_SIZE: usize = 1024 * 1024;
+pub const USER_CANONICAL_TOP_PAGE: usize = (1 << 38) - 0x1000;
 
 #[derive(Debug, Default)]
 #[repr(C)]

redox-rt/src/arch/x86_64.rs

@@ -23,6 +23,12 @@ use super::ForkScratchpad;
 pub const STACK_TOP: usize = 1 << 47;
 pub const STACK_SIZE: usize = 8 * 1024 * 1024;
 
+/// Last valid canonical user-space page (x86-64 canonical lower half ends at
+/// 0x7FFFFFFFFFFF, so the last page starts at 0x7FFFFFFFF000). Used to cap
+/// `mmap_min_addr` so that mappings near the top of the address space don't
+/// push it into the non-canonical hole.
+pub const USER_CANONICAL_TOP_PAGE: usize = 0x7FFFFFFFF000;
+
 #[derive(Debug, Default)]
 #[repr(C)]
 pub struct SigArea {

redox-rt/src/proc.rs

@@ -111,7 +111,16 @@ pub fn fexec_impl(
         .map(|interp| interp.min_mmap_addr)
         .unwrap_or(PAGE_SIZE);
     let mut update_min_mmap_addr = |addr: usize, size: usize| {
-        min_mmap_addr = cmp::max(min_mmap_addr, (addr + size).next_multiple_of(PAGE_SIZE));
+        let end = (addr + size).next_multiple_of(PAGE_SIZE);
+        // Cap at the last valid canonical user-space page (x86-64 canonical
+        // lower half ends at 0x7FFFFFFFFFFF). Without this cap, mapping the
+        // 8 MB stack at STACK_TOP - STACK_SIZE = 0x7FFFFFF800000 would set
+        // mmap_min to 0x800000000000, which is non-canonical and causes every
+        // subsequent anonymous mmap (e.g. Tcb::new in ld.so stage2) to fail
+        // with ENOMEM.
+        let canonical_top_page = crate::arch::USER_CANONICAL_TOP_PAGE;
+        let end = cmp::min(end, canonical_top_page);
+        min_mmap_addr = cmp::max(min_mmap_addr, end);
     };
 
     pread_all(&image_file, u64::from(header.e_phoff), phs).map_err(|_| Error::new(EIO))?;