Merge branch 'dlmalloc' into 'master'

Replace C dlmalloc

See merge request redox-os/relibc!430

.gitmodules

@@ -10,10 +10,10 @@
 [submodule "posix-regex"]
 	path = posix-regex
 	url = https://gitlab.redox-os.org/redox-os/posix-regex.git
-[submodule "pthreads-emb"]
-	path = pthreads-emb
-	url = https://gitlab.redox-os.org/redox-os/pthreads-emb.git
 [submodule "compiler-builtins"]
 	path = compiler-builtins
 	url = https://gitlab.redox-os.org/redox-os/compiler-builtins.git
 	branch = relibc_fix_dup_symbols
+[submodule "src/dlmalloc-rs"]
+	path = dlmalloc-rs
+	url = https://gitlab.redox-os.org/redox-os/dlmalloc-rs.git

Cargo.lock

@@ -117,6 +117,10 @@ version = "0.1.0"
 name = "crtn"
 version = "0.1.0"
 
+[[package]]
+name = "dlmalloc"
+version = "0.2.4"
+
 [[package]]
 name = "errno"
 version = "0.3.5"
@@ -371,6 +375,7 @@ dependencies = [
  "cbitset",
  "cc",
  "core_io",
+ "dlmalloc",
  "goblin",
  "libc",
  "memchr",

Cargo.toml

@@ -10,7 +10,7 @@ crate-type = ["staticlib"]
 
 [workspace]
 members = ["src/crt0", "src/crti", "src/crtn", "src/ld_so", "src/platform/redox/redox-exec"]
-exclude = ["core_io", "ralloc", "tests"]
+exclude = ["core_io", "ralloc", "tests", "dlmalloc-rs"]
 
 [build-dependencies]
 cbindgen = "0.26"
@@ -44,6 +44,11 @@ path = "ralloc"
 default-features = false
 optional = true
 
+[dependencies.dlmalloc]
+path = "dlmalloc-rs"
+default-features = false
+features = ["c_api"]
+
 [target.'cfg(target_os = "linux")'.dependencies]
 sc = "0.2.3"
 

Xargo.toml

@@ -1 +0,0 @@
-[dependencies.alloc]

src/header/sys_mman/mod.rs

@@ -26,6 +26,9 @@ pub const MAP_TYPE: c_int = 0x000F;
 pub const MAP_ANON: c_int = 0x0020;
 pub const MAP_ANONYMOUS: c_int = MAP_ANON;
 pub const MAP_STACK: c_int = 0x20000;
+pub const MAP_FAILED: *mut c_void = usize::wrapping_neg(1) as *mut c_void;
+
+pub const MREMAP_MAYMOVE: c_int = 1;
 
 pub const MS_ASYNC: c_int = 0x0001;
 pub const MS_INVALIDATE: c_int = 0x0002;
@@ -62,6 +65,18 @@ pub unsafe extern "C" fn mmap(
     Sys::mmap(addr, len, prot, flags, fildes, off)
 }
 
+#[no_mangle]
+unsafe extern "C" fn mremap(
+    old_address: *mut c_void,
+    old_size: usize,
+    new_size: usize,
+    flags: c_int,
+    mut args: ...
+) -> *mut c_void {
+    let new_address = args.arg::<*mut c_void>();
+    Sys::mremap(old_address, old_size, new_size, flags, new_address)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn mprotect(addr: *mut c_void, len: size_t, prot: c_int) -> c_int {
     Sys::mprotect(addr, len, prot)

src/ld_so/start.rs

@@ -11,7 +11,7 @@ use alloc::{
 use crate::{
     c_str::CStr,
     header::unistd,
-    platform::{get_auxv, get_auxvs, new_mspace, types::c_char},
+    platform::{get_auxv, get_auxvs, types::c_char},
     start::Stack,
     sync::mutex::Mutex,
     ALLOCATOR,
@@ -143,9 +143,7 @@ fn resolve_path_name(
 }
 #[no_mangle]
 pub extern "C" fn relibc_ld_so_start(sp: &'static mut Stack, ld_entry: usize) -> usize {
-    // First thing we initialize the mspace
-    ALLOCATOR.set_book_keeper(new_mspace());
-    // next we get the arguments, the environment, and the auxilary vector
+    // We get the arguments, the environment, and the auxilary vector
     let (argv, envs, auxv) = unsafe {
         let argv_start = sp.argv() as *mut usize;
         let (argv, argv_end) = get_argv(argv_start);
@@ -241,7 +239,7 @@ pub extern "C" fn relibc_ld_so_start(sp: &'static mut Stack, ld_entry: usize) ->
     };
     if let Some(tcb) = unsafe { Tcb::current() } {
         tcb.linker_ptr = Box::into_raw(Box::new(Mutex::new(linker)));
-        tcb.mspace = ALLOCATOR.get_book_keeper();
+        tcb.mspace = ALLOCATOR.get();
     }
     if is_manual {
         eprintln!("ld.so: entry '{}': {:#x}", path, entry);

src/ld_so/tcb.rs

@@ -6,7 +6,7 @@ use super::ExpectTlsFree;
 use crate::{
     header::sys_mman,
     ld_so::linker::Linker,
-    platform::{Pal, Sys},
+    platform::{Dlmalloc, Pal, Sys},
     sync::mutex::Mutex,
 };
 
@@ -48,7 +48,7 @@ pub struct Tcb {
     /// Pointer to dynamic linker
     pub linker_ptr: *const Mutex<Linker>,
     /// pointer to rust memory allocator structure
-    pub mspace: usize,
+    pub mspace: *const Mutex<Dlmalloc>,
 }
 
 impl Tcb {
@@ -70,7 +70,7 @@ impl Tcb {
                 masters_len: 0,
                 num_copied_masters: 0,
                 linker_ptr: ptr::null(),
-                mspace: 0,
+                mspace: ptr::null(),
             },
         );
 

src/platform/allocator/dlmalloc.rs

@@ -1,67 +0,0 @@
-use crate::ALLOCATOR;
-use core::{
-    alloc::{GlobalAlloc, Layout},
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
-use super::types::*;
-
-extern "C" {
-    fn create_mspace(capacity: size_t, locked: c_int) -> usize;
-    fn mspace_malloc(msp: usize, bytes: size_t) -> *mut c_void;
-    fn mspace_memalign(msp: usize, alignment: size_t, bytes: size_t) -> *mut c_void;
-    fn mspace_realloc(msp: usize, oldmem: *mut c_void, bytes: size_t) -> *mut c_void;
-    fn mspace_free(msp: usize, mem: *mut c_void);
-    //fn dlmalloc(bytes: size_t) -> *mut c_void;
-    //fn dlmemalign(alignment: size_t, bytes: size_t) -> *mut c_void;
-    //fn dlrealloc(oldmem: *mut c_void, bytes: size_t) -> *mut c_void;
-    //fn dlfree(mem: *mut c_void);
-}
-
-pub struct Allocator {
-    mstate: AtomicUsize,
-}
-
-pub const NEWALLOCATOR: Allocator = Allocator {
-    mstate: AtomicUsize::new(0),
-};
-
-impl Allocator {
-    pub fn set_book_keeper(&self, mstate: usize) {
-        self.mstate.store(mstate, Ordering::Relaxed);
-    }
-    pub fn get_book_keeper(&self) -> usize {
-        self.mstate.load(Ordering::Relaxed)
-    }
-}
-unsafe impl<'a> GlobalAlloc for Allocator {
-    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
-        alloc_align(layout.size(), layout.align()) as *mut u8
-    }
-
-    unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
-        free(ptr as *mut c_void)
-    }
-}
-
-pub unsafe fn alloc(size: usize) -> *mut c_void {
-    mspace_malloc(ALLOCATOR.get_book_keeper(), size)
-}
-
-pub unsafe fn alloc_align(size: usize, alignment: usize) -> *mut c_void {
-    mspace_memalign(ALLOCATOR.get_book_keeper(), alignment, size)
-}
-
-pub unsafe fn realloc(ptr: *mut c_void, size: size_t) -> *mut c_void {
-    mspace_realloc(ALLOCATOR.get_book_keeper(), ptr, size)
-}
-
-pub unsafe fn free(ptr: *mut c_void) {
-    mspace_free(ALLOCATOR.get_book_keeper(), ptr)
-}
-
-pub fn new_mspace() -> usize {
-    let capacity = 0; // don't specify capacity explicitly
-    let locked = 1;
-    unsafe { create_mspace(capacity, locked) }
-}

src/platform/allocator/dlmalloc/mod.rs

@@ -0,0 +1,99 @@
+use core::{
+    alloc::{GlobalAlloc, Layout},
+    cell::SyncUnsafeCell,
+    cmp,
+    mem::align_of,
+    ptr::{copy_nonoverlapping, write_bytes},
+};
+
+mod sys;
+use super::types::*;
+use crate::{sync::Mutex, ALLOCATOR};
+use dlmalloc::DlmallocCApi;
+
+pub type Dlmalloc = DlmallocCApi<sys::System>;
+
+pub const NEWALLOCATOR: Allocator = Allocator::new();
+
+pub struct Allocator(SyncUnsafeCell<Mutex<Dlmalloc>>);
+
+impl Allocator {
+    pub const fn new() -> Self {
+        Allocator(SyncUnsafeCell::new(Mutex::new(Dlmalloc::new(
+            sys::System::new(),
+        ))))
+    }
+
+    pub fn get(&self) -> *mut Mutex<Dlmalloc> {
+        self.0.get()
+    }
+}
+
+unsafe impl GlobalAlloc for Allocator {
+    #[inline]
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        if layout.align() <= align_of::<max_align_t>() {
+            (*self.get()).lock().malloc(layout.size())
+        } else {
+            (*self.get()).lock().memalign(layout.align(), layout.size())
+        }
+    }
+
+    #[inline]
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        (*self.get()).lock().free(ptr)
+    }
+
+    #[inline]
+    unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
+        let ptr = self.alloc(layout);
+        if !ptr.is_null() && (*self.get()).lock().calloc_must_clear(ptr) {
+            write_bytes(ptr, 0, layout.size());
+        }
+        ptr
+    }
+
+    #[inline]
+    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+        if layout.align() <= align_of::<max_align_t>() {
+            (*self.get()).lock().realloc(ptr, new_size)
+        } else {
+            let new = self.alloc(Layout::from_size_align_unchecked(new_size, layout.align()));
+            let old_size = layout.size();
+            let old_align = layout.align();
+
+            if !new.is_null() {
+                let size = cmp::min(old_size, new_size);
+                copy_nonoverlapping(ptr, new, size);
+            }
+
+            drop((old_size, old_align));
+            (*self.get()).lock().free(ptr);
+
+            new
+        }
+    }
+}
+
+pub unsafe fn alloc(size: size_t) -> *mut c_void {
+    (*ALLOCATOR.get()).lock().malloc(size).cast()
+}
+
+pub unsafe fn alloc_align(size: size_t, alignment: size_t) -> *mut c_void {
+    (*ALLOCATOR.get()).lock().memalign(alignment, size).cast()
+}
+
+pub unsafe fn realloc(ptr: *mut c_void, size: size_t) -> *mut c_void {
+    if ptr.is_null() {
+        (*ALLOCATOR.get()).lock().malloc(size).cast()
+    } else {
+        (*ALLOCATOR.get()).lock().realloc(ptr.cast(), size).cast()
+    }
+}
+
+pub unsafe fn free(ptr: *mut c_void) {
+    if ptr.is_null() {
+        return;
+    }
+    (*ALLOCATOR.get()).lock().free(ptr.cast())
+}

src/platform/allocator/dlmalloc/sys.rs

@@ -0,0 +1,130 @@
+use crate::{
+    header::{
+        sys_mman::{self, MAP_FAILED, MREMAP_MAYMOVE},
+        unistd::pthread_atfork,
+    },
+    platform::{types::*, Pal, Sys},
+    sync::Mutex,
+};
+use core::ptr;
+
+use dlmalloc::Allocator;
+
+/// System setting for Redox/Linux
+pub struct System {
+    _priv: (),
+}
+
+impl System {
+    pub const fn new() -> System {
+        System { _priv: () }
+    }
+}
+
+static LOCK: Mutex<()> = Mutex::new(());
+
+unsafe impl Allocator for System {
+    fn alloc(&self, size: usize) -> (*mut u8, usize, u32) {
+        let addr = unsafe {
+            Sys::mmap(
+                0 as *mut _,
+                size,
+                sys_mman::PROT_WRITE | sys_mman::PROT_READ,
+                sys_mman::MAP_ANON | sys_mman::MAP_PRIVATE,
+                -1,
+                0,
+            )
+        };
+        if addr as *mut c_void == MAP_FAILED {
+            (ptr::null_mut(), 0, 0)
+        } else {
+            (addr as *mut u8, size, 0)
+        }
+    }
+
+    fn remap(&self, ptr: *mut u8, oldsize: usize, newsize: usize, can_move: bool) -> *mut u8 {
+        let flags = if can_move { MREMAP_MAYMOVE } else { 0 };
+        let ptr = unsafe { Sys::mremap(ptr as *mut _, oldsize, newsize, flags, ptr::null_mut()) };
+        if ptr as *mut c_void == MAP_FAILED {
+            ptr::null_mut()
+        } else {
+            ptr as *mut u8
+        }
+    }
+
+    fn free_part(&self, ptr: *mut u8, oldsize: usize, newsize: usize) -> bool {
+        unsafe {
+            let rc = Sys::mremap(ptr as *mut _, oldsize, newsize, 0, ptr::null_mut());
+            if rc as *mut c_void != MAP_FAILED {
+                return true;
+            }
+            Sys::munmap(ptr.offset(newsize as isize) as *mut _, oldsize - newsize) == 0
+        }
+    }
+
+    fn free(&self, ptr: *mut u8, size: usize) -> bool {
+        unsafe { Sys::munmap(ptr as *mut _, size) == 0 }
+    }
+
+    fn can_release_part(&self, _flags: u32) -> bool {
+        true
+    }
+
+    fn allocates_zeros(&self) -> bool {
+        true
+    }
+
+    fn page_size(&self) -> usize {
+        4096
+    }
+}
+
+pub fn acquire_global_lock() {
+    unsafe {
+        // SAFETY: No data inside
+        LOCK.manual_lock();
+    }
+}
+
+pub(super) fn release_global_lock() {
+    unsafe {
+        // SAFETY: No data inside
+        LOCK.manual_unlock();
+    }
+}
+
+/// Allows the allocator to remain unsable in the child process,
+/// after a call to `fork(2)`
+///
+/// # Safety
+///
+/// if used, this function must be called,
+/// before any allocations are made with the global allocator.
+pub unsafe fn enable_alloc_after_fork() {
+    // atfork must only be called once, to avoid a deadlock,
+    // where the handler attempts to acquire the global lock twice
+    static mut FORK_PROTECTED: bool = false;
+
+    extern "C" fn _acquire_global_lock() {
+        acquire_global_lock()
+    }
+
+    extern "C" fn _release_global_lock() {
+        release_global_lock()
+    }
+
+    acquire_global_lock();
+    // if a process forks,
+    // it will acquire the lock before any other thread,
+    // protecting it from deadlock,
+    // due to the child being created with only the calling thread.
+    if !FORK_PROTECTED {
+        pthread_atfork(
+            Some(_acquire_global_lock),
+            Some(_release_global_lock),
+            Some(_release_global_lock),
+        );
+        FORK_PROTECTED = true;
+    }
+    release_global_lock();
+}

src/platform/allocator/ralloc.rs

@@ -2,6 +2,8 @@ extern crate ralloc;
 
 pub use ralloc::Allocator;
 
+pub const NEWALLOCATOR: Allocator = Allocator;
+
 unsafe fn alloc_inner(size: usize, offset: usize, align: usize) -> *mut c_void {
     let ptr = ralloc::alloc(size + offset, align);
     if !ptr.is_null() {

src/platform/linux/mod.rs

@@ -380,6 +380,16 @@ impl Pal for Sys {
         e(syscall!(MMAP, addr, len, prot, flags, fildes, off)) as *mut c_void
     }
 
+    unsafe fn mremap(
+        addr: *mut c_void,
+        len: usize,
+        new_len: usize,
+        flags: c_int,
+        args: *mut c_void,
+    ) -> *mut c_void {
+        e(syscall!(MREMAP, addr, len, new_len, flags, args)) as *mut c_void
+    }
+
     unsafe fn mprotect(addr: *mut c_void, len: usize, prot: c_int) -> c_int {
         e(syscall!(MPROTECT, addr, len, prot)) as c_int
     }
@@ -428,7 +438,7 @@ impl Pal for Sys {
 
             # Check if child or parent
             test rax, rax
-            jnz 1f
+            jnz 2f
 
             # Load registers
             pop rax
@@ -451,7 +461,7 @@ impl Pal for Sys {
             ud2
 
             # Return PID if parent
-            1:
+            2:
             ",
             inout("rax") SYS_CLONE => pid,
             inout("rdi") flags => _,

src/platform/mod.rs

@@ -5,7 +5,7 @@ use core::{fmt, ptr};
 pub use self::allocator::*;
 
 #[cfg(not(feature = "ralloc"))]
-#[path = "allocator/dlmalloc.rs"]
+#[path = "allocator/dlmalloc/mod.rs"]
 mod allocator;
 
 #[cfg(feature = "ralloc")]

src/platform/pal/mod.rs

@@ -151,6 +151,14 @@ pub trait Pal {
         off: off_t,
     ) -> *mut c_void;
 
+    unsafe fn mremap(
+        addr: *mut c_void,
+        len: usize,
+        new_len: usize,
+        flags: c_int,
+        args: *mut c_void,
+    ) -> *mut c_void;
+
     unsafe fn mprotect(addr: *mut c_void, len: usize, prot: c_int) -> c_int;
 
     unsafe fn msync(addr: *mut c_void, len: usize, flags: c_int) -> c_int;

src/platform/redox/mod.rs

@@ -1,5 +1,4 @@
 use core::{convert::TryFrom, mem, ptr, result::Result as CoreResult, slice, str};
-
 use syscall::{
     self,
     data::{Map, Stat as redox_stat, StatVfs as redox_statvfs, TimeSpec as redox_timespec},
@@ -13,7 +12,7 @@ use crate::{
         dirent::dirent,
         errno::{EINVAL, EIO, ENOMEM, ENOSYS, EPERM, ERANGE},
         fcntl,
-        sys_mman::{MAP_ANONYMOUS, PROT_READ, PROT_WRITE},
+        sys_mman::{MAP_ANONYMOUS, MAP_FAILED, PROT_READ, PROT_WRITE},
         sys_random,
         sys_resource::{rlimit, RLIM_INFINITY},
         sys_stat::{stat, S_ISGID, S_ISUID},
@@ -661,6 +660,16 @@ impl Pal for Sys {
         }
     }
 
+    unsafe fn mremap(
+        addr: *mut c_void,
+        len: usize,
+        new_len: usize,
+        flags: c_int,
+        args: *mut c_void,
+    ) -> *mut c_void {
+        MAP_FAILED
+    }
+
     unsafe fn mprotect(addr: *mut c_void, len: usize, prot: c_int) -> c_int {
         e(syscall::mprotect(
             addr as usize,

src/platform/types.rs

@@ -78,3 +78,8 @@ pub type clockid_t = c_int;
 pub type timer_t = *mut c_void;
 
 pub use crate::header::{bits_pthread::*, bits_sched::*};
+
+#[repr(C, align(16))]
+pub struct max_align_t {
+    _priv: [f64; 4],
+}

src/pthread/mod.rs

@@ -15,7 +15,6 @@ use crate::{
         tcb::{Master, Tcb},
     },
     platform::{types::*, Pal, Sys},
-    ALLOCATOR,
 };
 
 use crate::sync::{waitval::Waitval, Mutex};
@@ -159,6 +158,14 @@ pub(crate) unsafe fn create(
         mmap_size: stack_size,
     };
 
+    let current_tcb = Tcb::current().expect("no TCB!");
+    let new_tcb = Tcb::new(current_tcb.tls_len).map_err(|_| Errno(ENOMEM))?;
+
+    new_tcb.masters_ptr = current_tcb.masters_ptr;
+    new_tcb.masters_len = current_tcb.masters_len;
+    new_tcb.linker_ptr = current_tcb.linker_ptr;
+    new_tcb.mspace = current_tcb.mspace;
+
     let stack_end = stack_base.add(stack_size);
     let mut stack = stack_end as *mut usize;
     {
@@ -168,24 +175,9 @@ pub(crate) unsafe fn create(
         };
 
         push(0);
-        push(ptr as usize);
-
-        //WARNING: Stack must be 128-bit aligned for SSE
-        if let Some(tcb) = Tcb::current() {
-            push(tcb.mspace as usize);
-            push(tcb.linker_ptr as usize);
-            push(tcb.masters_len);
-            push(tcb.masters_ptr as usize);
-            push(tcb.tls_len);
-        } else {
-            push(ALLOCATOR.get_book_keeper());
-            push(0);
-            push(0);
-            push(0);
-            push(0);
-        }
-
         push(synchronization_mutex as usize);
+        push(ptr as usize);
+        push(new_tcb as *mut _ as usize);
 
         push(arg as usize);
         push(start_routine as usize);
@@ -210,21 +202,11 @@ pub(crate) unsafe fn create(
 unsafe extern "C" fn new_thread_shim(
     entry_point: unsafe extern "C" fn(*mut c_void) -> *mut c_void,
     arg: *mut c_void,
-    mutex: *const Mutex<()>,
-    tls_size: usize,
-    tls_masters_ptr: *mut Master,
-    tls_masters_len: usize,
-    tls_linker_ptr: *const Mutex<Linker>,
-    tls_mspace: usize,
+    tcb: *mut Tcb,
     pthread: *mut Pthread,
+    mutex: *const Mutex<()>,
 ) -> ! {
-    // TODO: Pass less arguments by allocating the TCB from the creator thread.
-    if !tls_masters_ptr.is_null() {
-        let tcb = Tcb::new(tls_size).unwrap();
-        tcb.masters_ptr = tls_masters_ptr;
-        tcb.masters_len = tls_masters_len;
-        tcb.linker_ptr = tls_linker_ptr;
-        tcb.mspace = tls_mspace;
+    if let Some(tcb) = tcb.as_mut() {
         tcb.copy_masters().unwrap();
         tcb.activate();
     }

src/start.rs

@@ -4,7 +4,7 @@ use core::{intrinsics, ptr};
 use crate::{
     header::{libgen, stdio, stdlib},
     ld_so::{self, linker::Linker},
-    platform::{self, get_auxvs, new_mspace, types::*, Pal, Sys},
+    platform::{self, get_auxvs, types::*, Pal, Sys},
     sync::mutex::Mutex,
     ALLOCATOR,
 };
@@ -80,13 +80,9 @@ fn alloc_init() {
     }
     unsafe {
         if let Some(tcb) = ld_so::tcb::Tcb::current() {
-            if tcb.mspace != 0 {
-                ALLOCATOR.set_book_keeper(tcb.mspace);
-            } else if ALLOCATOR.get_book_keeper() == 0 {
-                ALLOCATOR.set_book_keeper(new_mspace());
+            if !tcb.mspace.is_null() {
+                ALLOCATOR.get().write(tcb.mspace.read());
             }
-        } else if ALLOCATOR.get_book_keeper() == 0 {
-            ALLOCATOR.set_book_keeper(new_mspace());
         }
     }
 }
@@ -180,7 +176,7 @@ pub unsafe extern "C" fn relibc_start(sp: &'static Stack) -> ! {
 
     if let Some(tcb) = ld_so::tcb::Tcb::current() {
         // Update TCB mspace
-        tcb.mspace = ALLOCATOR.get_book_keeper();
+        tcb.mspace = ALLOCATOR.get();
 
         // Set linker pointer if necessary
         if tcb.linker_ptr.is_null() {

tests/wchar/wcsnrtombs.c

@@ -7,7 +7,7 @@ int main() {
     setlocale(LC_ALL, "");
 
     const wchar_t *src = L"こんにちは世界Привет мир";
-    char dst[20];
+    char dst[20] = {0};
     mbstate_t ps;
 
     size_t result = wcsnrtombs(dst, &src, 4, sizeof(dst), &ps);