diff --git a/ld_so/ld_script/x86_64-unknown-linux-gnu.ld b/ld_so/ld_script/x86_64-unknown-linux-gnu.ld index 389ddebc1e..c67cb55af5 100644 --- a/ld_so/ld_script/x86_64-unknown-linux-gnu.ld +++ b/ld_so/ld_script/x86_64-unknown-linux-gnu.ld @@ -101,12 +101,14 @@ SECTIONS .gcc_except_table : ONLY_IF_RW { *(.gcc_except_table .gcc_except_table.*) } .exception_ranges : ONLY_IF_RW { *(.exception_ranges*) } /* Thread Local Storage sections */ - .tdata : + __relibc_ldso_tls_start = .; + .tdata : ALIGN(4K) { PROVIDE_HIDDEN (__tdata_start = .); *(.tdata .tdata.* .gnu.linkonce.td.*) } .tbss : { *(.tbss .tbss.* .gnu.linkonce.tb.*) *(.tcommon) } + __relibc_ldso_tls_end = .; .preinit_array : { PROVIDE_HIDDEN (__preinit_array_start = .); diff --git a/ld_so/ld_script/x86_64-unknown-redox.ld b/ld_so/ld_script/x86_64-unknown-redox.ld index 68c922d374..00df435090 100644 --- a/ld_so/ld_script/x86_64-unknown-redox.ld +++ b/ld_so/ld_script/x86_64-unknown-redox.ld @@ -98,12 +98,14 @@ SECTIONS .gcc_except_table : ONLY_IF_RW { *(.gcc_except_table .gcc_except_table.*) } .exception_ranges : ONLY_IF_RW { *(.exception_ranges*) } /* Thread Local Storage sections */ - .tdata : + __relibc_ldso_tls_start = .; + .tdata : ALIGN(4K) { PROVIDE_HIDDEN (__tdata_start = .); *(.tdata .tdata.* .gnu.linkonce.td.*) } .tbss : { *(.tbss .tbss.* .gnu.linkonce.tb.*) *(.tcommon) } + __relibc_ldso_tls_end = .; .preinit_array : { PROVIDE_HIDDEN (__preinit_array_start = .); diff --git a/ld_so/src/lib.rs b/ld_so/src/lib.rs index 8f17a7fc01..9ba54a23c2 100644 --- a/ld_so/src/lib.rs +++ b/ld_so/src/lib.rs @@ -45,6 +45,8 @@ _start: # Call ld_so_start(stack, entry) mov rdi, rbp sub rsi, 5 + lea rdx, __relibc_ldso_tls_start + lea rcx, __relibc_ldso_tls_end call relibc_ld_so_start # Restore original stack, clear registers, and jump to new start function diff --git a/src/header/dl-tls/mod.rs b/src/header/dl-tls/mod.rs index 0ca6aab3c4..c5036b2d43 100644 --- a/src/header/dl-tls/mod.rs +++ b/src/header/dl-tls/mod.rs @@ -1,5 +1,9 @@ //! dl-tls implementation for Redox +use core::alloc::Layout; + +use alloc::alloc::alloc_zeroed; + use crate::{ld_so::tcb::Tcb, platform::types::*}; #[repr(C)] @@ -10,53 +14,58 @@ pub struct dl_tls_index { #[no_mangle] pub unsafe extern "C" fn __tls_get_addr(ti: *mut dl_tls_index) -> *mut c_void { + let tcb = Tcb::current().unwrap(); + trace!( - "__tls_get_addr({:p}: {:#x}, {:#x})", + "__tls_get_addr({:p}: {:#x}, {:#x}, masters_len={}, dtv_len={})", ti, (*ti).ti_module, - (*ti).ti_offset + (*ti).ti_offset, + tcb.masters().unwrap().len(), + tcb.dtv_mut().len() ); - let mod_id = (*ti).ti_module as usize; - let offset = if cfg!(target_arch = "riscv64") { - ((*ti).ti_offset as usize).wrapping_add(0x800) // dynamic offsets are 0x800-based on risc-v - } else { - (*ti).ti_offset as usize - }; - if mod_id > 0 { - if let Some(tcb) = Tcb::current() { - if let Some(masters) = tcb.masters() { - if let Some(master) = masters.get(mod_id - 1) { - // module id is 1-based - let addr = if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { - tcb.tls_end.sub(master.offset).add(offset) - } else { - // FIXME aarch64/risc-v only support static master - if mod_id == 1 && offset < tcb.tls_len { - tcb.tls_end.sub(tcb.tls_len).add(offset) - } else { - 0 as *mut u8 - } - }; - if !addr.is_null() { - trace!( - "__tls_get_addr({:p}: {:#x}, {:#x}) = {:p}", - ti, - (*ti).ti_module, - (*ti).ti_offset, - addr - ); - return addr as *mut c_void; - } - } - } - } + + if tcb.dtv_mut().unwrap_or_default().len() < tcb.masters().unwrap().len() { + // Reallocate DTV. + tcb.setup_dtv(tcb.masters().unwrap().len()); } - panic!( - "__tls_get_addr({:p}: {:#x}, {:#x}) failed", - ti, - (*ti).ti_module, - (*ti).ti_offset - ); + + let ti = &*ti; + let dtv_index = ti.ti_module as usize - 1; + + if tcb.dtv_mut().unwrap()[dtv_index].is_null() { + // Allocate TLS for module. + let master = &tcb.masters().unwrap()[ti.ti_module as usize - 1]; + + // FIXME(andypython): master.align + let layout = unsafe { + Layout::from_size_align_unchecked(master.offset /* aligned ph.p_memsz */, 16) + }; + + let module_tls = alloc_zeroed(layout); + + core::ptr::copy_nonoverlapping(master.ptr, module_tls, master.len); + + // Set the DTV entry. + tcb.dtv_mut().unwrap()[dtv_index] = module_tls; + } + + let mut ptr = tcb.dtv_mut().unwrap()[dtv_index]; + + if cfg!(target_arch = "riscv64") { + ptr = ptr.add(0x800 + ti.ti_offset as usize); // dynamic offsets are 0x800-based on risc-v + } else { + ptr = ptr.add(ti.ti_offset as usize); + } + + if ptr.is_null() { + panic!( + "__tls_get_addr({ti:p}: {:#x}, {:#x})", + ti.ti_module, ti.ti_offset + ); + } + + ptr.cast::() } // x86 can define a version that does not require stack alignment diff --git a/src/ld_so/linker.rs b/src/ld_so/linker.rs index 5bdccabe42..ecfc1f0172 100644 --- a/src/ld_so/linker.rs +++ b/src/ld_so/linker.rs @@ -60,7 +60,7 @@ const root_id: usize = 1; impl Linker { pub fn new(ld_library_path: Option) -> Self { Self { - ld_library_path: ld_library_path, + ld_library_path, next_object_id: root_id, next_tls_module_id: 1, tls_size: 0, @@ -70,6 +70,7 @@ impl Linker { } } + /// **Warning**: Switches to the program's TCB. Thread locals should not be accessed after this. pub fn load_program(&mut self, path: &str, base_addr: Option) -> Result { self.load_object(path, &None, base_addr, false)?; return Ok(self.objects.get(&root_id).unwrap().entry_point); @@ -167,13 +168,41 @@ impl Linker { )?; unsafe { - let tcb = match Tcb::current() { - Some(some) => some, - None => Tcb::new(self.tls_size)?, - }; - tcb.append_masters(tcb_masters); - tcb.copy_masters()?; - tcb.activate(); + if !dlopened { + // We are now loading the main program or its dependencies. The TLS for all initially + // loaded objects reside in the static TLS block. Depending on the architecture, the + // static TLS block is either placed before the TP or after the TP. + let tcb = Tcb::new(self.tls_size)?; + let tcb_ptr = tcb as *mut Tcb; + + // Setup the DTVs. + tcb.setup_dtv(tcb_masters.len()); + + for obj in new_objects.iter() { + if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + // Above the TP + tcb.dtv_mut().unwrap()[obj.tls_module_id - 1] = + tcb_ptr.cast::().sub(obj.tls_offset); + } else { + // Below the TP + tcb.dtv_mut().unwrap()[obj.tls_module_id - 1] = + tcb_ptr.add(1).cast::().add(obj.tls_offset); + } + } + + tcb.append_masters(tcb_masters); + // Copy the master data into the static TLS block. + tcb.copy_masters()?; + + tcb.activate(); + // XXX: Beyond this point, any thread local's for ld.so should not be accessed. Though, the + // TCB can still be accessed. + } else { + let tcb = Tcb::current().expect("failed to get current tcb"); + + // TLS variables for dlopen'ed objects are lazily allocated in `__tls_get_addr`. + tcb.append_masters(tcb_masters); + } } self.relocate(&new_objects, &objects_data)?; @@ -222,23 +251,29 @@ impl Linker { self.next_tls_module_id, self.tls_size, )?; + trace!( + "[ldso] loading object: {} at {:#x}", + name, + obj.mmap.as_ptr() as usize + ); new_objects.push(obj); objects_data.push(data); + self.next_object_id += 1; + self.next_tls_module_id += 1; if let Some(master) = tcb_master { - if self.next_tls_module_id == 1 { - // Hack to allocate TCB on the first TLS module - unsafe { - if Tcb::current().is_none() { - let tcb = Tcb::new(master.offset).expect_notls("failed to allocate TCB"); - tcb.activate(); - } - } + if !dlopened { + self.tls_size += master.offset; // => aligned ph.p_memsz } - self.next_tls_module_id += 1; - self.tls_size = master.offset; + tcb_masters.push(master); + } else { + tcb_masters.push(Master { + ptr: ptr::null_mut(), + len: 0, + offset: 0, + }); } let (runpath, dependencies) = { diff --git a/src/ld_so/start.rs b/src/ld_so/start.rs index 0d0b0265d7..251f9c4edb 100644 --- a/src/ld_so/start.rs +++ b/src/ld_so/start.rs @@ -7,10 +7,12 @@ use alloc::{ string::{String, ToString}, vec::Vec, }; +use generic_rt::ExpectTlsFree; use crate::{ c_str::CStr, header::unistd, + ld_so::tcb::Master, platform::{get_auxv, get_auxvs, types::c_char}, start::Stack, sync::mutex::Mutex, @@ -141,9 +143,42 @@ fn resolve_path_name( } None } + +static mut LDSO_MASTER: Master = Master { + ptr: core::ptr::null_mut(), + len: 0, + offset: 0, +}; + // TODO: Make unsafe #[no_mangle] -pub extern "C" fn relibc_ld_so_start(sp: &'static mut Stack, ld_entry: usize) -> usize { +pub extern "C" fn relibc_ld_so_start( + sp: &'static mut Stack, + ld_entry: usize, + self_tls_start: usize, + self_tls_end: usize, +) -> usize { + // Setup TLS and TCB for ourselves. + // + // On Redox, we need the TCB to setup in order to do anything. Also, thread local's like + // ERRNO may be accessed by the dynamic linker, so we need static TLS to be setup. + unsafe { + let tls_size = self_tls_end - self_tls_start; + + let tcb = Tcb::new(tls_size).expect_notls("ld.so: failed to allocate bootstrap TCB"); + + LDSO_MASTER.ptr = self_tls_start as *mut u8; + LDSO_MASTER.len = tls_size; + LDSO_MASTER.offset = tls_size.next_multiple_of(4096); // alignment set in linker script + + tcb.masters_ptr = &mut LDSO_MASTER; + tcb.masters_len = core::mem::size_of::(); + + tcb.copy_masters() + .expect_notls("ld.so: failed to copy TLS master data"); + tcb.activate(); + } + // We get the arguments, the environment, and the auxilary vector let (argv, envs, auxv) = unsafe { let argv_start = sp.argv() as *mut usize; diff --git a/src/ld_so/tcb.rs b/src/ld_so/tcb.rs index 1e05657660..e54f2ea2b9 100644 --- a/src/ld_so/tcb.rs +++ b/src/ld_so/tcb.rs @@ -60,6 +60,10 @@ pub struct Tcb { pub mspace: *const Mutex, /// Underlying pthread_t struct, pthread_self() returns &self.pthread pub pthread: Pthread, + + // Dynamic TLS Vector + pub dtv_ptr: *mut *mut u8, + pub dtv_len: usize, } #[cfg(target_os = "redox")] @@ -71,6 +75,8 @@ const _: () = { impl Tcb { /// Create a new TCB + /// + /// `size` is the size of the TLS in bytes. pub unsafe fn new(size: usize) -> Result<&'static mut Self> { let page_size = Sys::getpagesize(); let (abi_page, tls, tcb_page) = Self::os_new(size.next_multiple_of(page_size))?; @@ -101,6 +107,9 @@ impl Tcb { stack_size: 0, os_tid: UnsafeCell::new(OsTid::default()), }, + + dtv_ptr: ptr::null_mut(), + dtv_len: 0, }, ); @@ -198,6 +207,31 @@ impl Tcb { Self::os_arch_activate(&self.os_specific, self.tls_end as usize, self.tls_len); } + pub fn setup_dtv(&mut self, n: usize) { + if self.dtv_ptr.is_null() { + let mut dtv = vec![ptr::null_mut(); n]; + self.dtv_ptr = dtv.as_mut_ptr(); + self.dtv_len = dtv.len(); + mem::forget(dtv); + } else { + // Resize DTV. + let mut dtv = unsafe { Vec::from_raw_parts(self.dtv_ptr, self.dtv_len, self.dtv_len) }; + dtv.resize(n, ptr::null_mut()); + self.dtv_ptr = dtv.as_mut_ptr(); + self.dtv_len = dtv.len(); + + mem::forget(dtv); + } + } + + pub fn dtv_mut(&mut self) -> Option<&'static mut [*mut u8]> { + if self.dtv_len != 0 { + Some(unsafe { slice::from_raw_parts_mut(self.dtv_ptr, self.dtv_len) }) + } else { + None + } + } + /// Mapping with correct flags for TCB and TLS unsafe fn map(size: usize) -> Result<&'static mut [u8]> { let ptr = sys_mman::mmap(